mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 18:45:53 +00:00
rcourtman
df0735cc79
Preserve the incoming delivery-health ordering and demo install-link changes alongside the qualified Patrol action history. Retain prior source bindings and record the merged build and browser checks so the shared proof receipt reflects the integrated tree.
7233 lines
231 KiB
Go
7233 lines
231 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
stderrors "errors"
|
|
"fmt"
|
|
"math"
|
|
"math/rand"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/discovery"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
|
recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/system"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/pmg"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
|
"github.com/rs/zerolog"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
const (
|
|
defaultTaskTimeout = 90 * time.Second
|
|
minTaskTimeout = 30 * time.Second
|
|
maxTaskTimeout = 3 * time.Minute
|
|
)
|
|
|
|
const mockKeepRealPollingEnv = "PULSE_MOCK_KEEP_REAL_POLLING"
|
|
|
|
func keepRealPollingInMockMode() bool {
|
|
raw := strings.TrimSpace(strings.ToLower(os.Getenv(mockKeepRealPollingEnv)))
|
|
switch raw {
|
|
case "1", "true", "yes", "on":
|
|
return true
|
|
case "", "0", "false", "no", "off":
|
|
return false
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// alertManagerRestoreOptions keeps mock mode a clean room across process
|
|
// restarts. SetMockMode clears active alerts when the toggle flips, but a
|
|
// process that boots with mock mode already enabled never runs that path, so
|
|
// alerts raised against real infrastructure are restored from disk and served
|
|
// beside fixture data until something clears them.
|
|
func alertManagerRestoreOptions() []alerts.ManagerOption {
|
|
options := []alerts.ManagerOption{alerts.WithDurableAlertStore()}
|
|
if mock.IsMockEnabled() {
|
|
options = append(options, alerts.WithoutPersistedAlertRestore())
|
|
}
|
|
return options
|
|
}
|
|
|
|
// newProxmoxClientFunc is a variable that holds the function to create a new Proxmox client.
|
|
// It is used to allow mocking the client creation in tests.
|
|
var newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
|
|
return proxmox.NewClient(cfg)
|
|
}
|
|
|
|
// PVEClientInterface defines the interface for PVE clients (both regular and cluster)
|
|
type PVEClientInterface interface {
|
|
GetNodes(ctx context.Context) ([]proxmox.Node, error)
|
|
GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error)
|
|
GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error)
|
|
GetVMs(ctx context.Context, node string) ([]proxmox.VM, error)
|
|
GetContainers(ctx context.Context, node string) ([]proxmox.Container, error)
|
|
GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error)
|
|
GetAllStorage(ctx context.Context) ([]proxmox.Storage, error)
|
|
GetBackupTasks(ctx context.Context) ([]proxmox.Task, error)
|
|
GetTaskLog(ctx context.Context, node, upid string) ([]proxmox.TaskLogLine, error)
|
|
GetReplicationStatus(ctx context.Context) ([]proxmox.ReplicationJob, error)
|
|
GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error)
|
|
GetVMSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error)
|
|
GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error)
|
|
GetVMStatus(ctx context.Context, node string, vmid int) (*proxmox.VMStatus, error)
|
|
GetContainerStatus(ctx context.Context, node string, vmid int) (*proxmox.Container, error)
|
|
GetContainerConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error)
|
|
GetContainerInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.ContainerInterface, error)
|
|
GetClusterResources(ctx context.Context, resourceType string) ([]proxmox.ClusterResource, error)
|
|
IsClusterMember(ctx context.Context) (bool, error)
|
|
GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error)
|
|
GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error)
|
|
GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error)
|
|
GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error)
|
|
GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error)
|
|
GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error)
|
|
GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error)
|
|
GetNodePendingUpdates(ctx context.Context, node string) ([]proxmox.AptPackage, error)
|
|
GetCephStatus(ctx context.Context) (*proxmox.CephStatus, error)
|
|
GetCephDF(ctx context.Context) (*proxmox.CephDF, error)
|
|
}
|
|
|
|
// ResourceStoreInterface provides methods for polling optimization and resource access.
|
|
// When an agent is monitoring a node, we can reduce API polling for that node.
|
|
type ResourceStoreInterface interface {
|
|
// ShouldSkipAPIPolling returns true if API polling should be skipped for the hostname
|
|
// because an agent is providing richer data.
|
|
ShouldSkipAPIPolling(hostname string) bool
|
|
// GetPollingRecommendations returns a map of hostname -> polling multiplier.
|
|
// 0 = skip entirely, 0.5 = half frequency, 1 = normal
|
|
GetPollingRecommendations() map[string]float64
|
|
// GetAll returns all resources in the store (for WebSocket broadcasts)
|
|
GetAll() []unifiedresources.Resource
|
|
// PopulateFromSnapshot updates the store with data from a StateSnapshot
|
|
PopulateFromSnapshot(snapshot models.StateSnapshot)
|
|
}
|
|
|
|
// StaleThresholdResourceStore is implemented by stores that evaluate source
|
|
// freshness while ingesting monitor snapshots.
|
|
type StaleThresholdResourceStore interface {
|
|
SetStaleThresholds(thresholds map[unifiedresources.DataSource]time.Duration)
|
|
}
|
|
|
|
// SupplementalRecordStore is an optional extension for resource stores that can
|
|
// ingest source-native unified records in addition to legacy snapshots.
|
|
type SupplementalRecordStore interface {
|
|
PopulateSupplementalRecords(source unifiedresources.DataSource, records []unifiedresources.IngestRecord)
|
|
}
|
|
|
|
// AtomicSnapshotResourceStore is an optional extension for stores that can
|
|
// atomically replace their canonical registry from a snapshot plus
|
|
// supplemental records in a single swap.
|
|
type AtomicSnapshotResourceStore interface {
|
|
PopulateSnapshotAndSupplemental(snapshot models.StateSnapshot, recordsBySource map[unifiedresources.DataSource][]unifiedresources.IngestRecord)
|
|
}
|
|
|
|
// MetricsTargetResourceStore optionally resolves the history/metrics target for
|
|
// a canonical resource in the live unified store.
|
|
type MetricsTargetResourceStore interface {
|
|
MetricsTargetForResource(resourceID string) *unifiedresources.MetricsTarget
|
|
}
|
|
|
|
// UnifiedResourceFreshnessStore is an optional extension for stores that track
|
|
// their own canonical-resource freshness independent of state.LastUpdate.
|
|
type UnifiedResourceFreshnessStore interface {
|
|
UnifiedResourceFreshness() time.Time
|
|
}
|
|
|
|
// MonitorSupplementalRecordsProvider emits source-native records outside the
|
|
// poll-provider scheduling path (for example, dedicated background pollers).
|
|
type MonitorSupplementalRecordsProvider interface {
|
|
SupplementalRecords(m *Monitor, orgID string) []unifiedresources.IngestRecord
|
|
}
|
|
|
|
// MonitorSupplementalChangesProvider optionally emits canonical resource
|
|
// timeline changes alongside supplemental records.
|
|
type MonitorSupplementalChangesProvider interface {
|
|
SupplementalChanges(m *Monitor, orgID string) []unifiedresources.ResourceChange
|
|
}
|
|
|
|
// MonitorSupplementalInventoryReadinessProvider optionally reports when a
|
|
// supplemental provider's current org-scoped inventory is settled enough to be
|
|
// consumed by monitored-system grouping and support boundaries.
|
|
//
|
|
// Providers that suppress snapshot-owned sources must implement this contract
|
|
// so the monitor can fail closed until the canonical store has been rebuilt
|
|
// from a settled provider baseline.
|
|
type MonitorSupplementalInventoryReadinessProvider interface {
|
|
SupplementalInventoryReadyAt(m *Monitor, orgID string) (time.Time, bool)
|
|
}
|
|
|
|
// MonitorPhysicalDiskTemperatureHistoryProvider optionally exposes source-native
|
|
// physical-disk temperature history through the canonical monitoring chart
|
|
// boundary when Pulse's own stored history is shallow.
|
|
type MonitorPhysicalDiskTemperatureHistoryProvider interface {
|
|
PhysicalDiskTemperatureHistory(m *Monitor, orgID string, duration time.Duration) map[string][]MetricPoint
|
|
}
|
|
|
|
func getNodeDisplayName(instance *config.PVEInstance, nodeName string) string {
|
|
baseName := strings.TrimSpace(nodeName)
|
|
if baseName == "" {
|
|
baseName = "unknown-node"
|
|
}
|
|
|
|
if instance == nil {
|
|
return baseName
|
|
}
|
|
|
|
if instance.IsCluster {
|
|
_, displayName := config.PVEClusterNodePresentation(instance, baseName)
|
|
if displayName != "" {
|
|
return displayName
|
|
}
|
|
return baseName
|
|
}
|
|
|
|
friendly := strings.TrimSpace(instance.Name)
|
|
if friendly != "" {
|
|
return friendly
|
|
}
|
|
|
|
if baseName != "" && baseName != "unknown-node" {
|
|
return baseName
|
|
}
|
|
|
|
if label := normalizeEndpointHost(instance.Host); label != "" && !isLikelyIPAddress(label) {
|
|
return label
|
|
}
|
|
|
|
return baseName
|
|
}
|
|
|
|
func (m *Monitor) getInstanceConfig(instanceName string) *config.PVEInstance {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
if m.config == nil {
|
|
return nil
|
|
}
|
|
|
|
for i := range m.config.PVEInstances {
|
|
if strings.EqualFold(m.config.PVEInstances[i].Name, instanceName) {
|
|
instanceCopy := m.config.PVEInstances[i]
|
|
return &instanceCopy
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *Monitor) totalClientCount() int {
|
|
if m == nil {
|
|
return 0
|
|
}
|
|
|
|
total := 0
|
|
for _, provider := range m.pollProviderSnapshotWithBuiltins() {
|
|
if provider == nil {
|
|
continue
|
|
}
|
|
total += len(provider.ListInstances(m))
|
|
}
|
|
return total
|
|
}
|
|
|
|
func (m *Monitor) getPVEClient(name string) (PVEClientInterface, bool) {
|
|
if m == nil {
|
|
return nil, false
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
client, ok := m.pveClients[name]
|
|
return client, ok
|
|
}
|
|
|
|
func (m *Monitor) getPBSClient(name string) (*pbs.Client, bool) {
|
|
if m == nil {
|
|
return nil, false
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
client, ok := m.pbsClients[name]
|
|
return client, ok
|
|
}
|
|
|
|
func (m *Monitor) getPMGClient(name string) (*pmg.Client, bool) {
|
|
if m == nil {
|
|
return nil, false
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
client, ok := m.pmgClients[name]
|
|
return client, ok
|
|
}
|
|
|
|
// normalizeSMARTDeviceIdentifier strips a trailing type annotation (e.g.
|
|
// "sdd [scsi]" -> "sdd") and the /dev/ prefix so SMART temperatures match disks
|
|
// by device path even when smartctl reports a typed label.
|
|
func normalizeSMARTDeviceIdentifier(device string) string {
|
|
normalized := strings.TrimSpace(device)
|
|
if normalized == "" {
|
|
return ""
|
|
}
|
|
|
|
if base, suffix, found := strings.Cut(normalized, " ["); found && strings.HasSuffix(suffix, "]") {
|
|
normalized = base
|
|
}
|
|
|
|
normalized = strings.TrimSpace(normalized)
|
|
return strings.TrimPrefix(normalized, "/dev/")
|
|
}
|
|
|
|
func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) []models.PhysicalDisk {
|
|
if len(disks) == 0 || len(nodes) == 0 {
|
|
return disks
|
|
}
|
|
|
|
// Build temperature maps by node for both SMART and legacy NVMe data
|
|
smartTempsByNode := make(map[string][]models.DiskTemp)
|
|
nvmeTempsByNode := make(map[string][]models.NVMeTemp)
|
|
|
|
for _, node := range nodes {
|
|
log.Debug().
|
|
Str("nodeName", node.Name).
|
|
Bool("hasTemp", node.Temperature != nil).
|
|
Bool("tempAvailable", node.Temperature != nil && node.Temperature.Available).
|
|
Int("smartCount", func() int {
|
|
if node.Temperature != nil {
|
|
return len(node.Temperature.SMART)
|
|
}
|
|
return 0
|
|
}()).
|
|
Msg("mergeNVMeTempsIntoDisks: checking node temperature")
|
|
|
|
if node.Temperature == nil || !node.Temperature.Available {
|
|
continue
|
|
}
|
|
|
|
// Collect SMART temps (preferred source)
|
|
if len(node.Temperature.SMART) > 0 {
|
|
temps := make([]models.DiskTemp, len(node.Temperature.SMART))
|
|
copy(temps, node.Temperature.SMART)
|
|
smartTempsByNode[node.Name] = temps
|
|
log.Debug().
|
|
Str("nodeName", node.Name).
|
|
Int("smartTempCount", len(temps)).
|
|
Msg("mergeNVMeTempsIntoDisks: collected SMART temps for node")
|
|
}
|
|
|
|
// Collect legacy NVMe temps as fallback
|
|
if len(node.Temperature.NVMe) > 0 {
|
|
temps := make([]models.NVMeTemp, len(node.Temperature.NVMe))
|
|
copy(temps, node.Temperature.NVMe)
|
|
sort.Slice(temps, func(i, j int) bool {
|
|
return temps[i].Device < temps[j].Device
|
|
})
|
|
nvmeTempsByNode[node.Name] = temps
|
|
}
|
|
}
|
|
|
|
if len(smartTempsByNode) == 0 && len(nvmeTempsByNode) == 0 {
|
|
log.Debug().
|
|
Int("diskCount", len(disks)).
|
|
Msg("mergeNVMeTempsIntoDisks: no SMART or NVMe temperature data available")
|
|
return disks
|
|
}
|
|
|
|
log.Debug().
|
|
Int("smartNodeCount", len(smartTempsByNode)).
|
|
Int("nvmeNodeCount", len(nvmeTempsByNode)).
|
|
Int("diskCount", len(disks)).
|
|
Msg("mergeNVMeTempsIntoDisks: starting disk temperature merge")
|
|
|
|
updated := make([]models.PhysicalDisk, len(disks))
|
|
copy(updated, disks)
|
|
|
|
// Process SMART temperatures first (preferred method)
|
|
for i := range updated {
|
|
smartTemps, ok := smartTempsByNode[updated[i].Node]
|
|
log.Debug().
|
|
Str("diskDevPath", updated[i].DevPath).
|
|
Str("diskNode", updated[i].Node).
|
|
Bool("hasSMARTData", ok).
|
|
Int("smartTempCount", len(smartTemps)).
|
|
Msg("mergeNVMeTempsIntoDisks: checking disk for SMART temp match")
|
|
if !ok || len(smartTemps) == 0 {
|
|
continue
|
|
}
|
|
|
|
// Try to match by WWN (most reliable), but never guess when an
|
|
// identity appears more than once in the node snapshot.
|
|
if diskinventory.IsUsableHardwareID(updated[i].WWN) {
|
|
if temp := uniqueDiskTemperatureMatch(smartTemps, func(candidate models.DiskTemp) bool {
|
|
return diskinventory.IsUsableHardwareID(candidate.WWN) &&
|
|
strings.EqualFold(candidate.WWN, updated[i].WWN)
|
|
}); temp != nil && temp.Temperature > 0 && !temp.StandbySkipped {
|
|
setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart")
|
|
log.Debug().
|
|
Str("disk", updated[i].DevPath).
|
|
Str("wwn", updated[i].WWN).
|
|
Int("temp", temp.Temperature).
|
|
Msg("Matched SMART temperature by WWN")
|
|
}
|
|
}
|
|
|
|
// Fall back to serial number match (case-insensitive)
|
|
if diskinventory.IsUsableHardwareID(updated[i].Serial) && updated[i].Temperature == 0 {
|
|
if temp := uniqueDiskTemperatureMatch(smartTemps, func(candidate models.DiskTemp) bool {
|
|
return diskinventory.IsUsableHardwareID(candidate.Serial) &&
|
|
strings.EqualFold(candidate.Serial, updated[i].Serial)
|
|
}); temp != nil && temp.Temperature > 0 && !temp.StandbySkipped {
|
|
setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart")
|
|
log.Debug().
|
|
Str("disk", updated[i].DevPath).
|
|
Str("serial", updated[i].Serial).
|
|
Int("temp", temp.Temperature).
|
|
Msg("Matched SMART temperature by serial")
|
|
}
|
|
}
|
|
|
|
// Last resort: match by device path (normalized)
|
|
if updated[i].Temperature == 0 {
|
|
normalizedDevPath := normalizeSMARTDeviceIdentifier(updated[i].DevPath)
|
|
if temp := uniqueDiskTemperatureMatch(smartTemps, func(candidate models.DiskTemp) bool {
|
|
normalizedTempDev := normalizeSMARTDeviceIdentifier(candidate.Device)
|
|
return normalizedTempDev != "" && normalizedTempDev == normalizedDevPath
|
|
}); temp != nil && temp.Temperature > 0 && !temp.StandbySkipped {
|
|
setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart")
|
|
log.Debug().
|
|
Str("disk", updated[i].DevPath).
|
|
Int("temp", temp.Temperature).
|
|
Msg("Matched SMART temperature by device path")
|
|
}
|
|
}
|
|
}
|
|
|
|
// Process legacy NVMe temperatures for disks that didn't get SMART data
|
|
disksByNode := make(map[string][]int)
|
|
for i := range updated {
|
|
if strings.EqualFold(updated[i].Type, "nvme") && updated[i].Temperature == 0 {
|
|
disksByNode[updated[i].Node] = append(disksByNode[updated[i].Node], i)
|
|
}
|
|
}
|
|
|
|
for nodeName, diskIndexes := range disksByNode {
|
|
temps, ok := nvmeTempsByNode[nodeName]
|
|
if !ok || len(temps) == 0 {
|
|
continue
|
|
}
|
|
|
|
sort.Slice(diskIndexes, func(i, j int) bool {
|
|
return updated[diskIndexes[i]].DevPath < updated[diskIndexes[j]].DevPath
|
|
})
|
|
|
|
for idx, diskIdx := range diskIndexes {
|
|
if idx >= len(temps) {
|
|
break
|
|
}
|
|
|
|
tempVal := temps[idx].Temp
|
|
if tempVal <= 0 || math.IsNaN(tempVal) {
|
|
continue
|
|
}
|
|
|
|
setPhysicalDiskTemperature(&updated[diskIdx], int(math.Round(tempVal)), "proxmox_node_nvme")
|
|
log.Debug().
|
|
Str("disk", updated[diskIdx].DevPath).
|
|
Int("temp", updated[diskIdx].Temperature).
|
|
Msg("Matched legacy NVMe temperature by index")
|
|
}
|
|
}
|
|
|
|
return updated
|
|
}
|
|
|
|
func uniqueDiskTemperatureMatch(temperatures []models.DiskTemp, predicate func(models.DiskTemp) bool) *models.DiskTemp {
|
|
matchIndex := -1
|
|
for index := range temperatures {
|
|
if !predicate(temperatures[index]) {
|
|
continue
|
|
}
|
|
if matchIndex >= 0 {
|
|
return nil
|
|
}
|
|
matchIndex = index
|
|
}
|
|
if matchIndex < 0 {
|
|
return nil
|
|
}
|
|
return &temperatures[matchIndex]
|
|
}
|
|
|
|
func setPhysicalDiskTemperature(disk *models.PhysicalDisk, temperature int, source string) {
|
|
if disk == nil || temperature <= 0 {
|
|
return
|
|
}
|
|
disk.Temperature = temperature
|
|
if disk.Collection == nil {
|
|
disk.Collection = &diskinventory.CollectionStatus{}
|
|
}
|
|
disk.Collection.Temperature = diskinventory.Available(source)
|
|
}
|
|
|
|
// mergeHostAgentSMARTIntoDisks merges SMART temperature data from linked host agents
|
|
// into physical disks for Proxmox nodes. This allows disk temps collected by the
|
|
// pulse-agent running on a PVE node to populate the Physical Disks view.
|
|
func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.Node, hosts []models.Host) []models.PhysicalDisk {
|
|
if len(disks) == 0 || len(nodes) == 0 || len(hosts) == 0 {
|
|
return disks
|
|
}
|
|
|
|
// Build a map of host ID to host for quick lookup
|
|
hostByID := make(map[string]*models.Host, len(hosts))
|
|
for i := range hosts {
|
|
hostByID[hosts[i].ID] = &hosts[i]
|
|
}
|
|
|
|
// Build a map of node name to linked host's SMART data
|
|
smartByNodeName := make(map[string][]models.HostDiskSMART)
|
|
for _, node := range nodes {
|
|
if node.LinkedAgentID == "" {
|
|
continue
|
|
}
|
|
host, ok := hostByID[node.LinkedAgentID]
|
|
if !ok || len(host.Sensors.SMART) == 0 {
|
|
continue
|
|
}
|
|
smartByNodeName[node.Name] = host.Sensors.SMART
|
|
log.Debug().
|
|
Str("nodeName", node.Name).
|
|
Str("hostAgentID", node.LinkedAgentID).
|
|
Int("smartDiskCount", len(host.Sensors.SMART)).
|
|
Msg("mergeHostAgentSMARTIntoDisks: found linked host agent with SMART data")
|
|
}
|
|
|
|
if len(smartByNodeName) == 0 {
|
|
return disks
|
|
}
|
|
|
|
updated := make([]models.PhysicalDisk, len(disks))
|
|
copy(updated, disks)
|
|
|
|
for i := range updated {
|
|
smartData, ok := smartByNodeName[updated[i].Node]
|
|
if !ok || len(smartData) == 0 {
|
|
continue
|
|
}
|
|
|
|
// Find matching SMART entry by WWN, serial, or topology-scoped device path.
|
|
var matched *models.HostDiskSMART
|
|
uniqueMatch := func(predicate func(models.HostDiskSMART) bool) *models.HostDiskSMART {
|
|
matchIndex := -1
|
|
for index := range smartData {
|
|
if !predicate(smartData[index]) {
|
|
continue
|
|
}
|
|
if matchIndex >= 0 {
|
|
return nil
|
|
}
|
|
matchIndex = index
|
|
}
|
|
if matchIndex < 0 {
|
|
return nil
|
|
}
|
|
return &smartData[matchIndex]
|
|
}
|
|
|
|
// Try to match by WWN (most reliable)
|
|
if diskinventory.IsUsableHardwareID(updated[i].WWN) {
|
|
matched = uniqueMatch(func(candidate models.HostDiskSMART) bool {
|
|
return diskinventory.IsUsableHardwareID(candidate.WWN) &&
|
|
strings.EqualFold(candidate.WWN, updated[i].WWN) &&
|
|
diskTopologyCompatible(updated[i].Controller, updated[i].Target, candidate.Controller, candidate.Target)
|
|
})
|
|
}
|
|
|
|
// Fall back to serial number match
|
|
if matched == nil && diskinventory.IsUsableHardwareID(updated[i].Serial) {
|
|
matched = uniqueMatch(func(candidate models.HostDiskSMART) bool {
|
|
return diskinventory.IsUsableHardwareID(candidate.Serial) &&
|
|
strings.EqualFold(candidate.Serial, updated[i].Serial) &&
|
|
diskTopologyCompatible(updated[i].Controller, updated[i].Target, candidate.Controller, candidate.Target)
|
|
})
|
|
}
|
|
|
|
// Last resort: match by device path
|
|
if matched == nil {
|
|
normalizedDevPath := normalizeSMARTDeviceIdentifier(updated[i].DevPath)
|
|
matched = uniqueMatch(func(candidate models.HostDiskSMART) bool {
|
|
normalizedDiskDev := normalizeSMARTDeviceIdentifier(candidate.Device)
|
|
return normalizedDiskDev != "" &&
|
|
normalizedDiskDev == normalizedDevPath &&
|
|
diskTopologyCompatible(updated[i].Controller, updated[i].Target, candidate.Controller, candidate.Target)
|
|
})
|
|
}
|
|
|
|
if matched == nil {
|
|
continue
|
|
}
|
|
|
|
if strings.TrimSpace(updated[i].Model) == "" && strings.TrimSpace(matched.Model) != "" {
|
|
updated[i].Model = strings.TrimSpace(matched.Model)
|
|
}
|
|
if strings.TrimSpace(matched.Serial) != "" &&
|
|
(strings.TrimSpace(updated[i].Serial) == "" || shouldPromoteHostAgentSerial(updated[i], *matched)) {
|
|
updated[i].Serial = strings.TrimSpace(matched.Serial)
|
|
}
|
|
if strings.TrimSpace(updated[i].WWN) == "" && strings.TrimSpace(matched.WWN) != "" {
|
|
updated[i].WWN = strings.TrimSpace(matched.WWN)
|
|
}
|
|
if shouldPromoteHostAgentDiskType(updated[i].Type, matched.Type) {
|
|
updated[i].Type = strings.TrimSpace(matched.Type)
|
|
}
|
|
if strings.TrimSpace(updated[i].Controller) == "" && strings.TrimSpace(matched.Controller) != "" {
|
|
updated[i].Controller = strings.TrimSpace(matched.Controller)
|
|
}
|
|
if strings.TrimSpace(updated[i].Target) == "" && strings.TrimSpace(matched.Target) != "" {
|
|
updated[i].Target = strings.TrimSpace(matched.Target)
|
|
}
|
|
if updated[i].Size <= 0 && matched.SizeBytes > 0 {
|
|
updated[i].Size = matched.SizeBytes
|
|
}
|
|
if strings.TrimSpace(updated[i].StorageGroup) == "" && strings.TrimSpace(matched.Pool) != "" {
|
|
updated[i].StorageGroup = strings.TrimSpace(matched.Pool)
|
|
}
|
|
|
|
// Merge temperature if not already set
|
|
if !matched.Standby && updated[i].Temperature == 0 && matched.Temperature > 0 {
|
|
updated[i].Temperature = matched.Temperature
|
|
log.Debug().
|
|
Str("device", updated[i].DevPath).
|
|
Int("temp", matched.Temperature).
|
|
Msg("Matched host agent SMART temperature")
|
|
}
|
|
|
|
// Always merge SMART attributes from host agent
|
|
if matched.Attributes != nil {
|
|
updated[i].SmartAttributes = smartAttributesCopy(matched.Attributes)
|
|
if derivedWearout := deriveWearoutFromSMARTAttributes(matched.Attributes); derivedWearout >= 0 {
|
|
updated[i].Wearout = derivedWearout
|
|
}
|
|
}
|
|
if matched.IO != nil {
|
|
ioCopy := *matched.IO
|
|
updated[i].IO = &ioCopy
|
|
}
|
|
updated[i].Collection = diskinventory.MergeStatus(updated[i].Collection, matched.Collection)
|
|
|
|
if shouldUseHostAgentPhysicalDiskHealth(updated[i].Health, matched.Health) {
|
|
updated[i].Health = matched.Health
|
|
}
|
|
}
|
|
|
|
return updated
|
|
}
|
|
|
|
func shouldUseHostAgentPhysicalDiskHealth(existing, incoming string) bool {
|
|
incoming = strings.ToUpper(strings.TrimSpace(incoming))
|
|
if incoming == "" || incoming == "UNKNOWN" {
|
|
return false
|
|
}
|
|
existing = strings.ToUpper(strings.TrimSpace(existing))
|
|
if incoming == "FAILED" {
|
|
return true
|
|
}
|
|
return existing == "" || existing == "UNKNOWN"
|
|
}
|
|
|
|
func diskTopologyCompatible(leftController, leftTarget, rightController, rightTarget string) bool {
|
|
leftController = strings.TrimSpace(leftController)
|
|
leftTarget = strings.TrimSpace(leftTarget)
|
|
rightController = strings.TrimSpace(rightController)
|
|
rightTarget = strings.TrimSpace(rightTarget)
|
|
if leftController != "" && rightController != "" && !strings.EqualFold(leftController, rightController) {
|
|
return false
|
|
}
|
|
if leftTarget != "" && rightTarget != "" && !strings.EqualFold(leftTarget, rightTarget) {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func shouldPromoteHostAgentSerial(disk models.PhysicalDisk, smart models.HostDiskSMART) bool {
|
|
if smart.Collection != nil &&
|
|
strings.EqualFold(strings.TrimSpace(smart.Type), "sas") &&
|
|
smart.Collection.Serial.State == diskinventory.FieldAvailable &&
|
|
strings.HasPrefix(strings.ToLower(strings.TrimSpace(smart.Collection.Serial.Source)), "smartctl") {
|
|
return true
|
|
}
|
|
// Older agents did not carry collection provenance. Proxmox commonly puts
|
|
// a 64-bit SAS address in its serial field; a non-address smartctl serial is
|
|
// the actual drive identity and must replace that transport address.
|
|
return strings.EqualFold(strings.TrimSpace(disk.Type), "sas") &&
|
|
strings.EqualFold(strings.TrimSpace(smart.Type), "sas") &&
|
|
looksLikeSASAddress(disk.Serial) &&
|
|
!looksLikeSASAddress(smart.Serial)
|
|
}
|
|
|
|
func looksLikeSASAddress(value string) bool {
|
|
value = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(value)), "0x")
|
|
if len(value) != 16 {
|
|
return false
|
|
}
|
|
for _, char := range value {
|
|
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func shouldPromoteHostAgentDiskType(existing, incoming string) bool {
|
|
incoming = strings.ToLower(strings.TrimSpace(incoming))
|
|
if incoming == "" {
|
|
return false
|
|
}
|
|
existing = strings.ToLower(strings.TrimSpace(existing))
|
|
if existing == "" || existing == "unknown" || existing == "scsi" {
|
|
return true
|
|
}
|
|
// Proxmox reports coarse form-factor types (hdd/ssd) or misreads SAS
|
|
// members as sata; the agent's smartctl transport evidence is
|
|
// authoritative for the link type.
|
|
if incoming == "sas" && (existing == "sata" || existing == "hdd" || existing == "ssd") {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func deriveWearoutFromSMARTAttributes(attrs *models.SMARTAttributes) int {
|
|
if attrs == nil || attrs.PercentageUsed == nil {
|
|
return -1
|
|
}
|
|
return storagehealth.RemainingLifeFromPercentageUsed(*attrs.PercentageUsed)
|
|
}
|
|
|
|
func physicalDiskFromReadStateView(view *unifiedresources.PhysicalDiskView) models.PhysicalDisk {
|
|
if view == nil {
|
|
// Wearout 0 means no endurance remaining, so an absent view has to
|
|
// carry the unreported sentinel rather than the struct zero value.
|
|
return models.PhysicalDisk{Wearout: unifiedresources.WearoutUnreported}
|
|
}
|
|
|
|
disk := models.PhysicalDisk{
|
|
ID: view.ID(),
|
|
Node: view.Node(),
|
|
Instance: view.Instance(),
|
|
DevPath: view.DevPath(),
|
|
Model: view.Model(),
|
|
Serial: view.Serial(),
|
|
WWN: view.WWN(),
|
|
Type: view.DiskType(),
|
|
Controller: view.Controller(),
|
|
Target: view.Target(),
|
|
Size: view.SizeBytes(),
|
|
Health: view.Health(),
|
|
Wearout: view.Wearout(),
|
|
Temperature: view.Temperature(),
|
|
RPM: view.RPM(),
|
|
Used: view.Used(),
|
|
StorageGroup: view.StorageGroup(),
|
|
SmartAttributes: smartAttributesFromUnifiedMeta(view.SMART()),
|
|
IO: physicalDiskIOFromUnifiedMeta(view.IO()),
|
|
Collection: diskinventory.CloneStatus(view.Collection()),
|
|
LastChecked: view.LastSeen(),
|
|
}
|
|
if source, ok := view.SourceStatus(unifiedresources.SourceProxmox); ok {
|
|
disk.ExpectedUpdateInterval = time.Duration(source.ExpectedUpdateIntervalSeconds) * time.Second
|
|
}
|
|
return disk
|
|
|
|
}
|
|
|
|
func physicalDiskIOFromUnifiedMeta(in *unifiedresources.PhysicalDiskIOMeta) *models.DiskIO {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
return &models.DiskIO{
|
|
Device: in.Device,
|
|
ReadBytes: in.ReadBytes,
|
|
WriteBytes: in.WriteBytes,
|
|
ReadOps: in.ReadOps,
|
|
WriteOps: in.WriteOps,
|
|
ReadTime: in.ReadTimeMs,
|
|
WriteTime: in.WriteTimeMs,
|
|
IOTime: in.IOTimeMs,
|
|
}
|
|
}
|
|
|
|
func smartAttributesFromUnifiedMeta(in *unifiedresources.SMARTMeta) *models.SMARTAttributes {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := &models.SMARTAttributes{
|
|
PowerOnHours: cloneInt64Pointer(in.PowerOnHours),
|
|
PowerCycles: cloneInt64Pointer(in.PowerCycles),
|
|
ReallocatedSectors: cloneInt64Pointer(in.ReallocatedSectors),
|
|
PendingSectors: cloneInt64Pointer(in.PendingSectors),
|
|
OfflineUncorrectable: cloneInt64Pointer(in.OfflineUncorrectable),
|
|
UDMACRCErrors: cloneInt64Pointer(in.UDMACRCErrors),
|
|
PercentageUsed: cloneIntPointer(in.PercentageUsed),
|
|
AvailableSpare: cloneIntPointer(in.AvailableSpare),
|
|
MediaErrors: cloneInt64Pointer(in.MediaErrors),
|
|
UnsafeShutdowns: cloneInt64Pointer(in.UnsafeShutdowns),
|
|
}
|
|
if out.PowerOnHours == nil &&
|
|
out.PowerCycles == nil &&
|
|
out.ReallocatedSectors == nil &&
|
|
out.PendingSectors == nil &&
|
|
out.OfflineUncorrectable == nil &&
|
|
out.UDMACRCErrors == nil &&
|
|
out.PercentageUsed == nil &&
|
|
out.AvailableSpare == nil &&
|
|
out.MediaErrors == nil &&
|
|
out.UnsafeShutdowns == nil {
|
|
return nil
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneIntPointer(in *int) *int {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
value := *in
|
|
return &value
|
|
}
|
|
|
|
func cloneInt64Pointer(in *int64) *int64 {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
value := *in
|
|
return &value
|
|
}
|
|
|
|
func physicalDisksForInstanceFromReadState(readState unifiedresources.ReadState, instance string) []models.PhysicalDisk {
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]models.PhysicalDisk, 0)
|
|
for _, disk := range readState.PhysicalDisks() {
|
|
if disk == nil || disk.Instance() != instance {
|
|
continue
|
|
}
|
|
out = append(out, physicalDiskFromReadStateView(disk))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func nodesForInstanceFromReadState(readState unifiedresources.ReadState, instance string) []models.Node {
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]models.Node, 0)
|
|
for _, node := range readState.Nodes() {
|
|
if node == nil || node.Instance() != instance {
|
|
continue
|
|
}
|
|
out = append(out, nodeFromReadStateView(node))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostsFromReadState(readState unifiedresources.ReadState) []models.Host {
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]models.Host, 0)
|
|
for _, host := range readState.Hosts() {
|
|
if host == nil {
|
|
continue
|
|
}
|
|
out = append(out, hostFromReadStateView(host))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// writeSMARTMetrics writes SMART temperature history to the in-memory chart
|
|
// buffer and persists SMART attributes when the metrics store is enabled.
|
|
func (m *Monitor) writeSMARTMetrics(disk models.PhysicalDisk, now time.Time) {
|
|
if shouldSkipNativeMockStateMetricWrites() {
|
|
return
|
|
}
|
|
|
|
resourceID := unifiedresources.PhysicalDiskMetricID(disk)
|
|
if resourceID == "" {
|
|
return
|
|
}
|
|
|
|
// Temperature (always write if > 0)
|
|
if disk.Temperature > 0 {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now)
|
|
}
|
|
if m.metricsStore != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_temp", float64(disk.Temperature), now)
|
|
}
|
|
}
|
|
|
|
attrs := disk.SmartAttributes
|
|
if attrs == nil || m.metricsStore == nil {
|
|
return
|
|
}
|
|
|
|
// Common
|
|
if attrs.PowerOnHours != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_power_on_hours", float64(*attrs.PowerOnHours), now)
|
|
}
|
|
if attrs.PowerCycles != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_power_cycles", float64(*attrs.PowerCycles), now)
|
|
}
|
|
|
|
// SATA-specific
|
|
if attrs.ReallocatedSectors != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_reallocated_sectors", float64(*attrs.ReallocatedSectors), now)
|
|
}
|
|
if attrs.PendingSectors != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_pending_sectors", float64(*attrs.PendingSectors), now)
|
|
}
|
|
if attrs.OfflineUncorrectable != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_offline_uncorrectable", float64(*attrs.OfflineUncorrectable), now)
|
|
}
|
|
if attrs.UDMACRCErrors != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_crc_errors", float64(*attrs.UDMACRCErrors), now)
|
|
}
|
|
|
|
// NVMe-specific
|
|
if attrs.PercentageUsed != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_percentage_used", float64(*attrs.PercentageUsed), now)
|
|
}
|
|
if attrs.AvailableSpare != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_available_spare", float64(*attrs.AvailableSpare), now)
|
|
}
|
|
if attrs.MediaErrors != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_media_errors", float64(*attrs.MediaErrors), now)
|
|
}
|
|
if attrs.UnsafeShutdowns != nil {
|
|
m.metricsStore.Write("disk", resourceID, "smart_unsafe_shutdowns", float64(*attrs.UnsafeShutdowns), now)
|
|
}
|
|
}
|
|
|
|
// PollExecutor defines the contract for executing polling tasks.
|
|
type PollExecutor interface {
|
|
Execute(ctx context.Context, task PollTask)
|
|
}
|
|
|
|
type realExecutor struct {
|
|
monitor *Monitor
|
|
}
|
|
|
|
func newRealExecutor(m *Monitor) PollExecutor {
|
|
return &realExecutor{monitor: m}
|
|
}
|
|
|
|
func (r *realExecutor) Execute(ctx context.Context, task PollTask) {
|
|
if r == nil || r.monitor == nil {
|
|
return
|
|
}
|
|
|
|
if task.Run != nil {
|
|
task.Run(ctx)
|
|
return
|
|
}
|
|
|
|
switch strings.ToLower(task.InstanceType) {
|
|
case "pve":
|
|
if task.PVEClient == nil {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Msg("PollExecutor received nil PVE client")
|
|
return
|
|
}
|
|
r.monitor.pollPVEInstance(ctx, task.InstanceName, task.PVEClient)
|
|
case "pbs":
|
|
if task.PBSClient == nil {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Msg("PollExecutor received nil PBS client")
|
|
return
|
|
}
|
|
r.monitor.pollPBSInstance(ctx, task.InstanceName, task.PBSClient)
|
|
case "pmg":
|
|
if task.PMGClient == nil {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Msg("PollExecutor received nil PMG client")
|
|
return
|
|
}
|
|
r.monitor.pollPMGInstance(ctx, task.InstanceName, task.PMGClient)
|
|
default:
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", task.InstanceType).
|
|
Msg("PollExecutor received unsupported task type")
|
|
}
|
|
}
|
|
}
|
|
|
|
type instanceInfo struct {
|
|
Key string
|
|
Type InstanceType
|
|
DisplayName string
|
|
Connection string
|
|
Metadata map[string]string
|
|
}
|
|
|
|
type pollStatus struct {
|
|
LastSuccess time.Time
|
|
LastErrorAt time.Time
|
|
LastErrorMessage string
|
|
LastErrorCategory string
|
|
ConsecutiveFailures int
|
|
FirstFailureAt time.Time
|
|
}
|
|
|
|
type dlqInsight struct {
|
|
Reason string
|
|
FirstAttempt time.Time
|
|
LastAttempt time.Time
|
|
RetryCount int
|
|
NextRetry time.Time
|
|
}
|
|
|
|
type ErrorDetail struct {
|
|
At time.Time `json:"at"`
|
|
Message string `json:"message"`
|
|
Category string `json:"category"`
|
|
}
|
|
|
|
type InstancePollStatus struct {
|
|
LastSuccess *time.Time `json:"lastSuccess,omitempty"`
|
|
LastError *ErrorDetail `json:"lastError,omitempty"`
|
|
ConsecutiveFailures int `json:"consecutiveFailures"`
|
|
FirstFailureAt *time.Time `json:"firstFailureAt,omitempty"`
|
|
}
|
|
|
|
type InstanceBreaker struct {
|
|
State string `json:"state"`
|
|
Since *time.Time `json:"since,omitempty"`
|
|
LastTransition *time.Time `json:"lastTransition,omitempty"`
|
|
RetryAt *time.Time `json:"retryAt,omitempty"`
|
|
FailureCount int `json:"failureCount"`
|
|
}
|
|
|
|
type InstanceDLQ struct {
|
|
Present bool `json:"present"`
|
|
Reason string `json:"reason,omitempty"`
|
|
FirstAttempt *time.Time `json:"firstAttempt,omitempty"`
|
|
LastAttempt *time.Time `json:"lastAttempt,omitempty"`
|
|
RetryCount int `json:"retryCount,omitempty"`
|
|
NextRetry *time.Time `json:"nextRetry,omitempty"`
|
|
}
|
|
|
|
type InstanceHealth struct {
|
|
Key string `json:"key"`
|
|
Type string `json:"type"`
|
|
DisplayName string `json:"displayName"`
|
|
Instance string `json:"instance"`
|
|
Connection string `json:"connection"`
|
|
PollStatus InstancePollStatus `json:"pollStatus"`
|
|
Breaker InstanceBreaker `json:"breaker"`
|
|
DeadLetter InstanceDLQ `json:"deadLetter"`
|
|
Warnings []string `json:"warnings"`
|
|
}
|
|
|
|
func (h InstanceHealth) NormalizeCollections() InstanceHealth {
|
|
if h.Warnings == nil {
|
|
h.Warnings = []string{}
|
|
}
|
|
return h
|
|
}
|
|
|
|
// Monitor handles all monitoring operations
|
|
type Monitor struct {
|
|
config *config.Config
|
|
state *models.State
|
|
orgID string // Organization ID for tenant isolation (empty = default/legacy)
|
|
mockUnifiedViewMu sync.Mutex
|
|
mockUnifiedView monitorUnifiedStateView
|
|
mockUnifiedViewVersion uint64
|
|
mockUnifiedViewValid bool
|
|
pveClients map[string]PVEClientInterface
|
|
pbsClients map[string]*pbs.Client
|
|
pmgClients map[string]*pmg.Client
|
|
availabilityStatuses map[string]AvailabilityProbeStatus
|
|
availabilityByLocation map[string]map[string]AvailabilityProbeStatus
|
|
availabilityProbeTrackers map[string]availabilityProbeAssignmentTracker
|
|
pollProviders map[InstanceType]PollProvider
|
|
pollMetrics *PollMetrics
|
|
scheduler *AdaptiveScheduler
|
|
stalenessTracker *StalenessTracker
|
|
taskQueue *TaskQueue
|
|
pollTimeout time.Duration
|
|
circuitBreakers map[string]*circuitBreaker
|
|
deadLetterQueue *TaskQueue
|
|
failureCounts map[string]int
|
|
lastOutcome map[string]taskOutcome
|
|
backoffCfg backoffConfig
|
|
rng *rand.Rand
|
|
maxRetryAttempts int
|
|
tempCollector *TemperatureCollector // SSH-based temperature collector
|
|
guestMetadataStore *config.GuestMetadataStore
|
|
dockerMetadataStore *config.DockerMetadataStore
|
|
dockerReportOrderStore *config.DockerReportOrderStore
|
|
hostMetadataStore *config.HostMetadataStore
|
|
hostContinuityStore *config.HostContinuityStore
|
|
hostAgentLifecycleMu sync.RWMutex
|
|
mu sync.RWMutex
|
|
startTime time.Time
|
|
rateTracker *RateTracker
|
|
metricsHistory *MetricsHistory
|
|
metricsStore *metrics.Store // Persistent SQLite metrics storage
|
|
alertManager *alerts.Manager
|
|
alertResolvedAICallback func(*alerts.Alert)
|
|
alertTriggeredAICallback func(*alerts.Alert)
|
|
alertPushCallback func(*alerts.Alert)
|
|
connectionsSnapshotLister func() []alerts.ConnectionSnapshot // returns platform connection snapshots for the connection-degraded check
|
|
incidentStore *memory.IncidentStore
|
|
alertProjectionReplayMu sync.Mutex // serializes lifecycle projection replay passes
|
|
alertProjectionWG sync.WaitGroup // tracks scheduled background catch-up runs
|
|
notificationMgr *notifications.NotificationManager
|
|
deadMan *deadManRuntime
|
|
deadManProgressUnixNano atomic.Int64
|
|
deadManConfigMu sync.RWMutex
|
|
deadManConfig notifications.DeadManConfig
|
|
deadManConfigLoadErr error
|
|
deliveryHealthProjectionMu sync.Mutex // serializes delivery-health reads and alert projection
|
|
lastDeliveryHealthCheck time.Time // throttles the notification-delivery system alert evaluation; guarded by mu
|
|
configPersist *config.ConfigPersistence
|
|
discoveryService *discovery.Service // Background discovery service
|
|
activePollCount int32 // Number of active polling operations
|
|
pollCounter int64 // Counter for polling cycles
|
|
authFailures map[string]int // Track consecutive auth failures per node
|
|
lastAuthAttempt map[string]time.Time // Track last auth attempt time
|
|
lastClusterCheck map[string]time.Time // Track last cluster check for standalone nodes
|
|
lastPhysicalDiskPoll map[string]time.Time // Track last physical disk poll time per instance
|
|
lastPVEBackupPoll map[string]time.Time // Track last PVE backup poll per instance
|
|
vzdumpJobTaskCache map[string]vzdumpJobTaskCacheEntry // Cache synthesized per-guest tasks for multi-guest vzdump job runs, keyed by instance|UPID
|
|
lastPBSBackupPoll map[string]time.Time // Track last PBS backup poll per instance
|
|
pveBackupInventoryReady map[string]map[string]bool // Track PVE guest inventory readiness for backup orphan detection
|
|
pveBackupTemplateSubjects map[string]map[string]struct{} // Track template VMIDs excluded from runtime workloads but valid for backups
|
|
backupPermissionWarnings map[string]string // Track backup permission issues per instance (instance -> warning message)
|
|
persistence *config.ConfigPersistence // Add persistence for saving updated configs
|
|
pbsBackupPollers map[string]bool // Track PBS backup polling goroutines per instance
|
|
pbsBackupCacheTime map[string]map[pbsBackupGroupKey]time.Time // Track when each PBS backup group was last fetched
|
|
runtimePollingMu sync.RWMutex // Guards runtimePollingOverride; polling goroutines read it every cycle
|
|
runtimePollingOverride runtimePollingOverrides // Runtime polling-cadence overrides pushed by the settings API (#1619)
|
|
runtimeCtx context.Context // Context used while monitor is running
|
|
wsHub *websocket.Hub // Hub used for broadcasting state
|
|
diagMu sync.RWMutex // Protects diagnostic snapshot maps
|
|
nodeSnapshots map[string]NodeMemorySnapshot
|
|
guestSnapshots map[string]GuestMemorySnapshot
|
|
rrdCacheMu sync.RWMutex // Protects short-lived guest memory caches.
|
|
nodeRRDMemCache map[string]rrdMemCacheEntry
|
|
vmAgentMemCache map[string]agentMemCacheEntry
|
|
removedDockerHosts map[string]time.Time // Track deliberately removed Docker hosts (ID -> removal time)
|
|
dockerTokenBindings map[string]string // Track token ID -> Docker host identity bindings to enforce uniqueness
|
|
dockerIdentityFlaps map[string]*identityFlapTracker // Track per-host identity flapping (cloned VMs sharing machine-id)
|
|
removedKubernetesClusters map[string]time.Time // Track deliberately removed Kubernetes clusters (ID -> removal time)
|
|
kubernetesTokenBindings map[string]string // Track token ID -> agent ID bindings to enforce uniqueness
|
|
removedHostAgents map[string]time.Time // Track deliberately removed host agents (ID -> removal time)
|
|
hostTokenBindings map[string]string // Track tokenID:hostname -> host identity bindings
|
|
hostIdentityFlaps map[string]*identityFlapTracker // Track per-host-agent identity flapping (cloned machines sharing machine-id)
|
|
hostReportApplyLocksMu sync.Mutex
|
|
hostReportApplyLocks map[string]*hostReportApplyLock
|
|
hostReportOrderMu sync.Mutex
|
|
hostReportOrders map[string]hostReportOrder
|
|
dockerCommands map[string]*dockerHostCommand
|
|
dockerCommandIndex map[string]string
|
|
guestMetadataMu sync.RWMutex
|
|
guestMetadataCache map[string]guestMetadataCacheEntry
|
|
guestMetadataLimiterMu sync.Mutex
|
|
guestMetadataLimiter map[string]time.Time
|
|
guestMetadataSlots chan struct{}
|
|
guestMetadataMinRefresh time.Duration
|
|
guestMetadataRefreshJitter time.Duration
|
|
guestMetadataRetryBackoff time.Duration
|
|
guestMetadataHoldDuration time.Duration
|
|
guestAgentWorkSlots chan struct{}
|
|
// Configurable guest agent timeouts (refs #592)
|
|
guestAgentFSInfoTimeout time.Duration
|
|
guestAgentNetworkTimeout time.Duration
|
|
guestAgentOSInfoTimeout time.Duration
|
|
guestAgentVersionTimeout time.Duration
|
|
guestAgentRetries int
|
|
guestAgentVMBudget time.Duration
|
|
executor PollExecutor
|
|
breakerBaseRetry time.Duration
|
|
breakerMaxDelay time.Duration
|
|
breakerHalfOpenWindow time.Duration
|
|
instanceInfoCache map[string]*instanceInfo
|
|
pollStatusMap map[string]*pollStatus
|
|
dlqInsightMap map[string]*dlqInsight
|
|
nodeLastOnline map[string]time.Time // Track last time each node was seen online (for grace period)
|
|
nodePendingUpdatesCache map[string]pendingUpdatesCache // Cache pending updates per node (checked every 30 min)
|
|
pveMembershipMisses map[string]int // Consecutive authoritative membership absences by instance/node
|
|
resourceStore ResourceStoreInterface // Optional unified resource store for polling optimization
|
|
supplementalProviders map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider
|
|
recoveryManager *recoverymanager.Manager // Optional recovery store manager for backup rollups
|
|
recoveryIngestMu sync.Mutex
|
|
recoveryIngestRunning bool
|
|
recoveryIngestPending []recoveryIngestBatch
|
|
mockMetricsCancel context.CancelFunc
|
|
mockMetricsWg sync.WaitGroup
|
|
dockerChecker DockerChecker // Optional Docker checker for LXC containers
|
|
dockerCheckerConfiguredAt time.Time // Last time the Docker checker was configured
|
|
dockerCheckAllowedVMIDs map[int]struct{} // Optional VMID allowlist gating the LXC Docker socket probe; empty means all guests
|
|
dockerInventoryCollector DockerInventoryCollector // Optional Docker inventory collector for LXC containers
|
|
// Consecutive Docker socket probe failures per container ID, so persistent
|
|
// failures back off instead of re-running pct exec on every poll cycle.
|
|
dockerProbeFailureMu sync.Mutex
|
|
dockerProbeFailures map[string]*dockerProbeFailureState
|
|
// Dispatch time of the Docker probe/inventory command currently owning
|
|
// each guest. A guest with a live claim is never re-probed, so overlapping
|
|
// poll cycles (or cycles whose enrichment context expired while the agent
|
|
// was still running the previous pct exec) cannot stack identical commands
|
|
// on the Proxmox host (minipc probe-storm incident, 2026-08-20).
|
|
dockerProbesInFlight map[string]time.Time
|
|
// Consecutive Docker probe/inventory failures per Proxmox node. Once a
|
|
// node accumulates enough back-to-back failures the circuit breaker
|
|
// suspends all Docker command dispatch to it, because the node itself
|
|
// (not one guest) is the likely problem — e.g. pct exec crawling under
|
|
// NFS flapping.
|
|
dockerNodeProbeFailures map[string]*dockerProbeFailureState
|
|
// Agent profile cache to avoid disk I/O on every report (refs #1094)
|
|
agentProfileCacheMu sync.RWMutex
|
|
agentProfileCache *agentProfileCacheEntry
|
|
// Proxmox LXC filesystem cache: bounded pct df readings reported by an
|
|
// agent securely linked to the node that owns each running container.
|
|
proxmoxLXCFilesystemsMu sync.RWMutex
|
|
proxmoxLXCFilesystemsCache map[string]agentLXCFilesystemCacheEntry
|
|
|
|
// Cluster sensor cache: temperature data collected by an agent on one Proxmox
|
|
// cluster node via SSH to its siblings. Keyed by lowercase node name.
|
|
clusterSensorsMu sync.RWMutex
|
|
clusterSensorsCache map[string]clusterSensorsCacheEntry
|
|
mockChartCacheMu sync.RWMutex
|
|
mockChartMapCache map[mockChartMetricMapCacheKey]map[string][]MetricPoint
|
|
// Commercial feature gate for monitoring-owned Pro behaviour (external
|
|
// availability probes). Guarded by its own mutex so read-time derivations
|
|
// that already hold m.mu can consult it without lock re-entrancy.
|
|
licenseCheckerMu sync.RWMutex
|
|
licenseChecker func(feature string) bool
|
|
}
|
|
|
|
func (m *Monitor) setRuntimeContext(ctx context.Context, hub *websocket.Hub) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.runtimeCtx = ctx
|
|
m.wsHub = hub
|
|
}
|
|
|
|
func (m *Monitor) getRuntimeContext() context.Context {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
return m.runtimeCtx
|
|
}
|
|
|
|
// clusterSensorsCacheEntry stores temperature data collected by a sibling agent via SSH.
|
|
type clusterSensorsCacheEntry struct {
|
|
sensors models.HostSensorSummary
|
|
updatedAt time.Time
|
|
}
|
|
|
|
type rrdMemCacheEntry struct {
|
|
available uint64
|
|
used uint64
|
|
total uint64
|
|
hasAvail bool
|
|
hasUsed bool
|
|
hasTotal bool
|
|
netIn float64
|
|
netOut float64
|
|
hasNetIn bool
|
|
hasNetOut bool
|
|
fetchedAt time.Time
|
|
}
|
|
|
|
// pendingUpdatesCache caches apt pending updates count per node
|
|
type pendingUpdatesCache struct {
|
|
count int
|
|
checkedAt time.Time
|
|
}
|
|
|
|
// TTL for pending updates cache (30 minutes - balance between freshness and API load)
|
|
const pendingUpdatesCacheTTL = 30 * time.Minute
|
|
|
|
// agentProfileCacheEntry caches agent profiles and assignments to avoid disk I/O on every agent report.
|
|
// TTL is 60 seconds to balance freshness with performance.
|
|
type agentProfileCacheEntry struct {
|
|
profiles []models.AgentProfile
|
|
assignments []models.AgentProfileAssignment
|
|
loadedAt time.Time
|
|
}
|
|
|
|
const agentProfileCacheTTL = 60 * time.Second
|
|
|
|
// shouldRunBackupPoll determines whether a backup polling cycle should execute.
|
|
// Returns whether polling should run, a human-readable skip reason, and the timestamp to record.
|
|
func (m *Monitor) shouldRunBackupPoll(last time.Time, now time.Time) (bool, string, time.Time) {
|
|
if m == nil || m.config == nil {
|
|
return false, "configuration unavailable", last
|
|
}
|
|
|
|
if !m.backupPollingEnabledSetting() {
|
|
return false, "backup polling globally disabled", last
|
|
}
|
|
|
|
interval := m.backupPollingIntervalSetting()
|
|
if interval > 0 {
|
|
if !last.IsZero() && now.Sub(last) < interval {
|
|
next := last.Add(interval)
|
|
return false, fmt.Sprintf("next run scheduled for %s", next.Format(time.RFC3339)), last
|
|
}
|
|
return true, "", now
|
|
}
|
|
|
|
backupCycles := m.config.BackupPollingCycles
|
|
if backupCycles <= 0 {
|
|
backupCycles = 10
|
|
}
|
|
|
|
if m.pollCounter%int64(backupCycles) == 0 || m.pollCounter == 1 {
|
|
return true, "", now
|
|
}
|
|
|
|
remaining := int64(backupCycles) - (m.pollCounter % int64(backupCycles))
|
|
return false, fmt.Sprintf("next run in %d polling cycles", remaining), last
|
|
}
|
|
|
|
const (
|
|
dockerConnectionPrefix = "docker-"
|
|
kubernetesConnectionPrefix = "kubernetes-"
|
|
hostConnectionPrefix = "host-"
|
|
dockerOfflineGraceMultiplier = 4
|
|
dockerMinimumHealthWindow = 30 * time.Second
|
|
dockerMaximumHealthWindow = 10 * time.Minute
|
|
kubernetesOfflineGraceMultiplier = 4
|
|
kubernetesMinimumHealthWindow = 30 * time.Second
|
|
kubernetesMaximumHealthWindow = 10 * time.Minute
|
|
hostOfflineGraceMultiplier = 6
|
|
hostMinimumHealthWindow = 60 * time.Second
|
|
hostMaximumHealthWindow = 10 * time.Minute
|
|
nodeOfflineGracePeriod = 60 * time.Second // Grace period before marking Proxmox nodes offline
|
|
nodeRRDCacheTTL = 30 * time.Second
|
|
nodeRRDRequestTimeout = 2 * time.Second
|
|
)
|
|
|
|
type taskOutcome struct {
|
|
success bool
|
|
transient bool
|
|
err error
|
|
recordedAt time.Time
|
|
}
|
|
|
|
func nodeRRDCacheKey(instanceName, nodeName string) string {
|
|
return instanceName + "/" + nodeName
|
|
}
|
|
|
|
func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterface, instanceName, nodeName string) (rrdMemCacheEntry, error) {
|
|
if client == nil || nodeName == "" {
|
|
return rrdMemCacheEntry{}, fmt.Errorf("invalid arguments for RRD lookup")
|
|
}
|
|
|
|
now := time.Now()
|
|
cacheKey := nodeRRDCacheKey(instanceName, nodeName)
|
|
|
|
m.rrdCacheMu.RLock()
|
|
if entry, ok := m.nodeRRDMemCache[cacheKey]; ok && now.Sub(entry.fetchedAt) < nodeRRDCacheTTL {
|
|
m.rrdCacheMu.RUnlock()
|
|
return entry, nil
|
|
}
|
|
m.rrdCacheMu.RUnlock()
|
|
|
|
requestCtx, cancel := context.WithTimeout(ctx, nodeRRDRequestTimeout)
|
|
defer cancel()
|
|
|
|
points, err := client.GetNodeRRDData(requestCtx, nodeName, "hour", "AVERAGE", []string{"memavailable", "memused", "memtotal", "netin", "netout"})
|
|
if err != nil {
|
|
return rrdMemCacheEntry{}, err
|
|
}
|
|
|
|
var memAvailable uint64
|
|
var memUsed uint64
|
|
var memTotal uint64
|
|
var hasAvail bool
|
|
var hasUsed bool
|
|
var hasTotal bool
|
|
var netIn float64
|
|
var netOut float64
|
|
var hasNetIn bool
|
|
var hasNetOut bool
|
|
|
|
for i := len(points) - 1; i >= 0; i-- {
|
|
point := points[i]
|
|
|
|
if !hasTotal && point.MemTotal != nil && !math.IsNaN(*point.MemTotal) && !math.IsInf(*point.MemTotal, 0) && *point.MemTotal > 0 && *point.MemTotal <= math.MaxUint64 {
|
|
memTotal = uint64(math.Round(*point.MemTotal))
|
|
hasTotal = true
|
|
}
|
|
|
|
if !hasAvail && point.MemAvailable != nil && !math.IsNaN(*point.MemAvailable) && !math.IsInf(*point.MemAvailable, 0) && *point.MemAvailable >= 0 && *point.MemAvailable <= math.MaxUint64 {
|
|
memAvailable = uint64(math.Round(*point.MemAvailable))
|
|
hasAvail = true
|
|
}
|
|
|
|
if !hasUsed && point.MemUsed != nil && !math.IsNaN(*point.MemUsed) && !math.IsInf(*point.MemUsed, 0) && *point.MemUsed >= 0 && *point.MemUsed <= math.MaxUint64 {
|
|
memUsed = uint64(math.Round(*point.MemUsed))
|
|
hasUsed = true
|
|
}
|
|
|
|
if !hasNetIn && point.NetIn != nil && !math.IsNaN(*point.NetIn) && !math.IsInf(*point.NetIn, 0) {
|
|
netIn = *point.NetIn
|
|
hasNetIn = true
|
|
}
|
|
if !hasNetOut && point.NetOut != nil && !math.IsNaN(*point.NetOut) && !math.IsInf(*point.NetOut, 0) {
|
|
netOut = *point.NetOut
|
|
hasNetOut = true
|
|
}
|
|
}
|
|
|
|
if memTotal > 0 {
|
|
if memAvailable > memTotal {
|
|
memAvailable = 0
|
|
hasAvail = false
|
|
}
|
|
if memUsed > memTotal {
|
|
memUsed = 0
|
|
hasUsed = false
|
|
}
|
|
}
|
|
|
|
if !hasAvail && !hasUsed && !hasNetIn && !hasNetOut {
|
|
return rrdMemCacheEntry{}, fmt.Errorf("rrd node metrics not present")
|
|
}
|
|
|
|
entry := rrdMemCacheEntry{
|
|
available: memAvailable,
|
|
used: memUsed,
|
|
total: memTotal,
|
|
hasAvail: hasAvail,
|
|
hasUsed: hasUsed,
|
|
hasTotal: hasTotal,
|
|
netIn: netIn,
|
|
netOut: netOut,
|
|
hasNetIn: hasNetIn,
|
|
hasNetOut: hasNetOut,
|
|
fetchedAt: now,
|
|
}
|
|
|
|
m.rrdCacheMu.Lock()
|
|
if m.nodeRRDMemCache == nil {
|
|
m.nodeRRDMemCache = make(map[string]rrdMemCacheEntry)
|
|
}
|
|
m.nodeRRDMemCache[cacheKey] = entry
|
|
m.rrdCacheMu.Unlock()
|
|
|
|
return entry, nil
|
|
}
|
|
|
|
// RemoveDockerHost removes a docker host from the shared state and clears related alerts.
|
|
func (m *Monitor) GetConnectionStatuses() map[string]bool {
|
|
if m == nil {
|
|
return map[string]bool{}
|
|
}
|
|
|
|
if mock.IsMockEnabled() {
|
|
statuses := make(map[string]bool)
|
|
state := mock.CurrentFixtureGraph().State
|
|
for _, node := range state.Nodes {
|
|
key := "pve-" + node.Name
|
|
statuses[key] = strings.ToLower(node.Status) == "online"
|
|
if node.Host != "" {
|
|
statuses[node.Host] = strings.ToLower(node.Status) == "online"
|
|
}
|
|
}
|
|
for _, pbsInst := range state.PBSInstances {
|
|
key := "pbs-" + pbsInst.Name
|
|
statuses[key] = strings.ToLower(pbsInst.Status) != "offline"
|
|
if pbsInst.Host != "" {
|
|
statuses[pbsInst.Host] = strings.ToLower(pbsInst.Status) != "offline"
|
|
}
|
|
}
|
|
for _, pmgInst := range state.PMGInstances {
|
|
key := "pmg-" + pmgInst.Name
|
|
statuses[key] = strings.ToLower(pmgInst.Status) != "offline"
|
|
if pmgInst.Host != "" {
|
|
statuses[pmgInst.Host] = strings.ToLower(pmgInst.Status) != "offline"
|
|
}
|
|
}
|
|
|
|
for _, dockerHost := range state.DockerHosts {
|
|
key := dockerConnectionPrefix + dockerHost.ID
|
|
statuses[key] = strings.ToLower(dockerHost.Status) == "online"
|
|
}
|
|
return statuses
|
|
}
|
|
|
|
statuses := make(map[string]bool)
|
|
for _, provider := range m.pollProviderSnapshotWithBuiltins() {
|
|
for key, connected := range m.providerConnectionStatuses(provider) {
|
|
if strings.TrimSpace(key) == "" {
|
|
continue
|
|
}
|
|
statuses[key] = connected
|
|
}
|
|
}
|
|
return statuses
|
|
}
|
|
|
|
// checkContainerizedTempMonitoring logs a security warning if Pulse is running
|
|
// in a container with SSH-based temperature monitoring enabled
|
|
func checkContainerizedTempMonitoring() {
|
|
// Check if running in container
|
|
isContainer := os.Getenv("PULSE_DOCKER") == "true" || system.InContainer()
|
|
if !isContainer {
|
|
return
|
|
}
|
|
|
|
// Check if SSH keys exist (indicates temperature monitoring is configured)
|
|
homeDir := os.Getenv("HOME")
|
|
if homeDir == "" {
|
|
homeDir = "/home/pulse"
|
|
}
|
|
sshKeyPath := homeDir + "/.ssh/id_ed25519"
|
|
if _, err := os.Stat(sshKeyPath); err != nil {
|
|
// No SSH key found, temperature monitoring not configured
|
|
return
|
|
}
|
|
|
|
// Log warning
|
|
log.Warn().
|
|
Msg("SECURITY NOTICE: Pulse is running in a container with SSH-based temperature monitoring enabled. " +
|
|
"SSH private keys are stored inside the container, which could be a security risk if the container is compromised. " +
|
|
"Future versions will use agent-based architecture for better security. " +
|
|
"See documentation for hardening recommendations.")
|
|
}
|
|
|
|
// New creates a new Monitor instance
|
|
func New(cfg *config.Config) (*Monitor, error) {
|
|
if cfg == nil {
|
|
return nil, fmt.Errorf("config cannot be nil")
|
|
}
|
|
|
|
hostContinuityStore := config.NewHostContinuityStore(cfg.DataPath, nil)
|
|
if err := hostContinuityStore.LoadError(); err != nil {
|
|
return nil, fmt.Errorf("load host continuity lifecycle journal: %w", err)
|
|
}
|
|
dockerReportOrderStore := config.NewDockerReportOrderStore(cfg.DataPath, nil)
|
|
if err := dockerReportOrderStore.LoadError(); err != nil {
|
|
return nil, fmt.Errorf("load Docker report order journal: %w", err)
|
|
}
|
|
|
|
// Initialize temperature collector with sensors SSH key
|
|
// Will use root user for now - can be made configurable later
|
|
homeDir := os.Getenv("HOME")
|
|
if homeDir == "" {
|
|
homeDir = "/home/pulse"
|
|
}
|
|
sshKeyPath := filepath.Join(homeDir, ".ssh/id_ed25519_sensors")
|
|
tempCollector := NewTemperatureCollectorWithPort("root", sshKeyPath, cfg.SSHPort)
|
|
|
|
// Security warning if running in container with SSH temperature monitoring
|
|
checkContainerizedTempMonitoring()
|
|
|
|
stalenessTracker := NewStalenessTracker(getPollMetrics())
|
|
stalenessTracker.SetBounds(cfg.AdaptivePollingBaseInterval, cfg.AdaptivePollingMaxInterval)
|
|
taskQueue := NewTaskQueue()
|
|
deadLetterQueue := NewTaskQueue()
|
|
breakers := make(map[string]*circuitBreaker)
|
|
failureCounts := make(map[string]int)
|
|
lastOutcome := make(map[string]taskOutcome)
|
|
backoff := backoffConfig{
|
|
Initial: 5 * time.Second,
|
|
Multiplier: 2,
|
|
Jitter: 0.2,
|
|
Max: 5 * time.Minute,
|
|
}
|
|
|
|
if cfg.AdaptivePollingEnabled && cfg.AdaptivePollingMaxInterval > 0 && cfg.AdaptivePollingMaxInterval <= 15*time.Second {
|
|
backoff.Initial = 750 * time.Millisecond
|
|
backoff.Max = 6 * time.Second
|
|
}
|
|
|
|
var scheduler *AdaptiveScheduler
|
|
if cfg.AdaptivePollingEnabled {
|
|
scheduler = NewAdaptiveScheduler(SchedulerConfig{
|
|
BaseInterval: cfg.AdaptivePollingBaseInterval,
|
|
MinInterval: cfg.AdaptivePollingMinInterval,
|
|
MaxInterval: cfg.AdaptivePollingMaxInterval,
|
|
}, stalenessTracker, nil, nil)
|
|
}
|
|
|
|
minRefresh := cfg.GuestMetadataMinRefreshInterval
|
|
if minRefresh <= 0 {
|
|
minRefresh = config.DefaultGuestMetadataMinRefresh
|
|
}
|
|
jitter := cfg.GuestMetadataRefreshJitter
|
|
if jitter < 0 {
|
|
jitter = 0
|
|
}
|
|
retryBackoff := cfg.GuestMetadataRetryBackoff
|
|
if retryBackoff <= 0 {
|
|
retryBackoff = config.DefaultGuestMetadataRetryBackoff
|
|
}
|
|
concurrency := cfg.GuestMetadataMaxConcurrent
|
|
if concurrency <= 0 {
|
|
concurrency = config.DefaultGuestMetadataMaxConcurrent
|
|
}
|
|
holdDuration := defaultGuestMetadataHold
|
|
|
|
// Load guest agent timeout configuration from environment variables (refs #592)
|
|
guestAgentFSInfoTimeout := parsePositiveDurationEnv("GUEST_AGENT_FSINFO_TIMEOUT", defaultGuestAgentFSInfoTimeout)
|
|
guestAgentNetworkTimeout := parsePositiveDurationEnv("GUEST_AGENT_NETWORK_TIMEOUT", defaultGuestAgentNetworkTimeout)
|
|
guestAgentOSInfoTimeout := parsePositiveDurationEnv("GUEST_AGENT_OSINFO_TIMEOUT", defaultGuestAgentOSInfoTimeout)
|
|
guestAgentVersionTimeout := parsePositiveDurationEnv("GUEST_AGENT_VERSION_TIMEOUT", defaultGuestAgentVersionTimeout)
|
|
guestAgentRetries := parseNonNegativeIntEnv("GUEST_AGENT_RETRIES", defaultGuestAgentRetries)
|
|
guestAgentVMBudget := parseDurationEnv("GUEST_AGENT_VM_BUDGET", 0)
|
|
guestAgentVMMaxConcurrent := parseNonNegativeIntEnv("GUEST_AGENT_VM_MAX_CONCURRENT", defaultGuestAgentVMMaxConcurrent)
|
|
|
|
// Initialize persistent metrics store (SQLite) with configurable retention
|
|
var metricsStore *metrics.Store
|
|
metricsStoreConfig := metrics.DefaultConfig(cfg.DataPath)
|
|
if strings.TrimSpace(cfg.MetricsDBPath) != "" {
|
|
metricsStoreConfig.DBPath = cfg.MetricsDBPath
|
|
}
|
|
if cfg.MetricsRollupInterval > 0 {
|
|
metricsStoreConfig.RollupInterval = cfg.MetricsRollupInterval
|
|
}
|
|
// Override retention settings from config (allows tier-based pricing in future)
|
|
if cfg.MetricsRetentionRawHours > 0 {
|
|
metricsStoreConfig.RetentionRaw = time.Duration(cfg.MetricsRetentionRawHours) * time.Hour
|
|
}
|
|
if cfg.MetricsRetentionMinuteHours > 0 {
|
|
metricsStoreConfig.RetentionMinute = time.Duration(cfg.MetricsRetentionMinuteHours) * time.Hour
|
|
}
|
|
if cfg.MetricsRetentionHourlyDays > 0 {
|
|
metricsStoreConfig.RetentionHourly = time.Duration(cfg.MetricsRetentionHourlyDays) * 24 * time.Hour
|
|
}
|
|
if cfg.MetricsRetentionDailyDays > 0 {
|
|
metricsStoreConfig.RetentionDaily = time.Duration(cfg.MetricsRetentionDailyDays) * 24 * time.Hour
|
|
}
|
|
|
|
// In mock mode, extend hourly/daily retention to 90 days so seeded mock
|
|
// backfill (daily tier 30d, hourly tier 7d — see mockStoreSeedPlans)
|
|
// outlives its windows. Raw and minute tiers keep production defaults —
|
|
// the minute seed window (24h) matches the default retention, and short
|
|
// raw retention keeps live mock ticks from bloating the DB (the old
|
|
// 90-day raw retention caused metrics.db to grow to ~2 GB).
|
|
if mock.IsMockEnabled() {
|
|
metricsStoreConfig.WriteBufferSize = 2000
|
|
metricsStoreConfig.RetentionHourly = 90 * 24 * time.Hour
|
|
metricsStoreConfig.RetentionDaily = 90 * 24 * time.Hour
|
|
}
|
|
ms, err := metrics.NewStore(metricsStoreConfig)
|
|
if err != nil {
|
|
// Do not automatically delete the DB on error, as it causes data loss on transient errors (e.g. locks).
|
|
// If the DB is truly corrupted, the user should manually remove it.
|
|
log.Error().Err(err).Msg("failed to initialize persistent metrics store - continuing without metrics persistence")
|
|
} else {
|
|
if mock.IsMockEnabled() {
|
|
ms.SetMaxOpenConns(10)
|
|
}
|
|
metricsStore = ms
|
|
log.Info().
|
|
Str("path", metricsStoreConfig.DBPath).
|
|
Dur("retentionRaw", metricsStoreConfig.RetentionRaw).
|
|
Dur("retentionMinute", metricsStoreConfig.RetentionMinute).
|
|
Dur("retentionHourly", metricsStoreConfig.RetentionHourly).
|
|
Dur("retentionDaily", metricsStoreConfig.RetentionDaily).
|
|
Msg("Persistent metrics store initialized with configurable retention")
|
|
}
|
|
|
|
incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{
|
|
DataDir: cfg.DataPath,
|
|
})
|
|
|
|
m := &Monitor{
|
|
config: cfg,
|
|
state: models.NewState(),
|
|
pveClients: make(map[string]PVEClientInterface),
|
|
pbsClients: make(map[string]*pbs.Client),
|
|
pmgClients: make(map[string]*pmg.Client),
|
|
availabilityStatuses: make(map[string]AvailabilityProbeStatus),
|
|
availabilityByLocation: make(map[string]map[string]AvailabilityProbeStatus),
|
|
availabilityProbeTrackers: make(map[string]availabilityProbeAssignmentTracker),
|
|
pollProviders: make(map[InstanceType]PollProvider),
|
|
pollMetrics: getPollMetrics(),
|
|
scheduler: scheduler,
|
|
stalenessTracker: stalenessTracker,
|
|
taskQueue: taskQueue,
|
|
pollTimeout: derivePollTimeout(cfg),
|
|
deadLetterQueue: deadLetterQueue,
|
|
circuitBreakers: breakers,
|
|
failureCounts: failureCounts,
|
|
lastOutcome: lastOutcome,
|
|
backoffCfg: backoff,
|
|
rng: rand.New(rand.NewSource(time.Now().UnixNano())),
|
|
maxRetryAttempts: 5,
|
|
tempCollector: tempCollector,
|
|
guestMetadataStore: config.NewGuestMetadataStore(cfg.DataPath, nil),
|
|
dockerMetadataStore: config.NewDockerMetadataStore(cfg.DataPath, nil),
|
|
dockerReportOrderStore: dockerReportOrderStore,
|
|
hostMetadataStore: config.NewHostMetadataStore(cfg.DataPath, nil),
|
|
hostContinuityStore: hostContinuityStore,
|
|
startTime: time.Now(),
|
|
rateTracker: NewRateTracker(),
|
|
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points (~8h @ 30s)
|
|
metricsStore: metricsStore, // Persistent SQLite storage
|
|
alertManager: alerts.NewManagerWithDataDir(cfg.DataPath, alertManagerRestoreOptions()...),
|
|
incidentStore: incidentStore,
|
|
notificationMgr: notifications.NewNotificationManagerWithDataDir(cfg.PublicURL, cfg.DataPath),
|
|
deadMan: newDeadManRuntime(config.ResolveRuntimeDataDir(cfg.DataPath)),
|
|
configPersist: config.NewConfigPersistence(cfg.DataPath),
|
|
discoveryService: nil, // Will be initialized in Start()
|
|
authFailures: make(map[string]int),
|
|
lastAuthAttempt: make(map[string]time.Time),
|
|
lastClusterCheck: make(map[string]time.Time),
|
|
lastPhysicalDiskPoll: make(map[string]time.Time),
|
|
lastPVEBackupPoll: make(map[string]time.Time),
|
|
vzdumpJobTaskCache: make(map[string]vzdumpJobTaskCacheEntry),
|
|
lastPBSBackupPoll: make(map[string]time.Time),
|
|
pveBackupInventoryReady: make(map[string]map[string]bool),
|
|
pveBackupTemplateSubjects: make(map[string]map[string]struct{}),
|
|
backupPermissionWarnings: make(map[string]string),
|
|
persistence: config.NewConfigPersistence(cfg.DataPath),
|
|
pbsBackupPollers: make(map[string]bool),
|
|
pbsBackupCacheTime: make(map[string]map[pbsBackupGroupKey]time.Time),
|
|
nodeSnapshots: make(map[string]NodeMemorySnapshot),
|
|
guestSnapshots: make(map[string]GuestMemorySnapshot),
|
|
nodeRRDMemCache: make(map[string]rrdMemCacheEntry),
|
|
vmAgentMemCache: make(map[string]agentMemCacheEntry),
|
|
removedDockerHosts: make(map[string]time.Time),
|
|
dockerTokenBindings: make(map[string]string),
|
|
dockerIdentityFlaps: make(map[string]*identityFlapTracker),
|
|
removedKubernetesClusters: make(map[string]time.Time),
|
|
kubernetesTokenBindings: make(map[string]string),
|
|
removedHostAgents: make(map[string]time.Time),
|
|
hostTokenBindings: make(map[string]string),
|
|
hostIdentityFlaps: make(map[string]*identityFlapTracker),
|
|
hostReportApplyLocks: make(map[string]*hostReportApplyLock),
|
|
hostReportOrders: make(map[string]hostReportOrder),
|
|
clusterSensorsCache: make(map[string]clusterSensorsCacheEntry),
|
|
proxmoxLXCFilesystemsCache: make(map[string]agentLXCFilesystemCacheEntry),
|
|
dockerCommands: make(map[string]*dockerHostCommand),
|
|
dockerCommandIndex: make(map[string]string),
|
|
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
|
|
guestMetadataLimiter: make(map[string]time.Time),
|
|
guestMetadataMinRefresh: minRefresh,
|
|
guestMetadataRefreshJitter: jitter,
|
|
guestMetadataRetryBackoff: retryBackoff,
|
|
guestMetadataHoldDuration: holdDuration,
|
|
guestAgentFSInfoTimeout: guestAgentFSInfoTimeout,
|
|
guestAgentNetworkTimeout: guestAgentNetworkTimeout,
|
|
guestAgentOSInfoTimeout: guestAgentOSInfoTimeout,
|
|
guestAgentVersionTimeout: guestAgentVersionTimeout,
|
|
guestAgentRetries: guestAgentRetries,
|
|
guestAgentVMBudget: guestAgentVMBudget,
|
|
instanceInfoCache: make(map[string]*instanceInfo),
|
|
pollStatusMap: make(map[string]*pollStatus),
|
|
dlqInsightMap: make(map[string]*dlqInsight),
|
|
nodeLastOnline: make(map[string]time.Time),
|
|
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
|
pveMembershipMisses: make(map[string]int),
|
|
supplementalProviders: make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider),
|
|
}
|
|
|
|
m.breakerBaseRetry = 5 * time.Second
|
|
m.breakerMaxDelay = 5 * time.Minute
|
|
m.breakerHalfOpenWindow = 30 * time.Second
|
|
|
|
if cfg.AdaptivePollingEnabled && cfg.AdaptivePollingMaxInterval > 0 && cfg.AdaptivePollingMaxInterval <= 15*time.Second {
|
|
m.breakerBaseRetry = 2 * time.Second
|
|
m.breakerMaxDelay = 10 * time.Second
|
|
m.breakerHalfOpenWindow = 2 * time.Second
|
|
}
|
|
|
|
m.executor = newRealExecutor(m)
|
|
if queue := m.notificationMgr.GetQueue(); queue != nil {
|
|
queue.SetDeliveryHealthChangedCallback(m.reconcileNotificationDelivery)
|
|
}
|
|
m.hydrateRemovedHostAgents(time.Now().UTC())
|
|
m.alertManager.SetBackupIntentContextResolver(m.resolveBackupIntentContext)
|
|
m.alertManager.SetMetricWindowProvider(m.metricWindowPoints)
|
|
m.alertManager.EnableShadowFeed()
|
|
m.registerBuiltInPollProviders()
|
|
m.buildInstanceInfoCache(cfg)
|
|
|
|
// Initialize state with config values
|
|
m.state.TemperatureMonitoringEnabled = cfg.TemperatureMonitoringEnabled
|
|
|
|
if m.pollMetrics != nil {
|
|
m.pollMetrics.ResetQueueDepth(0)
|
|
}
|
|
|
|
// Load saved configurations
|
|
if alertConfig, err := m.configPersist.LoadAlertConfig(); err == nil {
|
|
m.alertManager.UpdateConfig(*alertConfig)
|
|
// Apply schedule settings to notification manager
|
|
m.notificationMgr.SetEnabled(alertConfig.Enabled && alertConfig.ActivationState == alerts.ActivationActive)
|
|
m.notificationMgr.SetCooldown(alertConfig.Schedule.Cooldown)
|
|
m.notificationMgr.SetGroupingConfig(
|
|
alertConfig.Schedule.Grouping.Enabled,
|
|
alertConfig.Schedule.Grouping.Window,
|
|
alertConfig.Schedule.Grouping.ByNode,
|
|
alertConfig.Schedule.Grouping.ByGuest,
|
|
)
|
|
m.notificationMgr.SetInitialNotifyTarget(alertConfig.Schedule.InitialNotify)
|
|
m.notificationMgr.SetNotifyOnResolve(alertConfig.Schedule.NotifyOnResolve)
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load alert configuration")
|
|
}
|
|
if intentPolicies, err := m.configPersist.LoadAlertIntentPolicies(); err == nil {
|
|
if err := m.alertManager.LoadIntentPolicies(*intentPolicies); err != nil {
|
|
log.Warn().Err(err).Msg("failed to install alert intent policies")
|
|
}
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load alert intent policies")
|
|
}
|
|
|
|
if emailConfig, err := m.configPersist.LoadEmailConfig(); err == nil {
|
|
m.notificationMgr.SetEmailConfig(*emailConfig)
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load email configuration")
|
|
}
|
|
|
|
if concurrency > 0 {
|
|
m.guestMetadataSlots = make(chan struct{}, concurrency)
|
|
}
|
|
if guestAgentVMMaxConcurrent > 0 {
|
|
m.guestAgentWorkSlots = make(chan struct{}, guestAgentVMMaxConcurrent)
|
|
}
|
|
|
|
if appriseConfig, err := m.configPersist.LoadAppriseConfig(); err == nil {
|
|
m.notificationMgr.SetAppriseConfig(*appriseConfig)
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load Apprise configuration")
|
|
}
|
|
if deadManConfig, err := m.configPersist.LoadDeadManConfig(); err == nil {
|
|
m.deadManConfig = *deadManConfig
|
|
} else {
|
|
m.deadManConfigLoadErr = err
|
|
log.Warn().Err(err).Msg("failed to load dead-man configuration")
|
|
}
|
|
|
|
// Migrate webhooks if needed (from unencrypted to encrypted)
|
|
if err := m.configPersist.MigrateWebhooksIfNeeded(); err != nil {
|
|
log.Warn().Err(err).Msg("failed to migrate webhooks")
|
|
}
|
|
|
|
if webhooks, err := m.configPersist.LoadWebhooks(); err == nil {
|
|
for _, webhook := range webhooks {
|
|
m.notificationMgr.AddWebhook(webhook)
|
|
}
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load webhook configuration")
|
|
}
|
|
|
|
// In mock mode the canonical sampler owns demo chart history by default.
|
|
// Support-only hybrid runs can opt back into real client initialization.
|
|
mockEnabled := mock.IsMockEnabled()
|
|
if mockEnabled && !keepRealPollingInMockMode() {
|
|
log.Info().Msg("mock mode enabled - real client initialization disabled")
|
|
} else {
|
|
m.initPVEClients(cfg)
|
|
m.initPBSClients(cfg)
|
|
m.initPMGClients(cfg)
|
|
}
|
|
|
|
// Initialize state stats
|
|
m.state.Stats = models.Stats{
|
|
StartTime: m.startTime,
|
|
Version: "2.0.0-go",
|
|
}
|
|
|
|
// Agent endpoints can receive reports as soon as New returns, before the
|
|
// monitoring loop goroutine reaches Start. Wire external alert delivery now
|
|
// so a first startup observation cannot create an active alert without also
|
|
// reaching notifications. Start rewires the escalation callback with its
|
|
// WebSocket hub and installs lifecycle projection handling.
|
|
m.wireExternalAlertCallbacks(nil)
|
|
|
|
return m, nil
|
|
}
|
|
|
|
// SetExecutor allows tests to override the poll executor; passing nil restores the default executor.
|
|
func (m *Monitor) SetExecutor(exec PollExecutor) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if exec == nil {
|
|
m.executor = newRealExecutor(m)
|
|
return
|
|
}
|
|
|
|
m.executor = exec
|
|
}
|
|
|
|
func (m *Monitor) buildInstanceInfoCache(cfg *config.Config) {
|
|
_ = cfg
|
|
m.refreshInstanceInfoCacheFromProviders()
|
|
}
|
|
|
|
func (m *Monitor) getExecutor() PollExecutor {
|
|
m.mu.RLock()
|
|
exec := m.executor
|
|
m.mu.RUnlock()
|
|
return exec
|
|
}
|
|
|
|
func clampInterval(value, min, max time.Duration) time.Duration {
|
|
if value <= 0 {
|
|
return min
|
|
}
|
|
if min > 0 && value < min {
|
|
return min
|
|
}
|
|
if max > 0 && value > max {
|
|
return max
|
|
}
|
|
return value
|
|
}
|
|
|
|
func (m *Monitor) effectivePVEPollingInterval() time.Duration {
|
|
if m == nil {
|
|
return effectivePVEPollingIntervalForConfig(nil)
|
|
}
|
|
return effectivePVEPollingIntervalForConfig(m.config)
|
|
}
|
|
|
|
func (m *Monitor) fixedIntervalForInstance(instanceType InstanceType, instanceName string) time.Duration {
|
|
if provider, ok := m.getPollProvider(instanceType).(FixedIntervalPollProvider); ok && provider != nil {
|
|
return provider.FixedInstanceInterval(m, instanceName)
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (m *Monitor) baseIntervalForInstanceType(instanceType InstanceType) time.Duration {
|
|
if provider := m.getPollProvider(instanceType); provider != nil {
|
|
if interval := provider.BaseInterval(m); interval > 0 {
|
|
return interval
|
|
}
|
|
}
|
|
|
|
if m == nil || m.config == nil {
|
|
return DefaultSchedulerConfig().BaseInterval
|
|
}
|
|
|
|
switch instanceType {
|
|
case InstanceTypePVE:
|
|
return m.effectivePVEPollingInterval()
|
|
case InstanceTypePBS:
|
|
return clampInterval(m.pbsPollingIntervalSetting(), 10*time.Second, time.Hour)
|
|
case InstanceTypePMG:
|
|
return clampInterval(m.pmgPollingIntervalSetting(), 10*time.Second, time.Hour)
|
|
default:
|
|
base := m.config.AdaptivePollingBaseInterval
|
|
if base <= 0 {
|
|
base = DefaultSchedulerConfig().BaseInterval
|
|
}
|
|
return clampInterval(base, time.Second, 0)
|
|
}
|
|
}
|
|
|
|
// Start begins the monitoring loop
|
|
func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
|
// Consolidate any duplicate cluster instances before starting
|
|
// This fixes the case where multiple agents registered from the same cluster
|
|
m.consolidateDuplicateClusters()
|
|
|
|
pollingInterval := m.effectivePVEPollingInterval()
|
|
log.Info().
|
|
Dur("pollingInterval", pollingInterval).
|
|
Msg("Starting monitoring loop")
|
|
|
|
m.setRuntimeContext(ctx, wsHub)
|
|
defer m.stopMockMetricsSampler()
|
|
|
|
// Best-effort startup cleanup: when direct PBS is configured, remove legacy
|
|
// PVE-proxied PBS backup points to prevent duplicate recovery entries.
|
|
m.purgeStalePVEPBSBackupsBestEffort(ctx)
|
|
|
|
if mock.IsMockEnabled() {
|
|
m.startMockMetricsSampler(ctx)
|
|
}
|
|
|
|
// Initialize and start discovery service if enabled
|
|
if mock.IsMockEnabled() {
|
|
log.Info().Msg("mock mode enabled - skipping discovery service")
|
|
m.discoveryService = nil
|
|
} else if m.config.DiscoveryEnabled {
|
|
discoverySubnet := m.config.DiscoverySubnet
|
|
if discoverySubnet == "" {
|
|
discoverySubnet = "auto"
|
|
}
|
|
cfgProvider := func() config.DiscoveryConfig {
|
|
return m.discoveryConfigSnapshot()
|
|
}
|
|
m.discoveryService = discovery.NewService(wsHub, 5*time.Minute, discoverySubnet, cfgProvider)
|
|
if m.discoveryService != nil {
|
|
m.discoveryService.Start(ctx)
|
|
log.Info().Msg("discovery service initialized and started")
|
|
} else {
|
|
log.Error().Msg("failed to initialize discovery service")
|
|
}
|
|
} else {
|
|
log.Info().Msg("discovery service disabled by configuration")
|
|
m.discoveryService = nil
|
|
}
|
|
|
|
// Set up alert callbacks. Projection replay is deliberately absent here:
|
|
// the canonical resource store is not attached yet, so a replay now could
|
|
// not advance the durable watermark and would walk the same events again
|
|
// at the SetResourceStore boundary, which owns the single scheduled
|
|
// catch-up pass.
|
|
m.alertManager.SubscribeLifecycleCallback(m.handleAlertLifecycleEvent)
|
|
m.wireExternalAlertCallbacks(wsHub)
|
|
m.markDeadManMonitoringProgress(time.Now().UTC())
|
|
if err := m.deadManConfigurationLoadError(); err != nil {
|
|
m.alertManager.RaiseSystemAlert(alerts.SystemAlertInput{
|
|
Type: alerts.DeadManStateAlertType,
|
|
Level: alerts.AlertLevelWarning,
|
|
Message: "Pulse could not read the encrypted external watchdog configuration. Watchdog monitoring is unavailable until the destination is saved again.",
|
|
Fingerprint: "configuration-load-failed",
|
|
})
|
|
}
|
|
if m.deadMan != nil {
|
|
go m.deadMan.run(
|
|
ctx,
|
|
func() string {
|
|
if m.alertManager == nil {
|
|
return ""
|
|
}
|
|
return m.deadManConfigSnapshot().PingURL
|
|
},
|
|
m.deadManMonitoringProgress,
|
|
m.alertManager,
|
|
)
|
|
}
|
|
|
|
// Create separate tickers for polling and broadcasting using the configured cadence
|
|
|
|
workerCount := m.totalClientCount()
|
|
m.startTaskWorkers(ctx, workerCount)
|
|
|
|
pollTicker := time.NewTicker(pollingInterval)
|
|
defer pollTicker.Stop()
|
|
|
|
broadcastTicker := time.NewTicker(pollingInterval)
|
|
defer broadcastTicker.Stop()
|
|
deadManProgressTicker := time.NewTicker(15 * time.Second)
|
|
defer deadManProgressTicker.Stop()
|
|
|
|
keepRealPolling := keepRealPollingInMockMode()
|
|
|
|
// Start connection retry mechanism for failed clients
|
|
// This handles cases where network/Proxmox isn't ready on initial startup
|
|
if !mock.IsMockEnabled() || keepRealPolling {
|
|
go m.retryFailedConnections(ctx)
|
|
}
|
|
|
|
// Do an immediate poll on start.
|
|
if mock.IsMockEnabled() {
|
|
if keepRealPolling {
|
|
log.Info().Msg("mock mode enabled - running mock alerts and real metric polling")
|
|
go m.checkMockAlerts()
|
|
go m.poll(ctx, wsHub)
|
|
} else {
|
|
log.Info().Msg("mock mode enabled - skipping real node polling")
|
|
go m.checkMockAlerts()
|
|
}
|
|
} else {
|
|
go m.poll(ctx, wsHub)
|
|
}
|
|
|
|
for {
|
|
select {
|
|
case now := <-deadManProgressTicker.C:
|
|
// This tick runs on the canonical monitor loop itself. A separate
|
|
// heartbeat goroutine can therefore prove that scheduling remains
|
|
// responsive instead of merely proving its own timer is alive.
|
|
m.markDeadManMonitoringProgress(now.UTC())
|
|
case <-pollTicker.C:
|
|
now := time.Now()
|
|
m.evaluateDockerAgents(now)
|
|
m.evaluateKubernetesAgents(now)
|
|
m.evaluateHostAgents(now)
|
|
m.cleanupRemovedDockerHosts(now)
|
|
m.cleanupRemovedKubernetesClusters(now)
|
|
m.cleanupRemovedHostAgents(now)
|
|
m.cleanupGuestMetadataCache(now)
|
|
m.cleanupDiagnosticSnapshots(now)
|
|
m.cleanupRRDCache(now)
|
|
m.cleanupTrackingMaps(now)
|
|
m.cleanupMetricsHistory()
|
|
m.cleanupRateTracker(now)
|
|
m.evaluateNotificationDelivery(now)
|
|
if mock.IsMockEnabled() {
|
|
// In mock mode, keep synthetic alerts fresh
|
|
go m.checkMockAlerts()
|
|
if keepRealPolling {
|
|
// Keep real metrics flowing while mock UI mode is active.
|
|
go m.poll(ctx, wsHub)
|
|
}
|
|
} else {
|
|
// Poll real infrastructure
|
|
go m.poll(ctx, wsHub)
|
|
}
|
|
// Connection-degraded alerts derive from the unified connections
|
|
// ledger, which lives behind the api layer. We invoke the
|
|
// registered lister here so a wedged PVE/PBS/PMG/VMware/TrueNAS
|
|
// connection escalates into the top-nav alert stream.
|
|
go m.checkConnectionAlerts()
|
|
|
|
case <-broadcastTicker.C:
|
|
// Broadcast current state regardless of polling status
|
|
// GetState lazily initializes the mock alert snapshot. Preserve that
|
|
// fixture maintenance even when no browser is subscribed; production
|
|
// monitors still avoid the full snapshot build on this fast path.
|
|
if mock.IsMockEnabled() {
|
|
_ = m.GetState()
|
|
}
|
|
if !currentStateBroadcasterHasSubscribers(wsHub, m.GetOrgID()) {
|
|
continue
|
|
}
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
// Use GetState() instead of m.state.GetSnapshot() to respect mock mode
|
|
state := m.GetState()
|
|
log.Debug().
|
|
Int("nodes", len(state.Nodes)).
|
|
Int("vms", len(state.VMs)).
|
|
Int("containers", len(state.Containers)).
|
|
Int("hosts", len(state.Hosts)).
|
|
Int("pbs", len(state.PBSInstances)).
|
|
Int("pbsBackups", len(state.Backups.PBS)).
|
|
Int("physicalDisks", len(state.PhysicalDisks)).
|
|
Msg("Broadcasting state update (ticker)")
|
|
}
|
|
m.broadcastCurrentState(wsHub)
|
|
|
|
case <-ctx.Done():
|
|
log.Info().Msg("monitoring loop stopped")
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// wireExternalAlertCallbacks installs callbacks whose effects leave the alert
|
|
// manager. It is safe to call again: these are single callback slots, so Start
|
|
// replaces the constructor-time escalation closure with one carrying its hub.
|
|
func (m *Monitor) wireExternalAlertCallbacks(wsHub *websocket.Hub) {
|
|
if m == nil || m.alertManager == nil {
|
|
return
|
|
}
|
|
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
|
m.handleAlertFired(alert)
|
|
})
|
|
// AI analysis bypasses activation and notification suppression so findings
|
|
// can be prepared while alert delivery is still pending review.
|
|
m.alertManager.SetAlertForAICallback(func(alert *alerts.Alert) {
|
|
log.Debug().Str("alertID", alert.ID).Msg("AI alert callback invoked (bypassing notification suppression)")
|
|
m.mu.RLock()
|
|
callback := m.alertTriggeredAICallback
|
|
m.mu.RUnlock()
|
|
if callback != nil {
|
|
callback(alert)
|
|
}
|
|
})
|
|
m.alertManager.SetResolvedCallback(func(alertID string) {
|
|
m.handleAlertResolved(alertID)
|
|
// Don't broadcast full state here - it causes a cascade with many guests.
|
|
// The frontend gets updated alerts through the regular broadcast ticker.
|
|
})
|
|
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
|
|
m.handleAlertEscalated(wsHub, alert, level)
|
|
})
|
|
}
|
|
|
|
// poll fetches data from all configured instances
|
|
func (m *Monitor) poll(_ context.Context, wsHub *websocket.Hub) {
|
|
defer recoverFromPanic("poll")
|
|
|
|
// Limit concurrent polls to 2 to prevent resource exhaustion
|
|
currentCount := atomic.AddInt32(&m.activePollCount, 1)
|
|
if currentCount > 2 {
|
|
atomic.AddInt32(&m.activePollCount, -1)
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Int32("activePolls", currentCount-1).Msg("too many concurrent polls, skipping")
|
|
}
|
|
return
|
|
}
|
|
defer atomic.AddInt32(&m.activePollCount, -1)
|
|
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Msg("starting polling cycle")
|
|
}
|
|
startTime := time.Now()
|
|
now := startTime
|
|
|
|
plannedTasks := m.buildScheduledTasks(now)
|
|
for _, task := range plannedTasks {
|
|
m.taskQueue.Upsert(task)
|
|
}
|
|
m.updateQueueDepthMetric()
|
|
|
|
// Update performance metrics atomically to prevent data races when
|
|
// multiple poll() goroutines run concurrently (e.g. mock mode transitions).
|
|
wsClients := 0
|
|
if wsHub != nil {
|
|
wsClients = wsHub.GetClientCount()
|
|
}
|
|
m.state.UpdatePollStats(
|
|
time.Since(startTime).Seconds(),
|
|
int64(time.Since(m.startTime).Seconds()),
|
|
wsClients,
|
|
)
|
|
|
|
// Sync alert state so broadcasts include the latest acknowledgement data
|
|
m.syncAlertsToState()
|
|
|
|
// Increment poll counter
|
|
m.mu.Lock()
|
|
m.pollCounter++
|
|
m.mu.Unlock()
|
|
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Dur("duration", time.Since(startTime)).Msg("polling cycle completed")
|
|
}
|
|
|
|
// Broadcasting is now handled by the timer in Start()
|
|
}
|
|
|
|
func (m *Monitor) startTaskWorkers(ctx context.Context, workers int) {
|
|
if m.taskQueue == nil {
|
|
return
|
|
}
|
|
if limiter := processPollTaskWorkerLimiter(); limiter != nil {
|
|
// A monitor owns one dispatcher, while the process-wide limiter owns
|
|
// actual task concurrency. This prevents the override from multiplying
|
|
// into a full worker pool for every tenant.
|
|
go m.dispatchTaskWorkers(ctx, limiter)
|
|
return
|
|
}
|
|
workers = resolveTaskWorkerCount(workers)
|
|
for i := 0; i < workers; i++ {
|
|
go m.taskWorker(ctx, i)
|
|
}
|
|
}
|
|
|
|
const maxPollTaskWorkers = 128
|
|
|
|
type pollTaskWorkerLimiter struct {
|
|
slots chan struct{}
|
|
}
|
|
|
|
func newPollTaskWorkerLimiter(limit int) *pollTaskWorkerLimiter {
|
|
if limit < 1 {
|
|
limit = 1
|
|
}
|
|
return &pollTaskWorkerLimiter{slots: make(chan struct{}, limit)}
|
|
}
|
|
|
|
func (l *pollTaskWorkerLimiter) acquire(ctx context.Context) bool {
|
|
select {
|
|
case l.slots <- struct{}{}:
|
|
return true
|
|
case <-ctx.Done():
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (l *pollTaskWorkerLimiter) release() {
|
|
<-l.slots
|
|
}
|
|
|
|
var (
|
|
processPollTaskWorkerLimiterOnce sync.Once
|
|
processPollTaskWorkerLimit *pollTaskWorkerLimiter
|
|
)
|
|
|
|
func processPollTaskWorkerLimiter() *pollTaskWorkerLimiter {
|
|
processPollTaskWorkerLimiterOnce.Do(func() {
|
|
override := pollTaskWorkerOverride()
|
|
if override == 0 {
|
|
return
|
|
}
|
|
processPollTaskWorkerLimit = newPollTaskWorkerLimiter(override)
|
|
log.Info().
|
|
Int("workers", override).
|
|
Msg("Poll task worker count overridden process-wide by POLL_TASK_WORKERS")
|
|
})
|
|
return processPollTaskWorkerLimit
|
|
}
|
|
|
|
func pollTaskWorkerOverride() int {
|
|
override := parseNonNegativeIntEnv("POLL_TASK_WORKERS", 0)
|
|
if override > maxPollTaskWorkers {
|
|
return maxPollTaskWorkers
|
|
}
|
|
return override
|
|
}
|
|
|
|
// resolveTaskWorkerCount applies the default clamp of [1, 10] to the
|
|
// client-derived worker count. POLL_TASK_WORKERS is reported here for
|
|
// diagnostics and tests; startTaskWorkers routes that override through the
|
|
// process-wide limiter instead of starting this many workers per monitor.
|
|
func resolveTaskWorkerCount(workers int) int {
|
|
if override := pollTaskWorkerOverride(); override > 0 {
|
|
return override
|
|
}
|
|
if workers < 1 {
|
|
workers = 1
|
|
}
|
|
if workers > 10 {
|
|
workers = 10
|
|
}
|
|
return workers
|
|
}
|
|
|
|
func (m *Monitor) dispatchTaskWorkers(ctx context.Context, limiter *pollTaskWorkerLimiter) {
|
|
defer recoverFromPanic("taskWorkerDispatcher")
|
|
|
|
for {
|
|
task, ok := m.taskQueue.WaitNext(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !limiter.acquire(ctx) {
|
|
return
|
|
}
|
|
|
|
go func() {
|
|
defer limiter.release()
|
|
defer recoverFromPanic(fmt.Sprintf("taskWorker-%s-%s", task.InstanceType, task.InstanceName))
|
|
m.executeAndRescheduleTask(ctx, task)
|
|
}()
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) taskWorker(ctx context.Context, id int) {
|
|
defer recoverFromPanic(fmt.Sprintf("taskWorker-%d", id))
|
|
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Int("worker", id).Msg("task worker started")
|
|
}
|
|
for {
|
|
task, ok := m.taskQueue.WaitNext(ctx)
|
|
if !ok {
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Int("worker", id).Msg("task worker stopping")
|
|
}
|
|
return
|
|
}
|
|
|
|
m.executeAndRescheduleTask(ctx, task)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) executeAndRescheduleTask(ctx context.Context, task ScheduledTask) {
|
|
m.executeScheduledTask(ctx, task)
|
|
m.rescheduleTask(task)
|
|
m.updateQueueDepthMetric()
|
|
}
|
|
|
|
func derivePollTimeout(cfg *config.Config) time.Duration {
|
|
timeout := defaultTaskTimeout
|
|
if cfg != nil && cfg.ConnectionTimeout > 0 {
|
|
// ConnectionTimeout is the budget for one provider request, while a
|
|
// poll cycle may enumerate hundreds of guests across many cluster
|
|
// nodes. A short per-request timeout must not shrink the whole-cycle
|
|
// budget below the default; that cancelled otherwise healthy large
|
|
// PVE generations at exactly 30 seconds (#1437).
|
|
if requestScaledTimeout := cfg.ConnectionTimeout * 2; requestScaledTimeout > timeout {
|
|
timeout = requestScaledTimeout
|
|
}
|
|
}
|
|
if timeout < minTaskTimeout {
|
|
timeout = minTaskTimeout
|
|
}
|
|
// Use configurable max timeout from config (set via MAX_POLL_TIMEOUT env var)
|
|
// Falls back to hardcoded maxTaskTimeout if config is nil or MaxPollTimeout not set
|
|
maxTimeout := maxTaskTimeout
|
|
if cfg != nil && cfg.MaxPollTimeout > 0 {
|
|
maxTimeout = cfg.MaxPollTimeout
|
|
}
|
|
if timeout > maxTimeout {
|
|
timeout = maxTimeout
|
|
}
|
|
return timeout
|
|
}
|
|
|
|
func (m *Monitor) taskExecutionTimeout(_ InstanceType) time.Duration {
|
|
if m == nil {
|
|
return defaultTaskTimeout
|
|
}
|
|
timeout := m.pollTimeout
|
|
if timeout <= 0 {
|
|
timeout = defaultTaskTimeout
|
|
}
|
|
return timeout
|
|
}
|
|
|
|
func (m *Monitor) executeScheduledTask(ctx context.Context, task ScheduledTask) {
|
|
if !m.allowExecution(task) {
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Msg("Task blocked by circuit breaker")
|
|
}
|
|
return
|
|
}
|
|
|
|
if m.pollMetrics != nil {
|
|
wait := time.Duration(0)
|
|
if !task.NextRun.IsZero() {
|
|
wait = time.Since(task.NextRun)
|
|
if wait < 0 {
|
|
wait = 0
|
|
}
|
|
}
|
|
instanceType := string(task.InstanceType)
|
|
if strings.TrimSpace(instanceType) == "" {
|
|
instanceType = "unknown"
|
|
}
|
|
m.pollMetrics.RecordQueueWait(instanceType, wait)
|
|
}
|
|
|
|
executor := m.getExecutor()
|
|
if executor == nil {
|
|
log.Error().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Msg("No poll executor configured; skipping task")
|
|
return
|
|
}
|
|
|
|
pollTask, ok := m.buildPollTask(task)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
taskCtx := ctx
|
|
var cancel context.CancelFunc
|
|
timeout := m.taskExecutionTimeout(task.InstanceType)
|
|
if timeout > 0 {
|
|
taskCtx, cancel = context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
}
|
|
|
|
executor.Execute(taskCtx, pollTask)
|
|
|
|
if timeout > 0 && stderrors.Is(taskCtx.Err(), context.DeadlineExceeded) {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Dur("timeout", timeout).
|
|
Msg("Polling task timed out; rescheduling with fresh worker")
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) buildPollTask(task ScheduledTask) (PollTask, bool) {
|
|
provider := m.getPollProvider(task.InstanceType)
|
|
if provider == nil {
|
|
log.Debug().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Msg("Skipping unsupported task type")
|
|
return PollTask{}, false
|
|
}
|
|
|
|
pollTask, err := provider.BuildPollTask(m, task.InstanceName)
|
|
if err != nil {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Err(err).
|
|
Msg("Skipping scheduled task")
|
|
return PollTask{}, false
|
|
}
|
|
|
|
if strings.TrimSpace(pollTask.InstanceName) == "" {
|
|
pollTask.InstanceName = task.InstanceName
|
|
}
|
|
if strings.TrimSpace(pollTask.InstanceType) == "" {
|
|
pollTask.InstanceType = string(task.InstanceType)
|
|
}
|
|
return pollTask, true
|
|
}
|
|
|
|
func (m *Monitor) rescheduleTask(task ScheduledTask) {
|
|
if m.taskQueue == nil {
|
|
return
|
|
}
|
|
|
|
key := schedulerKey(task.InstanceType, task.InstanceName)
|
|
m.mu.Lock()
|
|
outcome, hasOutcome := m.lastOutcome[key]
|
|
failureCount := m.failureCounts[key]
|
|
m.mu.Unlock()
|
|
|
|
if hasOutcome && !outcome.success {
|
|
if !outcome.transient || failureCount >= m.maxRetryAttempts {
|
|
m.sendToDeadLetter(task, outcome.err)
|
|
return
|
|
}
|
|
delay := m.backoffCfg.nextDelay(failureCount-1, m.randomFloat())
|
|
if delay <= 0 {
|
|
delay = 5 * time.Second
|
|
}
|
|
if m.config != nil && m.config.AdaptivePollingEnabled && m.config.AdaptivePollingMaxInterval > 0 && m.config.AdaptivePollingMaxInterval <= 15*time.Second {
|
|
maxDelay := 4 * time.Second
|
|
if delay > maxDelay {
|
|
delay = maxDelay
|
|
}
|
|
}
|
|
next := task
|
|
next.Interval = delay
|
|
next.NextRun = time.Now().Add(delay)
|
|
m.taskQueue.Upsert(next)
|
|
return
|
|
}
|
|
|
|
fixedInterval := m.fixedIntervalForInstance(task.InstanceType, task.InstanceName)
|
|
|
|
if m.scheduler == nil {
|
|
nextInterval := fixedInterval
|
|
if nextInterval <= 0 {
|
|
nextInterval = task.Interval
|
|
}
|
|
if nextInterval <= 0 {
|
|
nextInterval = m.baseIntervalForInstanceType(task.InstanceType)
|
|
}
|
|
if nextInterval <= 0 {
|
|
nextInterval = DefaultSchedulerConfig().BaseInterval
|
|
}
|
|
next := task
|
|
next.NextRun = time.Now().Add(nextInterval)
|
|
next.Interval = nextInterval
|
|
m.taskQueue.Upsert(next)
|
|
return
|
|
}
|
|
|
|
desc := InstanceDescriptor{
|
|
Name: task.InstanceName,
|
|
Type: task.InstanceType,
|
|
LastInterval: task.Interval,
|
|
LastScheduled: task.NextRun,
|
|
FixedInterval: fixedInterval,
|
|
}
|
|
if m.stalenessTracker != nil {
|
|
if snap, ok := m.stalenessTracker.snapshot(task.InstanceType, task.InstanceName); ok {
|
|
desc.LastSuccess = snap.LastSuccess
|
|
desc.LastFailure = snap.LastError
|
|
if snap.ChangeHash != "" {
|
|
desc.Metadata = TaskMetadata{ChangeHash: snap.ChangeHash}
|
|
}
|
|
}
|
|
}
|
|
|
|
tasks := m.scheduler.BuildPlan(time.Now(), []InstanceDescriptor{desc}, m.taskQueue.Size())
|
|
if len(tasks) == 0 {
|
|
next := task
|
|
nextInterval := task.Interval
|
|
if nextInterval <= 0 && m.config != nil {
|
|
nextInterval = m.config.AdaptivePollingBaseInterval
|
|
}
|
|
if nextInterval <= 0 {
|
|
nextInterval = DefaultSchedulerConfig().BaseInterval
|
|
}
|
|
next.Interval = nextInterval
|
|
next.NextRun = time.Now().Add(nextInterval)
|
|
m.taskQueue.Upsert(next)
|
|
return
|
|
}
|
|
for _, next := range tasks {
|
|
m.taskQueue.Upsert(next)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) sendToDeadLetter(task ScheduledTask, err error) {
|
|
if m.deadLetterQueue == nil {
|
|
log.Error().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Err(err).
|
|
Msg("Dead-letter queue unavailable; dropping task")
|
|
return
|
|
}
|
|
|
|
log.Error().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Err(err).
|
|
Msg("Routing task to dead-letter queue after repeated failures")
|
|
|
|
next := task
|
|
next.Interval = 30 * time.Minute
|
|
next.NextRun = time.Now().Add(next.Interval)
|
|
m.deadLetterQueue.Upsert(next)
|
|
m.updateDeadLetterMetrics()
|
|
|
|
key := schedulerKey(task.InstanceType, task.InstanceName)
|
|
now := time.Now()
|
|
|
|
m.mu.Lock()
|
|
if m.dlqInsightMap == nil {
|
|
m.dlqInsightMap = make(map[string]*dlqInsight)
|
|
}
|
|
info, ok := m.dlqInsightMap[key]
|
|
if !ok {
|
|
info = &dlqInsight{}
|
|
m.dlqInsightMap[key] = info
|
|
}
|
|
if info.FirstAttempt.IsZero() {
|
|
info.FirstAttempt = now
|
|
}
|
|
info.LastAttempt = now
|
|
info.RetryCount++
|
|
info.NextRetry = next.NextRun
|
|
if err != nil {
|
|
info.Reason = classifyDLQReason(err)
|
|
}
|
|
m.mu.Unlock()
|
|
}
|
|
|
|
func classifyDLQReason(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
if errors.IsRetryableError(err) {
|
|
return "max_retry_attempts"
|
|
}
|
|
return "permanent_failure"
|
|
}
|
|
|
|
func (m *Monitor) updateDeadLetterMetrics() {
|
|
if m.pollMetrics == nil || m.deadLetterQueue == nil {
|
|
return
|
|
}
|
|
|
|
size := m.deadLetterQueue.Size()
|
|
if size <= 0 {
|
|
m.pollMetrics.UpdateDeadLetterCounts(nil)
|
|
return
|
|
}
|
|
|
|
tasks := m.deadLetterQueue.PeekAll(size)
|
|
m.pollMetrics.UpdateDeadLetterCounts(tasks)
|
|
}
|
|
|
|
func (m *Monitor) updateBreakerMetric(instanceType InstanceType, instance string, breaker *circuitBreaker) {
|
|
if m.pollMetrics == nil || breaker == nil {
|
|
return
|
|
}
|
|
|
|
state, failures, retryAt, _, _ := breaker.stateDetails()
|
|
m.pollMetrics.SetBreakerState(string(instanceType), instance, state, failures, retryAt)
|
|
}
|
|
|
|
func (m *Monitor) randomFloat() float64 {
|
|
if m.rng == nil {
|
|
m.rng = rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
}
|
|
return m.rng.Float64()
|
|
}
|
|
|
|
func (m *Monitor) updateQueueDepthMetric() {
|
|
if m.pollMetrics == nil || m.taskQueue == nil {
|
|
return
|
|
}
|
|
snapshot := m.taskQueue.Snapshot()
|
|
m.pollMetrics.SetQueueDepth(snapshot.Depth)
|
|
m.pollMetrics.UpdateQueueSnapshot(snapshot)
|
|
}
|
|
|
|
func (m *Monitor) allowExecution(task ScheduledTask) bool {
|
|
if m.circuitBreakers == nil {
|
|
return true
|
|
}
|
|
key := schedulerKey(task.InstanceType, task.InstanceName)
|
|
breaker := m.ensureBreaker(key)
|
|
allowed := breaker.allow(time.Now())
|
|
m.updateBreakerMetric(task.InstanceType, task.InstanceName, breaker)
|
|
return allowed
|
|
}
|
|
|
|
func (m *Monitor) ensureBreaker(key string) *circuitBreaker {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if m.circuitBreakers == nil {
|
|
m.circuitBreakers = make(map[string]*circuitBreaker)
|
|
}
|
|
if breaker, ok := m.circuitBreakers[key]; ok {
|
|
return breaker
|
|
}
|
|
baseRetry := m.breakerBaseRetry
|
|
if baseRetry <= 0 {
|
|
baseRetry = 5 * time.Second
|
|
}
|
|
maxDelay := m.breakerMaxDelay
|
|
if maxDelay <= 0 {
|
|
maxDelay = 5 * time.Minute
|
|
}
|
|
halfOpen := m.breakerHalfOpenWindow
|
|
if halfOpen <= 0 {
|
|
halfOpen = 30 * time.Second
|
|
}
|
|
breaker := newCircuitBreaker(3, baseRetry, maxDelay, halfOpen)
|
|
m.circuitBreakers[key] = breaker
|
|
return breaker
|
|
}
|
|
|
|
func (m *Monitor) recordTaskResult(instanceType InstanceType, instance string, pollErr error) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
key := schedulerKey(instanceType, instance)
|
|
now := time.Now()
|
|
|
|
breaker := m.ensureBreaker(key)
|
|
|
|
m.mu.Lock()
|
|
status, ok := m.pollStatusMap[key]
|
|
if !ok {
|
|
status = &pollStatus{}
|
|
m.pollStatusMap[key] = status
|
|
}
|
|
|
|
if pollErr == nil {
|
|
if m.failureCounts != nil {
|
|
m.failureCounts[key] = 0
|
|
}
|
|
if m.lastOutcome != nil {
|
|
m.lastOutcome[key] = taskOutcome{
|
|
success: true,
|
|
transient: true,
|
|
err: nil,
|
|
recordedAt: now,
|
|
}
|
|
}
|
|
status.LastSuccess = now
|
|
status.ConsecutiveFailures = 0
|
|
status.FirstFailureAt = time.Time{}
|
|
// A recorded error means "current outstanding failure", not "last error
|
|
// ever". Clearing it on success keeps downstream consumers honest: the
|
|
// connections aggregator surfaces LastError as a live error banner and
|
|
// derives Unauthorized state from it, so a stale entry would show a
|
|
// red error on a healthy connection forever (#1493).
|
|
status.LastErrorAt = time.Time{}
|
|
status.LastErrorMessage = ""
|
|
status.LastErrorCategory = ""
|
|
m.mu.Unlock()
|
|
if breaker != nil {
|
|
breaker.recordSuccess()
|
|
m.updateBreakerMetric(instanceType, instance, breaker)
|
|
}
|
|
return
|
|
}
|
|
|
|
transient := isTransientError(pollErr)
|
|
category := "permanent"
|
|
if transient {
|
|
category = "transient"
|
|
}
|
|
if m.failureCounts != nil {
|
|
m.failureCounts[key] = m.failureCounts[key] + 1
|
|
}
|
|
if m.lastOutcome != nil {
|
|
m.lastOutcome[key] = taskOutcome{
|
|
success: false,
|
|
transient: transient,
|
|
err: pollErr,
|
|
recordedAt: now,
|
|
}
|
|
}
|
|
status.LastErrorAt = now
|
|
status.LastErrorMessage = pollErr.Error()
|
|
status.LastErrorCategory = category
|
|
status.ConsecutiveFailures++
|
|
if status.ConsecutiveFailures == 1 {
|
|
status.FirstFailureAt = now
|
|
}
|
|
m.mu.Unlock()
|
|
if breaker != nil {
|
|
breaker.recordFailure(now)
|
|
m.updateBreakerMetric(instanceType, instance, breaker)
|
|
}
|
|
}
|
|
|
|
// SchedulerHealthResponse contains complete scheduler health data for API exposure.
|
|
type SchedulerHealthResponse struct {
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
Enabled bool `json:"enabled"`
|
|
Queue QueueSnapshot `json:"queue"`
|
|
DeadLetter DeadLetterSnapshot `json:"deadLetter"`
|
|
Breakers []BreakerSnapshot `json:"breakers"`
|
|
Staleness []StalenessSnapshot `json:"staleness"`
|
|
Instances []InstanceHealth `json:"instances"`
|
|
}
|
|
|
|
// DeadLetterSnapshot contains dead-letter queue data.
|
|
type DeadLetterSnapshot struct {
|
|
Count int `json:"count"`
|
|
Tasks []DeadLetterTask `json:"tasks"`
|
|
}
|
|
|
|
func emptyDeadLetterSnapshot() DeadLetterSnapshot {
|
|
return DeadLetterSnapshot{
|
|
Tasks: []DeadLetterTask{},
|
|
}
|
|
}
|
|
|
|
func emptySchedulerHealthResponse(enabled bool) SchedulerHealthResponse {
|
|
return SchedulerHealthResponse{
|
|
UpdatedAt: time.Now(),
|
|
Enabled: enabled,
|
|
Queue: emptyQueueSnapshot(),
|
|
DeadLetter: emptyDeadLetterSnapshot(),
|
|
Breakers: []BreakerSnapshot{},
|
|
Staleness: []StalenessSnapshot{},
|
|
Instances: []InstanceHealth{},
|
|
}
|
|
}
|
|
|
|
// SchedulerHealth returns a complete snapshot of scheduler health for API exposure.
|
|
// DeadLetterCount reports the number of dead-lettered poll tasks using only
|
|
// the queue's own short-lived mutex. Liveness probes (/api/health) use this
|
|
// instead of SchedulerHealth, which refreshes provider caches under the
|
|
// monitor's write lock on every call — far more work and lock surface than a
|
|
// watchdog-polled liveness check should pay for.
|
|
func (m *Monitor) DeadLetterCount() int {
|
|
if m == nil || m.deadLetterQueue == nil {
|
|
return 0
|
|
}
|
|
return m.deadLetterQueue.Size()
|
|
}
|
|
|
|
// PlannedPollInterval reports the adaptive scheduler's currently planned
|
|
// interval for one instance, or zero when no plan exists (adaptive polling
|
|
// disabled, unknown instance, or startup before the first plan). Consumers
|
|
// use it to judge poll freshness against the schedule actually in force
|
|
// rather than the configured cadence, which adaptive polling deliberately
|
|
// stretches while data is fresh.
|
|
func (m *Monitor) PlannedPollInterval(instanceType InstanceType, instanceName string) time.Duration {
|
|
if m == nil || m.scheduler == nil {
|
|
return 0
|
|
}
|
|
if task, ok := m.scheduler.LastScheduled(instanceType, instanceName); ok {
|
|
return task.Interval
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (m *Monitor) SchedulerHealth() SchedulerHealthResponse {
|
|
response := emptySchedulerHealthResponse(m.config != nil && m.config.AdaptivePollingEnabled)
|
|
|
|
m.refreshInstanceInfoCacheFromProviders()
|
|
|
|
// Queue snapshot
|
|
if m.taskQueue != nil {
|
|
response.Queue = m.taskQueue.Snapshot()
|
|
if m.pollMetrics != nil {
|
|
m.pollMetrics.UpdateQueueSnapshot(response.Queue)
|
|
}
|
|
}
|
|
|
|
// Dead-letter queue snapshot
|
|
if m.deadLetterQueue != nil {
|
|
deadLetterTasks := m.deadLetterQueue.PeekAll(25) // limit to top 25
|
|
m.mu.RLock()
|
|
for i := range deadLetterTasks {
|
|
key := schedulerKey(InstanceType(deadLetterTasks[i].Type), deadLetterTasks[i].Instance)
|
|
if outcome, ok := m.lastOutcome[key]; ok && outcome.err != nil {
|
|
deadLetterTasks[i].LastError = outcome.err.Error()
|
|
}
|
|
if count, ok := m.failureCounts[key]; ok {
|
|
deadLetterTasks[i].Failures = count
|
|
}
|
|
}
|
|
m.mu.RUnlock()
|
|
response.DeadLetter = DeadLetterSnapshot{
|
|
Count: m.deadLetterQueue.Size(),
|
|
Tasks: deadLetterTasks,
|
|
}
|
|
m.updateDeadLetterMetrics()
|
|
}
|
|
|
|
// Circuit breaker snapshots
|
|
m.mu.RLock()
|
|
breakerSnapshots := make([]BreakerSnapshot, 0, len(m.circuitBreakers))
|
|
for key, breaker := range m.circuitBreakers {
|
|
state, failures, retryAt := breaker.State()
|
|
// Only include breakers that are not in default closed state with 0 failures
|
|
if state != "closed" || failures > 0 {
|
|
// Parse instance type and name from key
|
|
parts := strings.SplitN(key, "::", 2)
|
|
instanceType, instanceName := "unknown", key
|
|
if len(parts) == 2 {
|
|
instanceType, instanceName = parts[0], parts[1]
|
|
}
|
|
breakerSnapshots = append(breakerSnapshots, BreakerSnapshot{
|
|
Instance: instanceName,
|
|
Type: instanceType,
|
|
State: state,
|
|
Failures: failures,
|
|
RetryAt: retryAt,
|
|
})
|
|
}
|
|
}
|
|
m.mu.RUnlock()
|
|
response.Breakers = breakerSnapshots
|
|
|
|
// Staleness snapshots
|
|
if m.stalenessTracker != nil {
|
|
response.Staleness = m.stalenessTracker.Snapshot()
|
|
}
|
|
|
|
instanceInfos := make(map[string]*instanceInfo)
|
|
pollStatuses := make(map[string]pollStatus)
|
|
dlqInsights := make(map[string]dlqInsight)
|
|
breakerRefs := make(map[string]*circuitBreaker)
|
|
|
|
m.mu.RLock()
|
|
for k, v := range m.instanceInfoCache {
|
|
if v == nil {
|
|
continue
|
|
}
|
|
copyVal := *v
|
|
instanceInfos[k] = ©Val
|
|
}
|
|
for k, v := range m.pollStatusMap {
|
|
if v == nil {
|
|
continue
|
|
}
|
|
pollStatuses[k] = *v
|
|
}
|
|
for k, v := range m.dlqInsightMap {
|
|
if v == nil {
|
|
continue
|
|
}
|
|
dlqInsights[k] = *v
|
|
}
|
|
for k, v := range m.circuitBreakers {
|
|
if v != nil {
|
|
breakerRefs[k] = v
|
|
}
|
|
}
|
|
m.mu.RUnlock()
|
|
for key, breaker := range breakerRefs {
|
|
instanceType := InstanceType("unknown")
|
|
instanceName := key
|
|
if parts := strings.SplitN(key, "::", 2); len(parts) == 2 {
|
|
if parts[0] != "" {
|
|
instanceType = InstanceType(parts[0])
|
|
}
|
|
if parts[1] != "" {
|
|
instanceName = parts[1]
|
|
}
|
|
}
|
|
m.updateBreakerMetric(instanceType, instanceName, breaker)
|
|
}
|
|
|
|
keySet := make(map[string]struct{})
|
|
for k := range instanceInfos {
|
|
if k != "" {
|
|
keySet[k] = struct{}{}
|
|
}
|
|
}
|
|
for k := range pollStatuses {
|
|
if k != "" {
|
|
keySet[k] = struct{}{}
|
|
}
|
|
}
|
|
for k := range dlqInsights {
|
|
if k != "" {
|
|
keySet[k] = struct{}{}
|
|
}
|
|
}
|
|
for k := range breakerRefs {
|
|
if k != "" {
|
|
keySet[k] = struct{}{}
|
|
}
|
|
}
|
|
for _, task := range response.DeadLetter.Tasks {
|
|
if task.Instance == "" {
|
|
continue
|
|
}
|
|
keySet[schedulerKey(InstanceType(task.Type), task.Instance)] = struct{}{}
|
|
}
|
|
for _, snap := range response.Staleness {
|
|
if snap.Instance == "" {
|
|
continue
|
|
}
|
|
keySet[schedulerKey(InstanceType(snap.Type), snap.Instance)] = struct{}{}
|
|
}
|
|
|
|
if len(keySet) > 0 {
|
|
keys := make([]string, 0, len(keySet))
|
|
for k := range keySet {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
instances := make([]InstanceHealth, 0, len(keys))
|
|
for _, key := range keys {
|
|
instType := "unknown"
|
|
instName := key
|
|
if parts := strings.SplitN(key, "::", 2); len(parts) == 2 {
|
|
if parts[0] != "" {
|
|
instType = parts[0]
|
|
}
|
|
if parts[1] != "" {
|
|
instName = parts[1]
|
|
}
|
|
}
|
|
instType = strings.TrimSpace(instType)
|
|
instName = strings.TrimSpace(instName)
|
|
|
|
info := instanceInfos[key]
|
|
display := instName
|
|
connection := ""
|
|
if info != nil {
|
|
if instType == "unknown" || instType == "" {
|
|
if info.Type != "" {
|
|
instType = string(info.Type)
|
|
}
|
|
}
|
|
if strings.Contains(info.Key, "::") {
|
|
if parts := strings.SplitN(info.Key, "::", 2); len(parts) == 2 {
|
|
if instName == key {
|
|
instName = parts[1]
|
|
}
|
|
if (instType == "" || instType == "unknown") && parts[0] != "" {
|
|
instType = parts[0]
|
|
}
|
|
}
|
|
}
|
|
if info.DisplayName != "" {
|
|
display = info.DisplayName
|
|
}
|
|
if info.Connection != "" {
|
|
connection = info.Connection
|
|
}
|
|
}
|
|
display = strings.TrimSpace(display)
|
|
connection = strings.TrimSpace(connection)
|
|
if display == "" {
|
|
display = instName
|
|
}
|
|
if display == "" {
|
|
display = connection
|
|
}
|
|
if instType == "" {
|
|
instType = "unknown"
|
|
}
|
|
if instName == "" {
|
|
instName = key
|
|
}
|
|
|
|
status, hasStatus := pollStatuses[key]
|
|
instanceStatus := InstancePollStatus{}
|
|
if hasStatus {
|
|
instanceStatus.ConsecutiveFailures = status.ConsecutiveFailures
|
|
instanceStatus.LastSuccess = timePtr(status.LastSuccess)
|
|
if !status.FirstFailureAt.IsZero() {
|
|
instanceStatus.FirstFailureAt = timePtr(status.FirstFailureAt)
|
|
}
|
|
if !status.LastErrorAt.IsZero() && status.LastErrorMessage != "" {
|
|
instanceStatus.LastError = &ErrorDetail{
|
|
At: status.LastErrorAt,
|
|
Message: status.LastErrorMessage,
|
|
Category: status.LastErrorCategory,
|
|
}
|
|
}
|
|
}
|
|
|
|
breakerInfo := InstanceBreaker{
|
|
State: "closed",
|
|
FailureCount: 0,
|
|
}
|
|
if br, ok := breakerRefs[key]; ok && br != nil {
|
|
state, failures, retryAt, since, lastTransition := br.stateDetails()
|
|
if state != "" {
|
|
breakerInfo.State = state
|
|
}
|
|
breakerInfo.FailureCount = failures
|
|
breakerInfo.RetryAt = timePtr(retryAt)
|
|
breakerInfo.Since = timePtr(since)
|
|
breakerInfo.LastTransition = timePtr(lastTransition)
|
|
}
|
|
|
|
dlqInfo := InstanceDLQ{Present: false}
|
|
if dlq, ok := dlqInsights[key]; ok {
|
|
dlqInfo.Present = true
|
|
dlqInfo.Reason = dlq.Reason
|
|
dlqInfo.FirstAttempt = timePtr(dlq.FirstAttempt)
|
|
dlqInfo.LastAttempt = timePtr(dlq.LastAttempt)
|
|
dlqInfo.RetryCount = dlq.RetryCount
|
|
dlqInfo.NextRetry = timePtr(dlq.NextRetry)
|
|
}
|
|
|
|
// Collect any warnings for this instance
|
|
var warnings []string
|
|
if instType == "pve" {
|
|
if warning, ok := m.backupPermissionWarnings[instName]; ok {
|
|
warnings = append(warnings, warning)
|
|
}
|
|
}
|
|
|
|
instances = append(instances, InstanceHealth{
|
|
Key: key,
|
|
Type: instType,
|
|
DisplayName: display,
|
|
Instance: instName,
|
|
Connection: connection,
|
|
PollStatus: instanceStatus,
|
|
Breaker: breakerInfo,
|
|
DeadLetter: dlqInfo,
|
|
Warnings: warnings,
|
|
}.NormalizeCollections())
|
|
}
|
|
|
|
response.Instances = instances
|
|
} else {
|
|
response.Instances = []InstanceHealth{}
|
|
}
|
|
|
|
return response
|
|
}
|
|
|
|
func isTransientError(err error) bool {
|
|
if err == nil {
|
|
return true
|
|
}
|
|
if errors.IsRetryableError(err) {
|
|
return true
|
|
}
|
|
if stderrors.Is(err, context.Canceled) || stderrors.Is(err, context.DeadlineExceeded) {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (m *Monitor) GetState() models.StateSnapshot {
|
|
if m == nil {
|
|
return models.StateSnapshot{}
|
|
}
|
|
|
|
// Check if mock mode is enabled
|
|
if mock.IsMockEnabled() {
|
|
state := mock.CurrentFixtureGraph().State
|
|
if state.ActiveAlerts == nil && m.alertManager != nil {
|
|
// Populate snapshot lazily if the cache hasn't been filled yet.
|
|
mock.UpdateAlertSnapshots(m.alertManager.GetActiveAlerts(), m.alertManager.GetRecentlyResolved())
|
|
state = mock.CurrentFixtureGraph().State
|
|
}
|
|
return state
|
|
}
|
|
if m.state == nil {
|
|
return models.StateSnapshot{}
|
|
}
|
|
|
|
state := m.state.GetSnapshot()
|
|
// Keep externally served alert arrays aligned with the live alert manager
|
|
// even between explicit sync points, so APIs do not expose stale alert
|
|
// counts or recently resolved incidents from cached state.
|
|
state.ActiveAlerts = m.activeAlertsSnapshot()
|
|
state.RecentlyResolved = m.recentlyResolvedAlertsSnapshot()
|
|
// Surface filesystems reported by a unified pulse-agent inside a guest
|
|
// (for example ZFS mounts that qemu-guest-agent's get-fsinfo cannot see
|
|
// on PBS, #1438) in the guest overview disk listing.
|
|
state.MergeLinkedHostDisksIntoGuests()
|
|
return state
|
|
}
|
|
|
|
// ReadSnapshot returns a snapshot of the current infrastructure state,
|
|
// respecting mock mode when enabled.
|
|
//
|
|
// This is the preferred accessor for consumer code that needs the full
|
|
// StateSnapshot (e.g., chart rendering, reporting, AI state queries).
|
|
// This method satisfies models.SnapshotProvider — the single canonical
|
|
// interface that all consumer packages depend on. Fields available
|
|
// via ReadState should be accessed there instead when practical.
|
|
func (m *Monitor) ReadSnapshot() models.StateSnapshot {
|
|
return m.GetState()
|
|
}
|
|
|
|
// BackupsSnapshot returns the current backup state.
|
|
func (m *Monitor) BackupsSnapshot() models.Backups {
|
|
return m.GetState().Backups
|
|
}
|
|
|
|
// PBSInstancesSnapshot returns the current PBS instances.
|
|
func (m *Monitor) PBSInstancesSnapshot() []models.PBSInstance {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
pbsViews := readState.PBSInstances()
|
|
if len(pbsViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
instances := make([]models.PBSInstance, 0, len(pbsViews))
|
|
for _, instance := range pbsViews {
|
|
if instance == nil {
|
|
continue
|
|
}
|
|
instances = append(instances, pbsInstanceFromReadStateView(instance))
|
|
}
|
|
return instances
|
|
}
|
|
|
|
// ReplicationJobsSnapshot returns the current replication jobs.
|
|
func (m *Monitor) ReplicationJobsSnapshot() []models.ReplicationJob {
|
|
return m.GetState().ReplicationJobs
|
|
}
|
|
|
|
// ConnectionHealthSnapshot returns the current connection health map.
|
|
func (m *Monitor) ConnectionHealthSnapshot() map[string]bool {
|
|
return m.GetState().ConnectionHealth
|
|
}
|
|
|
|
// HostsSnapshot returns the current hosts.
|
|
func (m *Monitor) HostsSnapshot() []models.Host {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
readState = m.readStateWithStandaloneHostContinuity(readState)
|
|
|
|
hostViews := readState.Hosts()
|
|
if len(hostViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
hosts := make([]models.Host, 0, len(hostViews))
|
|
for _, host := range hostViews {
|
|
if host == nil {
|
|
continue
|
|
}
|
|
hosts = append(hosts, hostFromReadStateView(host))
|
|
}
|
|
return hosts
|
|
}
|
|
|
|
// VMsSnapshot returns the current VMs.
|
|
func (m *Monitor) VMsSnapshot() []models.VM {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
vmViews := readState.VMs()
|
|
if len(vmViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
vms := make([]models.VM, 0, len(vmViews))
|
|
for _, vm := range vmViews {
|
|
if vm == nil {
|
|
continue
|
|
}
|
|
vms = append(vms, vmFromReadStateView(vm))
|
|
}
|
|
return vms
|
|
}
|
|
|
|
// ContainersSnapshot returns the current system containers.
|
|
func (m *Monitor) ContainersSnapshot() []models.Container {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
containerViews := readState.Containers()
|
|
if len(containerViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
containers := make([]models.Container, 0, len(containerViews))
|
|
for _, container := range containerViews {
|
|
if container == nil {
|
|
continue
|
|
}
|
|
containers = append(containers, containerFromReadStateView(container))
|
|
}
|
|
return containers
|
|
}
|
|
|
|
// NodesSnapshot returns the current Proxmox nodes.
|
|
func (m *Monitor) NodesSnapshot() []models.Node {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
nodeViews := readState.Nodes()
|
|
if len(nodeViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
nodes := make([]models.Node, 0, len(nodeViews))
|
|
for _, node := range nodeViews {
|
|
if node == nil {
|
|
continue
|
|
}
|
|
nodes = append(nodes, nodeFromReadStateView(node))
|
|
}
|
|
return nodes
|
|
}
|
|
|
|
// DockerHostsSnapshot returns the current Docker hosts.
|
|
func (m *Monitor) DockerHostsSnapshot() []models.DockerHost {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
dockerViews := readState.DockerHosts()
|
|
if len(dockerViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
hosts := make([]models.DockerHost, 0, len(dockerViews))
|
|
for _, host := range dockerViews {
|
|
if host == nil {
|
|
continue
|
|
}
|
|
hosts = append(hosts, dockerHostFromReadStateView(host))
|
|
}
|
|
return hosts
|
|
}
|
|
|
|
// StorageSnapshot returns the current storage pools.
|
|
func (m *Monitor) StorageSnapshot() []models.Storage {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
storagePools := readState.StoragePools()
|
|
if len(storagePools) == 0 {
|
|
return nil
|
|
}
|
|
|
|
storage := make([]models.Storage, 0, len(storagePools))
|
|
for _, pool := range storagePools {
|
|
if pool == nil {
|
|
continue
|
|
}
|
|
storage = append(storage, storageFromReadStateView(pool))
|
|
}
|
|
return storage
|
|
}
|
|
|
|
func storageFromReadStateView(view *unifiedresources.StoragePoolView) models.Storage {
|
|
if view == nil {
|
|
return models.Storage{}
|
|
}
|
|
|
|
storageID := strings.TrimSpace(view.SourceID())
|
|
if storageID == "" {
|
|
storageID = strings.TrimSpace(view.ID())
|
|
}
|
|
|
|
nodes := view.AccessibleNodes()
|
|
nodeIDs := storageNodeIDsFromReadState(view.Instance(), nodes)
|
|
total := view.DiskTotal()
|
|
used := view.DiskUsed()
|
|
free := total - used
|
|
if free < 0 {
|
|
free = 0
|
|
}
|
|
|
|
return models.Storage{
|
|
ID: storageID,
|
|
Name: view.Name(),
|
|
Node: view.Node(),
|
|
Instance: view.Instance(),
|
|
Nodes: nodes,
|
|
NodeIDs: nodeIDs,
|
|
NodeCount: len(nodes),
|
|
Type: view.StorageType(),
|
|
Status: string(view.Status()),
|
|
Path: view.Path(),
|
|
Total: total,
|
|
Used: used,
|
|
Free: free,
|
|
Usage: view.DiskPercent(),
|
|
Content: view.Content(),
|
|
Shared: view.Shared(),
|
|
Enabled: view.Enabled(),
|
|
Active: view.Active(),
|
|
LastSeen: view.LastSeen(),
|
|
ZFSPool: storageZFSPoolFromReadStateView(view),
|
|
}
|
|
}
|
|
|
|
func proxmoxRatioFromPercent(percent float64) float64 {
|
|
if math.IsNaN(percent) || math.IsInf(percent, 0) || percent <= 0 {
|
|
return 0
|
|
}
|
|
return percent / 100
|
|
}
|
|
|
|
func nodeFromReadStateView(view *unifiedresources.NodeView) models.Node {
|
|
if view == nil {
|
|
return models.Node{}
|
|
}
|
|
|
|
name := view.NodeName()
|
|
displayName := ""
|
|
if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != name {
|
|
displayName = trimmed
|
|
}
|
|
|
|
return models.Node{
|
|
ID: firstNonEmptyString(view.SourceID(), view.ID()),
|
|
Name: name,
|
|
DisplayName: displayName,
|
|
Instance: view.Instance(),
|
|
Host: view.HostURL(),
|
|
GuestURL: view.GuestURL(),
|
|
Status: string(view.Status()),
|
|
Type: "node",
|
|
CPU: proxmoxRatioFromPercent(view.CPUPercent()),
|
|
Memory: models.Memory{Used: view.MemoryUsed(), Total: view.MemoryTotal(), Free: maxInt64(0, view.MemoryTotal()-view.MemoryUsed()), Usage: view.MemoryPercent()},
|
|
Disk: models.Disk{Used: view.DiskUsed(), Total: view.DiskTotal(), Free: maxInt64(0, view.DiskTotal()-view.DiskUsed()), Usage: view.DiskPercent()},
|
|
Uptime: view.Uptime(),
|
|
LoadAverage: view.LoadAverage(),
|
|
KernelVersion: view.KernelVersion(),
|
|
PVEVersion: view.PVEVersion(),
|
|
CPUInfo: view.CPUInfo(),
|
|
NetworkInterfaces: hostNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
Temperature: view.TemperatureDetails(),
|
|
TemperatureMonitoringEnabled: view.TemperatureMonitoringEnabled(),
|
|
LastSeen: view.LastSeen(),
|
|
ConnectionHealth: view.ConnectionHealth(),
|
|
IsClusterMember: view.IsClusterMember(),
|
|
ClusterName: view.ClusterName(),
|
|
PendingUpdates: view.PendingUpdates(),
|
|
PendingUpdatesCheckedAt: view.PendingUpdatesCheckedAt(),
|
|
PendingUpdatesStatus: view.PendingUpdatesStatus(),
|
|
PendingUpdatesReason: view.PendingUpdatesReason(),
|
|
LinkedAgentID: view.LinkedAgentID(),
|
|
}
|
|
}
|
|
|
|
func hostFromReadStateView(view *unifiedresources.HostView) models.Host {
|
|
if view == nil {
|
|
return models.Host{}
|
|
}
|
|
|
|
displayName := ""
|
|
if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != view.Hostname() {
|
|
displayName = trimmed
|
|
}
|
|
|
|
return models.Host{
|
|
ID: firstNonEmptyString(view.AgentID(), view.ID()),
|
|
Hostname: view.Hostname(),
|
|
DisplayName: displayName,
|
|
Platform: view.Platform(),
|
|
OSName: view.OSName(),
|
|
OSVersion: view.OSVersion(),
|
|
KernelVersion: view.KernelVersion(),
|
|
Architecture: view.Architecture(),
|
|
CPUCount: view.CPUCount(),
|
|
CPUUsage: view.CPUPercent(),
|
|
Memory: hostMemoryFromReadStateView(view),
|
|
LoadAverage: view.LoadAverage(),
|
|
Disks: hostDisksFromReadStateView(view.Disks()),
|
|
DiskIO: hostDiskIOFromReadStateView(view.DiskIO()),
|
|
NetworkInterfaces: hostNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
Sensors: hostSensorsFromReadStateView(view.Sensors()),
|
|
RAID: hostRAIDFromReadStateView(view.RAID()),
|
|
Unraid: hostUnraidFromReadStateView(view.Unraid()),
|
|
Ceph: hostCephFromReadStateView(view.Ceph()),
|
|
Status: string(view.Status()),
|
|
UptimeSeconds: view.UptimeSeconds(),
|
|
IntervalSeconds: view.IntervalSeconds(),
|
|
LastSeen: view.LastSeen(),
|
|
AgentVersion: view.AgentVersion(),
|
|
IntegrationSource: view.IntegrationSource(),
|
|
MachineID: view.MachineID(),
|
|
CommandsEnabled: view.CommandsEnabled(),
|
|
ReportIP: view.ReportIP(),
|
|
TokenID: view.TokenID(),
|
|
TokenName: view.TokenName(),
|
|
TokenHint: view.TokenHint(),
|
|
TokenLastUsedAt: view.TokenLastUsedAt(),
|
|
Tags: view.Tags(),
|
|
DiskExclude: view.DiskExclude(),
|
|
IsLegacy: view.IsLegacy(),
|
|
NetInRate: view.NetInRate(),
|
|
NetOutRate: view.NetOutRate(),
|
|
DiskReadRate: view.DiskReadRate(),
|
|
DiskWriteRate: view.DiskWriteRate(),
|
|
LinkedNodeID: view.LinkedNodeID(),
|
|
LinkedVMID: view.LinkedVMID(),
|
|
LinkedContainerID: view.LinkedContainerID(),
|
|
}
|
|
}
|
|
|
|
func vmFromReadStateView(view *unifiedresources.VMView) models.VM {
|
|
if view == nil {
|
|
return models.VM{}
|
|
}
|
|
|
|
totalMemory := view.MemoryTotal()
|
|
usedMemory := view.MemoryUsed()
|
|
totalDisk := view.DiskTotal()
|
|
usedDisk := view.DiskUsed()
|
|
|
|
return models.VM{
|
|
ID: firstNonEmptyString(view.SourceID(), view.ID()),
|
|
VMID: view.VMID(),
|
|
Name: view.Name(),
|
|
Node: view.Node(),
|
|
Instance: view.Instance(),
|
|
Status: string(view.Status()),
|
|
Type: "qemu",
|
|
CPU: proxmoxRatioFromPercent(view.CPUPercent()),
|
|
CPUs: view.CPUs(),
|
|
Memory: models.Memory{Total: totalMemory, Used: usedMemory, Free: maxInt64(0, totalMemory-usedMemory), Usage: view.MemoryPercent()},
|
|
Disk: models.Disk{Used: usedDisk, Total: totalDisk, Free: maxInt64(0, totalDisk-usedDisk), Usage: view.DiskPercent()},
|
|
Disks: guestDisksFromReadStateView(view.Disks()),
|
|
DiskStatusReason: view.DiskStatusReason(),
|
|
IPAddresses: view.IPAddresses(),
|
|
OSName: view.OSName(),
|
|
OSVersion: view.OSVersion(),
|
|
AgentVersion: view.AgentVersion(),
|
|
NetworkInterfaces: guestNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
NetworkIn: maxInt64(0, int64(view.NetIn())),
|
|
NetworkOut: maxInt64(0, int64(view.NetOut())),
|
|
DiskRead: maxInt64(0, int64(view.DiskRead())),
|
|
DiskWrite: maxInt64(0, int64(view.DiskWrite())),
|
|
Uptime: view.Uptime(),
|
|
Template: view.Template(),
|
|
LastBackup: view.LastBackup(),
|
|
BackupInProgress: view.BackupInProgress(),
|
|
Tags: view.Tags(),
|
|
Lock: view.Lock(),
|
|
LastSeen: view.LastSeen(),
|
|
}
|
|
}
|
|
|
|
func containerFromReadStateView(view *unifiedresources.ContainerView) models.Container {
|
|
if view == nil {
|
|
return models.Container{}
|
|
}
|
|
|
|
totalMemory := view.MemoryTotal()
|
|
usedMemory := view.MemoryUsed()
|
|
totalDisk := view.DiskTotal()
|
|
usedDisk := view.DiskUsed()
|
|
|
|
return models.Container{
|
|
ID: firstNonEmptyString(view.SourceID(), view.ID()),
|
|
VMID: view.VMID(),
|
|
Name: view.Name(),
|
|
Node: view.Node(),
|
|
Instance: view.Instance(),
|
|
Status: string(view.Status()),
|
|
Type: firstNonEmptyString(view.ContainerType(), "lxc"),
|
|
CPU: proxmoxRatioFromPercent(view.CPUPercent()),
|
|
CPUs: view.CPUs(),
|
|
Memory: models.Memory{Total: totalMemory, Used: usedMemory, Free: maxInt64(0, totalMemory-usedMemory), Usage: view.MemoryPercent()},
|
|
Disk: models.Disk{Used: usedDisk, Total: totalDisk, Free: maxInt64(0, totalDisk-usedDisk), Usage: view.DiskPercent()},
|
|
Disks: guestDisksFromReadStateView(view.Disks()),
|
|
NetworkIn: maxInt64(0, int64(view.NetIn())),
|
|
NetworkOut: maxInt64(0, int64(view.NetOut())),
|
|
DiskRead: maxInt64(0, int64(view.DiskRead())),
|
|
DiskWrite: maxInt64(0, int64(view.DiskWrite())),
|
|
Uptime: view.Uptime(),
|
|
Template: view.Template(),
|
|
LastBackup: view.LastBackup(),
|
|
BackupInProgress: view.BackupInProgress(),
|
|
Tags: view.Tags(),
|
|
Lock: view.Lock(),
|
|
LastSeen: view.LastSeen(),
|
|
IPAddresses: view.IPAddresses(),
|
|
NetworkInterfaces: guestNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
OSName: view.OSName(),
|
|
IsOCI: view.IsOCI(),
|
|
OSTemplate: view.OSTemplate(),
|
|
HasDocker: view.HasDocker(),
|
|
DockerCheckedAt: view.DockerCheckedAt(),
|
|
}
|
|
}
|
|
|
|
func guestDisksFromReadStateView(disks []unifiedresources.DiskInfo) []models.Disk {
|
|
if len(disks) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.Disk, 0, len(disks))
|
|
for _, disk := range disks {
|
|
out = append(out, models.Disk{
|
|
Total: disk.Total,
|
|
Used: disk.Used,
|
|
Free: disk.Free,
|
|
Usage: disk.Usage,
|
|
Mountpoint: disk.Mountpoint,
|
|
Type: disk.Filesystem,
|
|
Device: disk.Device,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func guestNetworkInterfacesFromReadStateView(interfaces []unifiedresources.NetworkInterface) []models.GuestNetworkInterface {
|
|
if len(interfaces) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.GuestNetworkInterface, 0, len(interfaces))
|
|
for _, iface := range interfaces {
|
|
out = append(out, models.GuestNetworkInterface{
|
|
Name: iface.Name,
|
|
MAC: iface.MAC,
|
|
Addresses: append([]string(nil), iface.Addresses...),
|
|
RXBytes: maxInt64(0, int64(iface.RXBytes)),
|
|
TXBytes: maxInt64(0, int64(iface.TXBytes)),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func pbsInstanceFromReadStateView(view *unifiedresources.PBSInstanceView) models.PBSInstance {
|
|
if view == nil {
|
|
return models.PBSInstance{}
|
|
}
|
|
|
|
return models.PBSInstance{
|
|
ID: firstNonEmptyString(view.InstanceID(), view.ID()),
|
|
Name: view.Name(),
|
|
Host: view.HostURL(),
|
|
GuestURL: view.GuestURL(),
|
|
Status: string(view.Status()),
|
|
Version: view.Version(),
|
|
CPU: view.CPUPercent(),
|
|
Memory: view.MemoryPercent(),
|
|
MemoryUsed: view.MemoryUsed(),
|
|
MemoryTotal: view.MemoryTotal(),
|
|
Uptime: view.UptimeSeconds(),
|
|
Datastores: view.DatastoreDetails(),
|
|
BackupJobs: view.BackupJobs(),
|
|
SyncJobs: view.SyncJobs(),
|
|
VerifyJobs: view.VerifyJobs(),
|
|
PruneJobs: view.PruneJobs(),
|
|
GarbageJobs: view.GarbageJobs(),
|
|
ConnectionHealth: view.ConnectionHealth(),
|
|
LastSeen: view.LastSeen(),
|
|
}
|
|
}
|
|
|
|
func hostMemoryFromReadStateView(view *unifiedresources.HostView) models.Memory {
|
|
if view == nil {
|
|
return models.Memory{}
|
|
}
|
|
total := view.MemoryTotal()
|
|
used := view.MemoryUsed()
|
|
return models.Memory{
|
|
Total: total,
|
|
Used: used,
|
|
Free: maxInt64(0, total-used),
|
|
Usage: view.MemoryPercent(),
|
|
SwapUsed: view.SwapUsed(),
|
|
SwapTotal: view.SwapTotal(),
|
|
}
|
|
}
|
|
|
|
func hostDisksFromReadStateView(disks []unifiedresources.DiskInfo) []models.Disk {
|
|
if len(disks) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.Disk, 0, len(disks))
|
|
for _, disk := range disks {
|
|
out = append(out, models.Disk{
|
|
Total: disk.Total,
|
|
Used: disk.Used,
|
|
Free: disk.Free,
|
|
Usage: disk.Usage,
|
|
Mountpoint: disk.Mountpoint,
|
|
Type: disk.Filesystem,
|
|
Device: disk.Device,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostDiskIOFromReadStateView(diskIO []unifiedresources.HostDiskIOMeta) []models.DiskIO {
|
|
if len(diskIO) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.DiskIO, 0, len(diskIO))
|
|
for _, entry := range diskIO {
|
|
out = append(out, models.DiskIO{
|
|
Device: entry.Device,
|
|
ReadBytes: entry.ReadBytes,
|
|
WriteBytes: entry.WriteBytes,
|
|
ReadOps: entry.ReadOps,
|
|
WriteOps: entry.WriteOps,
|
|
IOTime: entry.IOTimeMs,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostNetworkInterfacesFromReadStateView(interfaces []unifiedresources.NetworkInterface) []models.HostNetworkInterface {
|
|
if len(interfaces) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.HostNetworkInterface, 0, len(interfaces))
|
|
for _, iface := range interfaces {
|
|
out = append(out, models.HostNetworkInterface{
|
|
Name: iface.Name,
|
|
MAC: iface.MAC,
|
|
Addresses: append([]string(nil), iface.Addresses...),
|
|
RXBytes: iface.RXBytes,
|
|
TXBytes: iface.TXBytes,
|
|
SpeedMbps: int64PtrCopy(iface.SpeedMbps),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostSensorsFromReadStateView(sensors *unifiedresources.HostSensorMeta) models.HostSensorSummary {
|
|
if sensors == nil {
|
|
return models.HostSensorSummary{}
|
|
}
|
|
out := models.HostSensorSummary{}
|
|
if len(sensors.TemperatureCelsius) > 0 {
|
|
out.TemperatureCelsius = make(map[string]float64, len(sensors.TemperatureCelsius))
|
|
for k, v := range sensors.TemperatureCelsius {
|
|
out.TemperatureCelsius[k] = v
|
|
}
|
|
}
|
|
if len(sensors.FanRPM) > 0 {
|
|
out.FanRPM = make(map[string]float64, len(sensors.FanRPM))
|
|
for k, v := range sensors.FanRPM {
|
|
out.FanRPM[k] = v
|
|
}
|
|
}
|
|
if len(sensors.PowerWatts) > 0 {
|
|
out.PowerWatts = make(map[string]float64, len(sensors.PowerWatts))
|
|
for k, v := range sensors.PowerWatts {
|
|
out.PowerWatts[k] = v
|
|
}
|
|
}
|
|
if len(sensors.Additional) > 0 {
|
|
out.Additional = make(map[string]float64, len(sensors.Additional))
|
|
for k, v := range sensors.Additional {
|
|
out.Additional[k] = v
|
|
}
|
|
}
|
|
if len(sensors.Custom) > 0 {
|
|
out.Custom = make([]models.HostCustomSensorMetric, len(sensors.Custom))
|
|
for i, metric := range sensors.Custom {
|
|
out.Custom[i] = models.HostCustomSensorMetric{
|
|
ID: metric.ID,
|
|
Name: metric.Name,
|
|
Group: metric.Group,
|
|
Subgroup: metric.Subgroup,
|
|
Kind: metric.Kind,
|
|
Unit: metric.Unit,
|
|
Value: cloneFloat64Ptr(metric.Value),
|
|
Status: metric.Status,
|
|
ObservedAt: metric.ObservedAt,
|
|
EventAt: cloneReportTimePtr(metric.EventAt),
|
|
Error: metric.Error,
|
|
AlertOnError: metric.AlertOnError,
|
|
Stale: metric.Stale,
|
|
}
|
|
}
|
|
}
|
|
if len(sensors.GPU) > 0 {
|
|
out.GPU = make([]models.HostGPUSensor, len(sensors.GPU))
|
|
for i, gpu := range sensors.GPU {
|
|
out.GPU[i] = models.HostGPUSensor{
|
|
ID: gpu.ID,
|
|
Name: gpu.Name,
|
|
TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius),
|
|
UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent),
|
|
MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes),
|
|
MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes),
|
|
}
|
|
}
|
|
}
|
|
if sensors.ThermalState != nil {
|
|
out.ThermalState = hostThermalStateFromReadStateView(sensors.ThermalState)
|
|
}
|
|
if len(sensors.SMART) > 0 {
|
|
out.SMART = make([]models.HostDiskSMART, 0, len(sensors.SMART))
|
|
for _, smart := range sensors.SMART {
|
|
out.SMART = append(out.SMART, models.HostDiskSMART{
|
|
Device: smart.Device,
|
|
Model: smart.Model,
|
|
Serial: smart.Serial,
|
|
WWN: smart.WWN,
|
|
Type: smart.Type,
|
|
Controller: smart.Controller,
|
|
Target: smart.Target,
|
|
SizeBytes: smart.SizeBytes,
|
|
Temperature: smart.Temperature,
|
|
Health: smart.Health,
|
|
Standby: smart.Standby,
|
|
Pool: smart.Pool,
|
|
IO: physicalDiskIOFromUnifiedMeta(smart.IO),
|
|
Collection: diskinventory.CloneStatus(smart.Collection),
|
|
Attributes: smartAttributesCopy(smart.Attributes),
|
|
})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostThermalStateFromReadStateView(in *unifiedresources.HostThermalState) *models.HostThermalState {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
return &models.HostThermalState{
|
|
Source: in.Source,
|
|
Pressure: in.Pressure,
|
|
ThermalWarningLevel: cloneIntPtr(in.ThermalWarningLevel),
|
|
PerformanceWarningLevel: cloneIntPtr(in.PerformanceWarningLevel),
|
|
CPUPowerStatus: cloneIntPtr(in.CPUPowerStatus),
|
|
LimitsPercent: cloneStringIntMap(in.LimitsPercent),
|
|
}
|
|
}
|
|
|
|
func hostRAIDFromReadStateView(raid []unifiedresources.HostRAIDMeta) []models.HostRAIDArray {
|
|
if len(raid) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.HostRAIDArray, 0, len(raid))
|
|
for _, entry := range raid {
|
|
devices := make([]models.HostRAIDDevice, 0, len(entry.Devices))
|
|
for _, device := range entry.Devices {
|
|
devices = append(devices, models.HostRAIDDevice{
|
|
Device: device.Device,
|
|
State: device.State,
|
|
Slot: device.Slot,
|
|
})
|
|
}
|
|
out = append(out, models.HostRAIDArray{
|
|
Device: entry.Device,
|
|
Name: entry.Name,
|
|
Level: entry.Level,
|
|
State: entry.State,
|
|
TotalDevices: entry.TotalDevices,
|
|
ActiveDevices: entry.ActiveDevices,
|
|
WorkingDevices: entry.WorkingDevices,
|
|
FailedDevices: entry.FailedDevices,
|
|
SpareDevices: entry.SpareDevices,
|
|
UUID: entry.UUID,
|
|
Devices: devices,
|
|
RebuildPercent: entry.RebuildPercent,
|
|
RebuildSpeed: entry.RebuildSpeed,
|
|
Operation: entry.Operation,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostUnraidFromReadStateView(unraid *unifiedresources.HostUnraidMeta) *models.HostUnraidStorage {
|
|
if unraid == nil {
|
|
return nil
|
|
}
|
|
out := &models.HostUnraidStorage{
|
|
ArrayStarted: unraid.ArrayStarted,
|
|
ArrayState: unraid.ArrayState,
|
|
SyncAction: unraid.SyncAction,
|
|
SyncProgress: unraid.SyncProgress,
|
|
SyncErrors: unraid.SyncErrors,
|
|
NumDisks: unraid.NumDisks,
|
|
NumProtected: unraid.NumProtected,
|
|
NumDisabled: unraid.NumDisabled,
|
|
NumInvalid: unraid.NumInvalid,
|
|
NumMissing: unraid.NumMissing,
|
|
}
|
|
if len(unraid.Disks) > 0 {
|
|
out.Disks = make([]models.HostUnraidDisk, 0, len(unraid.Disks))
|
|
for _, disk := range unraid.Disks {
|
|
out.Disks = append(out.Disks, models.HostUnraidDisk{
|
|
Name: disk.Name,
|
|
Device: disk.Device,
|
|
Role: disk.Role,
|
|
Status: disk.Status,
|
|
RawStatus: disk.RawStatus,
|
|
Model: disk.Model,
|
|
Serial: disk.Serial,
|
|
Filesystem: disk.Filesystem,
|
|
Transport: disk.Transport,
|
|
SizeBytes: disk.SizeBytes,
|
|
UsedBytes: disk.UsedBytes,
|
|
FreeBytes: disk.FreeBytes,
|
|
Temperature: disk.Temperature,
|
|
SpunDown: disk.SpunDown,
|
|
ReadCount: disk.ReadCount,
|
|
WriteCount: disk.WriteCount,
|
|
ErrorCount: disk.ErrorCount,
|
|
Slot: disk.Slot,
|
|
})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostCephFromReadStateView(ceph *unifiedresources.HostCephMeta) *models.HostCephCluster {
|
|
if ceph == nil {
|
|
return nil
|
|
}
|
|
out := &models.HostCephCluster{
|
|
FSID: ceph.FSID,
|
|
Health: models.HostCephHealth{
|
|
Status: ceph.Health.Status,
|
|
},
|
|
MonMap: models.HostCephMonitorMap{
|
|
Epoch: ceph.MonMap.Epoch,
|
|
NumMons: ceph.MonMap.NumMons,
|
|
},
|
|
MgrMap: models.HostCephManagerMap{
|
|
Available: ceph.MgrMap.Available,
|
|
NumMgrs: ceph.MgrMap.NumMgrs,
|
|
ActiveMgr: ceph.MgrMap.ActiveMgr,
|
|
Standbys: ceph.MgrMap.Standbys,
|
|
},
|
|
OSDMap: models.HostCephOSDMap{
|
|
Epoch: ceph.OSDMap.Epoch,
|
|
NumOSDs: ceph.OSDMap.NumOSDs,
|
|
NumUp: ceph.OSDMap.NumUp,
|
|
NumIn: ceph.OSDMap.NumIn,
|
|
NumDown: ceph.OSDMap.NumDown,
|
|
NumOut: ceph.OSDMap.NumOut,
|
|
},
|
|
PGMap: models.HostCephPGMap{
|
|
NumPGs: ceph.PGMap.NumPGs,
|
|
BytesTotal: ceph.PGMap.BytesTotal,
|
|
BytesUsed: ceph.PGMap.BytesUsed,
|
|
BytesAvailable: ceph.PGMap.BytesAvailable,
|
|
DataBytes: ceph.PGMap.DataBytes,
|
|
UsagePercent: ceph.PGMap.UsagePercent,
|
|
DegradedRatio: ceph.PGMap.DegradedRatio,
|
|
MisplacedRatio: ceph.PGMap.MisplacedRatio,
|
|
ReadBytesPerSec: ceph.PGMap.ReadBytesPerSec,
|
|
WriteBytesPerSec: ceph.PGMap.WriteBytesPerSec,
|
|
ReadOpsPerSec: ceph.PGMap.ReadOpsPerSec,
|
|
WriteOpsPerSec: ceph.PGMap.WriteOpsPerSec,
|
|
},
|
|
CollectedAt: ceph.CollectedAt,
|
|
}
|
|
if len(ceph.Health.Summary) > 0 {
|
|
out.Health.Summary = make([]models.HostCephHealthSummary, 0, len(ceph.Health.Summary))
|
|
for _, summary := range ceph.Health.Summary {
|
|
out.Health.Summary = append(out.Health.Summary, models.HostCephHealthSummary{
|
|
Severity: summary.Severity,
|
|
Message: summary.Message,
|
|
})
|
|
}
|
|
}
|
|
if len(ceph.Health.Checks) > 0 {
|
|
out.Health.Checks = make(map[string]models.HostCephCheck, len(ceph.Health.Checks))
|
|
for name, check := range ceph.Health.Checks {
|
|
out.Health.Checks[name] = models.HostCephCheck{
|
|
Severity: check.Severity,
|
|
Message: check.Message,
|
|
Detail: append([]string(nil), check.Detail...),
|
|
}
|
|
}
|
|
}
|
|
if len(ceph.MonMap.Monitors) > 0 {
|
|
out.MonMap.Monitors = make([]models.HostCephMonitor, 0, len(ceph.MonMap.Monitors))
|
|
for _, monitor := range ceph.MonMap.Monitors {
|
|
out.MonMap.Monitors = append(out.MonMap.Monitors, models.HostCephMonitor{
|
|
Name: monitor.Name,
|
|
Rank: monitor.Rank,
|
|
Addr: monitor.Addr,
|
|
Status: monitor.Status,
|
|
})
|
|
}
|
|
}
|
|
if len(ceph.Pools) > 0 {
|
|
out.Pools = make([]models.HostCephPool, 0, len(ceph.Pools))
|
|
for _, pool := range ceph.Pools {
|
|
out.Pools = append(out.Pools, models.HostCephPool{
|
|
ID: pool.ID,
|
|
Name: pool.Name,
|
|
BytesUsed: pool.BytesUsed,
|
|
BytesAvailable: pool.BytesAvailable,
|
|
Objects: pool.Objects,
|
|
PercentUsed: pool.PercentUsed,
|
|
})
|
|
}
|
|
}
|
|
if len(ceph.Services) > 0 {
|
|
out.Services = make([]models.HostCephService, 0, len(ceph.Services))
|
|
for _, service := range ceph.Services {
|
|
out.Services = append(out.Services, models.HostCephService{
|
|
Type: service.Type,
|
|
Running: service.Running,
|
|
Total: service.Total,
|
|
Daemons: append([]string(nil), service.Daemons...),
|
|
})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func dockerHostFromReadStateView(view *unifiedresources.DockerHostView) models.DockerHost {
|
|
if view == nil {
|
|
return models.DockerHost{}
|
|
}
|
|
|
|
totalMemory := view.TotalMemoryBytes()
|
|
if totalMemory == 0 {
|
|
totalMemory = view.MemoryTotal()
|
|
}
|
|
usedMemory := view.MemoryUsed()
|
|
freeMemory := maxInt64(0, totalMemory-usedMemory)
|
|
|
|
return models.DockerHost{
|
|
ID: firstNonEmptyString(view.HostSourceID(), view.ID()),
|
|
AgentID: view.AgentID(),
|
|
Hostname: view.Hostname(),
|
|
DisplayName: view.DisplayName(),
|
|
CustomDisplayName: view.CustomDisplayName(),
|
|
MachineID: view.MachineID(),
|
|
OS: view.OS(),
|
|
KernelVersion: view.KernelVersion(),
|
|
Architecture: view.Architecture(),
|
|
Runtime: view.Runtime(),
|
|
RuntimeVersion: view.RuntimeVersion(),
|
|
DockerVersion: view.DockerVersion(),
|
|
CollectionMode: view.CollectionMode(),
|
|
CPUs: view.CPUs(),
|
|
TotalMemoryBytes: totalMemory,
|
|
UptimeSeconds: view.UptimeSeconds(),
|
|
CPUUsage: view.CPUPercent(),
|
|
LoadAverage: view.LoadAverage(),
|
|
Memory: models.Memory{
|
|
Total: totalMemory,
|
|
Used: usedMemory,
|
|
Free: freeMemory,
|
|
Usage: view.MemoryPercent(),
|
|
},
|
|
Disks: hostDisksFromReadStateView(view.Disks()),
|
|
NetworkInterfaces: hostNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
Status: string(view.Status()),
|
|
LastSeen: view.LastSeen(),
|
|
IntervalSeconds: view.IntervalSeconds(),
|
|
AgentVersion: view.AgentVersion(),
|
|
Containers: view.Containers(),
|
|
Services: view.Services(),
|
|
Tasks: view.Tasks(),
|
|
Swarm: dockerSwarmFromReadStateView(view.Swarm()),
|
|
TokenID: view.TokenID(),
|
|
TokenName: view.TokenName(),
|
|
TokenHint: view.TokenHint(),
|
|
TokenLastUsedAt: view.TokenLastUsedAt(),
|
|
Hidden: view.Hidden(),
|
|
PendingUninstall: view.PendingUninstall(),
|
|
Command: view.Command(),
|
|
IsLegacy: view.IsLegacy(),
|
|
IdentityConflict: view.IdentityConflict(),
|
|
NetInRate: view.NetInRate(),
|
|
NetOutRate: view.NetOutRate(),
|
|
DiskReadRate: view.DiskReadRate(),
|
|
DiskWriteRate: view.DiskWriteRate(),
|
|
}
|
|
}
|
|
|
|
func dockerSwarmFromReadStateView(in *unifiedresources.DockerSwarmInfo) *models.DockerSwarmInfo {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
return &models.DockerSwarmInfo{
|
|
NodeID: in.NodeID,
|
|
NodeRole: in.NodeRole,
|
|
LocalState: in.LocalState,
|
|
ControlAvailable: in.ControlAvailable,
|
|
ClusterID: in.ClusterID,
|
|
ClusterName: in.ClusterName,
|
|
Scope: in.Scope,
|
|
Error: in.Error,
|
|
}
|
|
}
|
|
|
|
func maxInt64(a, b int64) int64 {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func int64PtrCopy(in *int64) *int64 {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
out := *in
|
|
return &out
|
|
}
|
|
|
|
func smartAttributesCopy(in *models.SMARTAttributes) *models.SMARTAttributes {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
out := *in
|
|
out.PowerOnHours = int64PtrCopy(in.PowerOnHours)
|
|
out.PowerCycles = int64PtrCopy(in.PowerCycles)
|
|
out.ReallocatedSectors = int64PtrCopy(in.ReallocatedSectors)
|
|
out.PendingSectors = int64PtrCopy(in.PendingSectors)
|
|
out.OfflineUncorrectable = int64PtrCopy(in.OfflineUncorrectable)
|
|
out.UDMACRCErrors = int64PtrCopy(in.UDMACRCErrors)
|
|
if in.PercentageUsed != nil {
|
|
value := *in.PercentageUsed
|
|
out.PercentageUsed = &value
|
|
}
|
|
if in.AvailableSpare != nil {
|
|
value := *in.AvailableSpare
|
|
out.AvailableSpare = &value
|
|
}
|
|
out.MediaErrors = int64PtrCopy(in.MediaErrors)
|
|
out.UnsafeShutdowns = int64PtrCopy(in.UnsafeShutdowns)
|
|
return &out
|
|
}
|
|
|
|
func firstNonEmptyString(values ...string) string {
|
|
for _, value := range values {
|
|
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
|
return trimmed
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func storageZFSPoolFromReadStateView(view *unifiedresources.StoragePoolView) *models.ZFSPool {
|
|
if view == nil {
|
|
return nil
|
|
}
|
|
|
|
state := strings.TrimSpace(view.ZFSPoolState())
|
|
if !view.IsZFS() && state == "" && view.ZFSReadErrors() == 0 && view.ZFSWriteErrors() == 0 && view.ZFSChecksumErrors() == 0 {
|
|
return nil
|
|
}
|
|
|
|
return &models.ZFSPool{
|
|
Name: view.Name(),
|
|
State: state,
|
|
ReadErrors: view.ZFSReadErrors(),
|
|
WriteErrors: view.ZFSWriteErrors(),
|
|
ChecksumErrors: view.ZFSChecksumErrors(),
|
|
}
|
|
}
|
|
|
|
func storageNodeIDsFromReadState(instance string, nodes []string) []string {
|
|
if len(nodes) == 0 {
|
|
return nil
|
|
}
|
|
|
|
nodeIDs := make([]string, 0, len(nodes))
|
|
for _, node := range nodes {
|
|
node = strings.TrimSpace(node)
|
|
if node == "" {
|
|
continue
|
|
}
|
|
if instance == "" {
|
|
nodeIDs = append(nodeIDs, node)
|
|
continue
|
|
}
|
|
nodeIDs = append(nodeIDs, instance+"-"+node)
|
|
}
|
|
if len(nodeIDs) == 0 {
|
|
return nil
|
|
}
|
|
return nodeIDs
|
|
}
|
|
|
|
// ActiveAlertsSnapshot returns the current active alerts.
|
|
func (m *Monitor) ActiveAlertsSnapshot() []models.Alert {
|
|
return m.activeAlertsSnapshot()
|
|
}
|
|
|
|
// RecentlyResolvedSnapshot returns the recently resolved alerts.
|
|
func (m *Monitor) RecentlyResolvedSnapshot() []models.ResolvedAlert {
|
|
return m.recentlyResolvedAlertsSnapshot()
|
|
}
|
|
|
|
// PVEBackupsSnapshot returns the current PVE backups.
|
|
func (m *Monitor) PVEBackupsSnapshot() models.PVEBackups {
|
|
return m.GetState().PVEBackups
|
|
}
|
|
|
|
// PBSBackupsSnapshot returns the current PBS recovery artifacts.
|
|
func (m *Monitor) PBSBackupsSnapshot() []models.PBSBackup {
|
|
return m.GetState().PBSBackups
|
|
}
|
|
|
|
// BuildFrontendState returns the current state converted to frontend format.
|
|
// This replaces the GetState().ToFrontend() pattern in consumer code.
|
|
func (m *Monitor) BuildFrontendState() models.StateFrontend {
|
|
return m.buildBroadcastFrontendStateFromSnapshot(m.GetState())
|
|
}
|
|
|
|
// BuildBroadcastFrontendState returns frontend state ready for websocket
|
|
// broadcasts, including the unified resource payload when a resource store is
|
|
// configured.
|
|
func (m *Monitor) BuildBroadcastFrontendState() models.StateFrontend {
|
|
return m.buildBroadcastFrontendStateFromSnapshot(m.GetState())
|
|
}
|
|
|
|
func buildFrontendStateFromSnapshot(snapshot models.StateSnapshot) models.StateFrontend {
|
|
return snapshot.ToFrontend()
|
|
}
|
|
|
|
func (m *Monitor) buildBroadcastFrontendStateFromSnapshot(snapshot models.StateSnapshot) models.StateFrontend {
|
|
frontendState := buildFrontendStateFromSnapshot(snapshot)
|
|
m.updateResourceStoreForRead(snapshot)
|
|
if m != nil && m.alertManager != nil {
|
|
if liveAlerts := m.activeAlertsSnapshot(); len(liveAlerts) > 0 || len(frontendState.ActiveAlerts) > 0 {
|
|
frontendState.ActiveAlerts = liveAlerts
|
|
}
|
|
}
|
|
unifiedView := m.currentUnifiedStateView()
|
|
metricsTargetResolver := broadcastMetricsTargetResolver(unifiedView.readState)
|
|
broadcastResources := unifiedresources.CoalescePresentationHostResources(unifiedView.resources)
|
|
broadcastResources = m.applyPersistedMetadataToUnifiedResources(broadcastResources)
|
|
broadcastResources = unifiedresources.AttachResourceHealth(
|
|
broadcastResources,
|
|
resourceHealthAlerts(frontendState.ActiveAlerts),
|
|
time.Now().UTC(),
|
|
)
|
|
broadcastFrontendResources, broadcastCatalogs := convertResourcesForBroadcast(broadcastResources, metricsTargetResolver)
|
|
frontendState.Resources = broadcastFrontendResources
|
|
frontendState.CapabilityCatalog = broadcastCatalogs.capabilities
|
|
frontendState.PolicyCatalog = broadcastCatalogs.policies
|
|
frontendState.AISafeSummaryCatalog = broadcastCatalogs.aiSafeSummaries
|
|
frontendState.ConnectedInfrastructure = buildConnectedInfrastructure(broadcastResources, snapshot)
|
|
if !unifiedView.freshness.IsZero() {
|
|
frontendState.LastUpdate = unifiedView.freshness.UnixMilli()
|
|
}
|
|
return frontendState
|
|
}
|
|
|
|
// GetLiveStateSnapshot returns the underlying monitor state snapshot without
|
|
// applying global mock mode overrides.
|
|
//
|
|
// This is useful for agent management endpoints that need to reflect actual
|
|
// registrations even when mock mode is enabled for the UI/demo experience.
|
|
func (m *Monitor) GetLiveStateSnapshot() models.StateSnapshot {
|
|
if m == nil || m.state == nil {
|
|
return models.EmptyStateSnapshot()
|
|
}
|
|
return m.state.GetSnapshot()
|
|
}
|
|
|
|
// GetLiveHostsSnapshot returns the underlying registered host agents without
|
|
// applying global mock mode overrides.
|
|
func (m *Monitor) GetLiveHostsSnapshot() []models.Host {
|
|
if m == nil || m.state == nil {
|
|
return nil
|
|
}
|
|
return m.state.GetSnapshot().Hosts
|
|
}
|
|
|
|
// SetOrgID sets the organization ID for this monitor instance.
|
|
// This is used for tenant isolation in multi-tenant deployments.
|
|
func (m *Monitor) SetOrgID(orgID string) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.orgID = strings.TrimSpace(orgID)
|
|
}
|
|
|
|
// GetOrgID returns the organization ID for this monitor instance.
|
|
// Returns empty string for default/legacy monitors.
|
|
func (m *Monitor) GetOrgID() string {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
return m.orgID
|
|
}
|
|
|
|
type stateBroadcaster interface {
|
|
BroadcastState(state interface{})
|
|
BroadcastStateToTenant(orgID string, state interface{})
|
|
}
|
|
|
|
type currentStateBroadcaster interface {
|
|
BroadcastCurrentState()
|
|
BroadcastCurrentStateToTenant(orgID string)
|
|
}
|
|
|
|
type stateSubscriberCounter interface {
|
|
GetClientCount() int
|
|
GetTenantClientCount(orgID string) int
|
|
}
|
|
|
|
func stateBroadcasterHasSubscribers(hub stateBroadcaster, orgID string) bool {
|
|
if isNilStateBroadcaster(hub) {
|
|
return false
|
|
}
|
|
|
|
counter, ok := hub.(stateSubscriberCounter)
|
|
if !ok {
|
|
return true
|
|
}
|
|
|
|
orgID = strings.TrimSpace(orgID)
|
|
if orgID != "" {
|
|
return counter.GetTenantClientCount(orgID) > 0
|
|
}
|
|
return counter.GetClientCount() > 0
|
|
}
|
|
|
|
func currentStateBroadcasterHasSubscribers(hub currentStateBroadcaster, orgID string) bool {
|
|
if isNilCurrentStateBroadcaster(hub) {
|
|
return false
|
|
}
|
|
|
|
counter, ok := hub.(stateSubscriberCounter)
|
|
if !ok {
|
|
return true
|
|
}
|
|
|
|
orgID = strings.TrimSpace(orgID)
|
|
if orgID != "" {
|
|
return counter.GetTenantClientCount(orgID) > 0
|
|
}
|
|
return counter.GetClientCount() > 0
|
|
}
|
|
|
|
// broadcastState broadcasts state to WebSocket clients.
|
|
// Monitors with an explicit org ID (including "default") are tenant-scoped.
|
|
// Legacy monitors without an org ID broadcast globally.
|
|
func (m *Monitor) broadcastState(hub stateBroadcaster, frontendState interface{}) {
|
|
if isNilStateBroadcaster(hub) {
|
|
return
|
|
}
|
|
|
|
orgID := strings.TrimSpace(m.GetOrgID())
|
|
if !stateBroadcasterHasSubscribers(hub, orgID) {
|
|
return
|
|
}
|
|
if orgID != "" {
|
|
hub.BroadcastStateToTenant(orgID, frontendState)
|
|
} else {
|
|
hub.BroadcastState(frontendState)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) broadcastCurrentState(hub currentStateBroadcaster) {
|
|
if isNilCurrentStateBroadcaster(hub) {
|
|
return
|
|
}
|
|
|
|
orgID := strings.TrimSpace(m.GetOrgID())
|
|
if !currentStateBroadcasterHasSubscribers(hub, orgID) {
|
|
return
|
|
}
|
|
if orgID != "" {
|
|
hub.BroadcastCurrentStateToTenant(orgID)
|
|
} else {
|
|
hub.BroadcastCurrentState()
|
|
}
|
|
}
|
|
|
|
func isNilStateBroadcaster(hub stateBroadcaster) bool {
|
|
if hub == nil {
|
|
return true
|
|
}
|
|
|
|
value := reflect.ValueOf(hub)
|
|
switch value.Kind() {
|
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
|
|
return value.IsNil()
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func isNilCurrentStateBroadcaster(hub currentStateBroadcaster) bool {
|
|
if hub == nil {
|
|
return true
|
|
}
|
|
|
|
value := reflect.ValueOf(hub)
|
|
switch value.Kind() {
|
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
|
|
return value.IsNil()
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) broadcastEscalatedAlert(hub *websocket.Hub, alert *alerts.Alert) {
|
|
if hub == nil || alert == nil {
|
|
return
|
|
}
|
|
|
|
hub.BroadcastAlertToTenant(m.GetOrgID(), alert)
|
|
}
|
|
|
|
// SetMockMode switches between mock data and real infrastructure data at runtime.
|
|
func (m *Monitor) SetMockMode(enable bool) error {
|
|
current := mock.IsMockEnabled()
|
|
if current == enable {
|
|
log.Info().Bool("mockMode", enable).Msg("mock mode already in desired state")
|
|
return nil
|
|
}
|
|
|
|
if enable {
|
|
m.stopMockMetricsSampler()
|
|
if err := mock.SetEnabled(true); err != nil {
|
|
return err
|
|
}
|
|
m.alertManager.ClearActiveAlerts()
|
|
m.mu.Lock()
|
|
m.resetStateLocked()
|
|
m.metricsHistory.Reset()
|
|
m.mu.Unlock()
|
|
m.StopDiscoveryService()
|
|
m.mu.RLock()
|
|
ctx := m.runtimeCtx
|
|
m.mu.RUnlock()
|
|
if ctx != nil {
|
|
m.startMockMetricsSampler(ctx)
|
|
}
|
|
log.Info().Msg("switched monitor to mock mode")
|
|
} else {
|
|
m.stopMockMetricsSampler()
|
|
if err := mock.SetEnabled(false); err != nil {
|
|
return err
|
|
}
|
|
m.alertManager.ClearActiveAlerts()
|
|
m.mu.Lock()
|
|
m.resetStateLocked()
|
|
m.metricsHistory.Reset()
|
|
m.mu.Unlock()
|
|
log.Info().Msg("switched monitor to real data mode")
|
|
}
|
|
|
|
m.mu.RLock()
|
|
ctx := m.runtimeCtx
|
|
hub := m.wsHub
|
|
m.mu.RUnlock()
|
|
|
|
if hub != nil {
|
|
m.broadcastCurrentState(hub)
|
|
}
|
|
|
|
if enable && ctx != nil && keepRealPollingInMockMode() {
|
|
// Keep real metrics flowing while mock mode is enabled.
|
|
go m.poll(ctx, hub)
|
|
}
|
|
|
|
if !enable && ctx != nil {
|
|
// Kick off an immediate poll to repopulate state with live data.
|
|
go m.poll(ctx, hub)
|
|
if hub != nil && m.config.DiscoveryEnabled {
|
|
go m.StartDiscoveryService(ctx, hub, m.config.DiscoverySubnet)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *Monitor) resetStateLocked() {
|
|
m.state = models.NewState()
|
|
m.state.Stats = models.Stats{
|
|
StartTime: m.startTime,
|
|
Version: "2.0.0-go",
|
|
}
|
|
}
|
|
|
|
// GetStartTime returns the monitor start time
|
|
func (m *Monitor) GetStartTime() time.Time {
|
|
return m.startTime
|
|
}
|
|
|
|
// GetDiscoveryService returns the discovery service
|
|
func (m *Monitor) GetDiscoveryService() *discovery.Service {
|
|
return m.discoveryService
|
|
}
|
|
|
|
// StartDiscoveryService starts the discovery service if not already running
|
|
func (m *Monitor) StartDiscoveryService(ctx context.Context, wsHub *websocket.Hub, subnet string) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if m.discoveryService != nil {
|
|
log.Debug().Msg("discovery service already running")
|
|
return
|
|
}
|
|
|
|
if subnet == "" {
|
|
subnet = "auto"
|
|
}
|
|
|
|
cfgProvider := func() config.DiscoveryConfig {
|
|
return m.discoveryConfigSnapshot()
|
|
}
|
|
|
|
m.discoveryService = discovery.NewService(wsHub, 5*time.Minute, subnet, cfgProvider)
|
|
if m.discoveryService != nil {
|
|
m.discoveryService.Start(ctx)
|
|
log.Info().Str("subnet", subnet).Msg("discovery service started")
|
|
} else {
|
|
log.Error().Msg("failed to create discovery service")
|
|
}
|
|
}
|
|
|
|
// StopDiscoveryService stops the discovery service if running
|
|
func (m *Monitor) StopDiscoveryService() {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if m.discoveryService != nil {
|
|
m.discoveryService.Stop()
|
|
m.discoveryService = nil
|
|
log.Info().Msg("discovery service stopped")
|
|
}
|
|
}
|
|
|
|
// EnableTemperatureMonitoring enables temperature data collection
|
|
func (m *Monitor) EnableTemperatureMonitoring() {
|
|
// Temperature collection is always enabled when tempCollector is initialized
|
|
// This method exists for interface compatibility
|
|
log.Info().Msg("temperature monitoring enabled")
|
|
}
|
|
|
|
// DisableTemperatureMonitoring disables temperature data collection
|
|
func (m *Monitor) DisableTemperatureMonitoring() {
|
|
// Temperature collection is always enabled when tempCollector is initialized
|
|
// This method exists for interface compatibility
|
|
log.Info().Msg("temperature monitoring disabled")
|
|
}
|
|
|
|
// SetResourceStore sets the resource store for polling optimization.
|
|
// When set, the monitor will check if it should reduce polling frequency
|
|
// for nodes that have host agents providing data.
|
|
func (m *Monitor) SetResourceStore(store ResourceStoreInterface) {
|
|
if thresholdStore, ok := store.(StaleThresholdResourceStore); ok {
|
|
thresholdStore.SetStaleThresholds(m.resourceStaleThresholds())
|
|
}
|
|
|
|
m.mu.Lock()
|
|
m.resourceStore = store
|
|
incidentStore := m.incidentStore
|
|
m.mu.Unlock()
|
|
m.installOperatorIntentResolver(store)
|
|
log.Info().Msg("resource store set for polling optimization")
|
|
|
|
timelineAttached := false
|
|
if incidentStore != nil {
|
|
if timelineStore, ok := store.(memory.IncidentTimelineStore); ok {
|
|
incidentStore.SetResourceTimelineStore(timelineStore)
|
|
timelineAttached = true
|
|
} else {
|
|
incidentStore.SetResourceTimelineStore(nil)
|
|
}
|
|
}
|
|
if timelineAttached {
|
|
// The canonical boundary: with the durable resource store attached,
|
|
// restart repair can reach resource history as well as the incident
|
|
// fallback cache, and the replay watermark may advance. Run it in the
|
|
// background — router construction and health serving must not wait
|
|
// behind an un-projected backlog.
|
|
m.scheduleAlertProjectionCatchUp()
|
|
}
|
|
|
|
// Immediately backfill the store from current state so ReadState
|
|
// consumers have data as soon as the store is wired.
|
|
// Guard against minimally initialized monitors (e.g., test fixtures
|
|
// with bare &Monitor{}) where m.state may be nil.
|
|
if store != nil && m.state != nil {
|
|
m.updateResourceStore(m.GetState())
|
|
}
|
|
}
|
|
|
|
// SetSupplementalRecordsProvider configures source-native resource providers
|
|
// that ingest alongside the legacy state snapshot path.
|
|
func (m *Monitor) SetSupplementalRecordsProvider(source unifiedresources.DataSource, provider MonitorSupplementalRecordsProvider) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))
|
|
if normalized == "" {
|
|
return
|
|
}
|
|
|
|
m.mu.Lock()
|
|
if m.supplementalProviders == nil {
|
|
m.supplementalProviders = make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider)
|
|
}
|
|
if provider == nil {
|
|
delete(m.supplementalProviders, normalized)
|
|
} else {
|
|
m.supplementalProviders[normalized] = provider
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
m.updateResourceStore(m.GetState())
|
|
}
|
|
|
|
// SetLicenseChecker wires the commercial feature gate used by monitoring-owned
|
|
// Pro behaviour. It mirrors the alert manager's checker so entitlement lookups
|
|
// stay request-free inside the polling loop.
|
|
func (m *Monitor) SetLicenseChecker(checker func(feature string) bool) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.licenseCheckerMu.Lock()
|
|
m.licenseChecker = checker
|
|
m.licenseCheckerMu.Unlock()
|
|
}
|
|
|
|
func (m *Monitor) hasLicensedFeature(feature string) bool {
|
|
if m == nil {
|
|
return false
|
|
}
|
|
m.licenseCheckerMu.RLock()
|
|
checker := m.licenseChecker
|
|
m.licenseCheckerMu.RUnlock()
|
|
if checker == nil {
|
|
return false
|
|
}
|
|
return checker(feature)
|
|
}
|
|
|
|
// SetRecoveryManager wires the recovery store manager for best-effort ingestion of
|
|
// recovery points derived from polled backup/snapshot data.
|
|
func (m *Monitor) SetRecoveryManager(manager *recoverymanager.Manager) {
|
|
m.mu.Lock()
|
|
m.recoveryManager = manager
|
|
m.mu.Unlock()
|
|
|
|
// Try cleanup during wiring so monitors that are already running still get
|
|
// the migration once a recovery manager becomes available.
|
|
go m.purgeStalePVEPBSBackupsBestEffort(context.Background())
|
|
}
|
|
|
|
// GetNotificationManager returns the notification manager
|
|
func (m *Monitor) GetNotificationManager() *notifications.NotificationManager {
|
|
return m.notificationMgr
|
|
}
|
|
|
|
// GetConfigPersistence returns the config persistence manager
|
|
func (m *Monitor) GetConfigPersistence() *config.ConfigPersistence {
|
|
return m.configPersist
|
|
}
|
|
|
|
// GetMetricsStore returns the persistent metrics store
|
|
func (m *Monitor) GetMetricsStore() *metrics.Store {
|
|
return m.metricsStore
|
|
}
|
|
|
|
// GetMetricsHistory returns the in-memory metrics history for trend analysis
|
|
// This is used by the AI context builder to compute trends and predictions
|
|
func (m *Monitor) GetMetricsHistory() *MetricsHistory {
|
|
return m.metricsHistory
|
|
}
|
|
|
|
// GetUnifiedResources returns the current unified resource view for this monitor.
|
|
// Returns nil when no resource store is configured.
|
|
func (m *Monitor) GetUnifiedResources() []unifiedresources.Resource {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
if store == nil {
|
|
return nil
|
|
}
|
|
|
|
return store.GetAll()
|
|
}
|
|
|
|
// MetricsTargetForResource resolves the metrics-store target for a canonical
|
|
// unified resource ID. Resolution goes through the canonical unified view
|
|
// (the same re-ingested registry the UI and /api/state read) because
|
|
// callers hold IDs from that view: the raw resource store's canonical IDs
|
|
// depend on per-boot ingest order for merged-source hosts, so the view and
|
|
// the store can disagree about the same machine's ID. The raw store stays
|
|
// as a fallback. Returns nil when nothing can resolve a target.
|
|
func (m *Monitor) MetricsTargetForResource(resourceID string) *unifiedresources.MetricsTarget {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
|
|
if view := m.GetUnifiedReadStateOrSnapshot(); view != nil {
|
|
if resolver, ok := view.(MetricsTargetResourceStore); ok {
|
|
if target := resolver.MetricsTargetForResource(resourceID); target != nil {
|
|
return target
|
|
}
|
|
}
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
if store == nil {
|
|
return nil
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return resolver.MetricsTargetForResource(resourceID)
|
|
}
|
|
|
|
// resourceChangeTimeline is the slice of the resource store needed to read
|
|
// the recorded state timeline for a canonical resource ID.
|
|
type resourceChangeTimeline interface {
|
|
GetRecentChanges(canonicalID string, since time.Time, limit int) ([]unifiedresources.ResourceChange, error)
|
|
}
|
|
|
|
// RecentResourceChanges returns the recorded change timeline for a canonical
|
|
// unified resource ID since the given time, newest first (store order).
|
|
// Returns nil when no resource store is wired or it does not record changes.
|
|
func (m *Monitor) RecentResourceChanges(resourceID string, since time.Time, limit int) []unifiedresources.ResourceChange {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
if store == nil {
|
|
return nil
|
|
}
|
|
|
|
timeline, ok := store.(resourceChangeTimeline)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
changes, err := timeline.GetRecentChanges(resourceID, since, limit)
|
|
if err != nil {
|
|
log.Warn().Err(err).Str("resourceID", resourceID).Msg("failed to read resource change timeline")
|
|
return nil
|
|
}
|
|
return changes
|
|
}
|
|
|
|
type monitorUnifiedStateView struct {
|
|
resources []unifiedresources.Resource
|
|
readState unifiedresources.ReadState
|
|
freshness time.Time
|
|
}
|
|
|
|
type unifiedResourceReadStateLister interface {
|
|
GetAll() []unifiedresources.Resource
|
|
}
|
|
|
|
func monitorUnifiedStateViewFromSnapshot(snapshot models.StateSnapshot) monitorUnifiedStateView {
|
|
registry := unifiedresources.NewRegistry(nil)
|
|
registry.IngestSnapshot(snapshot)
|
|
adapter := unifiedresources.NewMonitorAdapter(registry)
|
|
return monitorUnifiedStateView{
|
|
resources: registry.List(),
|
|
readState: adapter,
|
|
freshness: snapshot.LastUpdate,
|
|
}
|
|
}
|
|
|
|
func monitorUnifiedStateViewFromResources(resources []unifiedresources.Resource, freshness time.Time) monitorUnifiedStateView {
|
|
registry := unifiedresources.NewRegistry(nil)
|
|
registry.IngestResources(resources)
|
|
adapter := unifiedresources.NewMonitorAdapter(registry)
|
|
return monitorUnifiedStateView{
|
|
resources: registry.List(),
|
|
readState: adapter,
|
|
freshness: freshness,
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) unifiedStateViewWithStandaloneHostContinuity(view monitorUnifiedStateView) monitorUnifiedStateView {
|
|
if m == nil || view.readState == nil {
|
|
return view
|
|
}
|
|
|
|
readState := m.readStateWithStandaloneHostContinuity(view.readState)
|
|
view.readState = readState
|
|
|
|
lister, ok := readState.(unifiedResourceReadStateLister)
|
|
if !ok {
|
|
return view
|
|
}
|
|
|
|
resources := lister.GetAll()
|
|
view.resources = resources
|
|
if view.freshness.IsZero() {
|
|
view.freshness = latestUnifiedResourceLastSeen(resources)
|
|
}
|
|
return view
|
|
}
|
|
|
|
func latestUnifiedResourceLastSeen(resources []unifiedresources.Resource) time.Time {
|
|
var latest time.Time
|
|
for _, resource := range resources {
|
|
if resource.LastSeen.After(latest) {
|
|
latest = resource.LastSeen
|
|
}
|
|
}
|
|
return latest
|
|
}
|
|
|
|
func (m *Monitor) currentUnifiedStateView() monitorUnifiedStateView {
|
|
if m == nil {
|
|
return monitorUnifiedStateView{}
|
|
}
|
|
|
|
if mock.IsMockEnabled() {
|
|
// Read the version before the snapshot so a tick landing in between
|
|
// caches newer data under an older token (harmless rebuild next
|
|
// call) rather than ever serving stale data under a newer one.
|
|
version := mock.FixtureDataVersion()
|
|
m.mockUnifiedViewMu.Lock()
|
|
if m.mockUnifiedViewValid && m.mockUnifiedViewVersion == version {
|
|
view := m.mockUnifiedView
|
|
m.mockUnifiedViewMu.Unlock()
|
|
return view
|
|
}
|
|
m.mockUnifiedViewMu.Unlock()
|
|
|
|
resources, freshness := mock.UnifiedResourceSnapshot()
|
|
if len(resources) > 0 || !freshness.IsZero() {
|
|
// Consumers share this view between ticks, mirroring the
|
|
// sharing semantics the persistent-store ReadState path has
|
|
// always had in real mode: views are read-only.
|
|
view := monitorUnifiedStateViewFromResources(resources, freshness)
|
|
m.mockUnifiedViewMu.Lock()
|
|
m.mockUnifiedView = view
|
|
m.mockUnifiedViewVersion = version
|
|
m.mockUnifiedViewValid = true
|
|
m.mockUnifiedViewMu.Unlock()
|
|
return view
|
|
}
|
|
return monitorUnifiedStateViewFromSnapshot(m.GetState())
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
state := m.state
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromSnapshot(m.GetState()))
|
|
}
|
|
|
|
resources := store.GetAll()
|
|
freshness := unifiedResourceFreshness(store, state)
|
|
|
|
if readState, ok := store.(unifiedresources.ReadState); ok {
|
|
return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateView{
|
|
resources: resources,
|
|
readState: readState,
|
|
freshness: freshness,
|
|
})
|
|
}
|
|
|
|
if len(resources) > 0 || state == nil {
|
|
return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromResources(resources, freshness))
|
|
}
|
|
|
|
return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromSnapshot(m.GetState()))
|
|
}
|
|
|
|
func (m *Monitor) currentUnifiedResourceFreshness() time.Time {
|
|
if m == nil {
|
|
return time.Time{}
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
state := m.state
|
|
m.mu.RUnlock()
|
|
return unifiedResourceFreshness(store, state)
|
|
}
|
|
|
|
func unifiedResourceFreshness(store ResourceStoreInterface, state *models.State) time.Time {
|
|
freshness := time.Time{}
|
|
if freshnessStore, ok := store.(UnifiedResourceFreshnessStore); ok {
|
|
freshness = freshnessStore.UnifiedResourceFreshness()
|
|
}
|
|
if freshness.IsZero() && state != nil {
|
|
freshness = state.GetLastUpdate()
|
|
}
|
|
return freshness
|
|
}
|
|
|
|
// UnifiedResourceSnapshot returns a canonical unified-resource seed plus the
|
|
// associated freshness marker. In mock mode it returns the shared mock
|
|
// unified-resource fixture graph rather than the live resource store.
|
|
func (m *Monitor) UnifiedResourceSnapshot() ([]unifiedresources.Resource, time.Time) {
|
|
view := m.currentUnifiedStateView()
|
|
// REST /api/resources seeds its registry from this snapshot. Apply the
|
|
// same user-metadata hydration (container customUrl) as the websocket
|
|
// broadcast path, so the two payload shapes cannot drift.
|
|
return m.applyPersistedMetadataToUnifiedResources(view.resources), view.freshness
|
|
}
|
|
|
|
// GetUnifiedReadState returns a typed unified read-state provider when the
|
|
// configured resource store supports it.
|
|
func (m *Monitor) GetUnifiedReadState() unifiedresources.ReadState {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
if store == nil {
|
|
return nil
|
|
}
|
|
|
|
readState, ok := store.(unifiedresources.ReadState)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return readState
|
|
}
|
|
|
|
// GetUnifiedReadStateOrSnapshot returns unified read-state when available.
|
|
// If the monitor has not been wired with a resource store yet, it creates an
|
|
// ephemeral snapshot-backed adapter to preserve read access without exposing
|
|
// direct state reads to consumer packages.
|
|
func (m *Monitor) GetUnifiedReadStateOrSnapshot() unifiedresources.ReadState {
|
|
return m.currentUnifiedStateView().readState
|
|
}
|
|
|
|
// shouldSkipNodeMetrics returns true if we should skip detailed metric polling
|
|
// for the given node because a host agent is providing richer data.
|
|
// This helps reduce API load when agents are active.
|
|
func (m *Monitor) shouldSkipNodeMetrics(nodeName string) bool {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
return false
|
|
}
|
|
|
|
should := store.ShouldSkipAPIPolling(nodeName)
|
|
if should {
|
|
log.Debug().
|
|
Str("node", nodeName).
|
|
Msg("Skipping detailed node metrics - host agent provides data")
|
|
}
|
|
return should
|
|
}
|
|
|
|
// readPathRegistryFreshness bounds how stale the unified registry may be
|
|
// when a read path (state API, websocket hydrate) serves it without
|
|
// rebuilding. It only needs to be wide enough to collapse same-cycle rebuild
|
|
// storms (agent report + broadcast + state request); it must stay well under
|
|
// the poll cadence so every poll-cycle broadcast still publishes a fresh
|
|
// registry.
|
|
const readPathRegistryFreshness = 2 * time.Second
|
|
|
|
// readRefreshResourceStore is implemented by stores whose read paths can
|
|
// refresh without queueing behind ingest rebuilds (see MonitorAdapter).
|
|
type readRefreshResourceStore interface {
|
|
TryReplaceRegistryForRead(snapshot models.StateSnapshot, maxAge time.Duration, supply func() map[unifiedresources.DataSource][]unifiedresources.IngestRecord) bool
|
|
}
|
|
|
|
// updateResourceStoreForRead keeps the resource store fresh on behalf of a
|
|
// read path. Ingest boundaries (agent reports) call updateResourceStore and
|
|
// always rebuild; serving state must not redo work an ingest just did, and
|
|
// above all must not queue behind an in-flight rebuild whose change-record
|
|
// and identity-pin persistence can take seconds per transaction on slow
|
|
// volumes (#1665: /api/state stuck for minutes with SQLite on NFS).
|
|
func (m *Monitor) updateResourceStoreForRead(state models.StateSnapshot) {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
return
|
|
}
|
|
readStore, ok := store.(readRefreshResourceStore)
|
|
if !ok {
|
|
m.updateResourceStore(state)
|
|
return
|
|
}
|
|
|
|
if thresholdStore, ok := store.(StaleThresholdResourceStore); ok {
|
|
thresholdStore.SetStaleThresholds(m.resourceStaleThresholds())
|
|
}
|
|
snapshotForStore := state
|
|
if ownedSources := m.providerOwnedSnapshotSources(); len(ownedSources) > 0 {
|
|
snapshotForStore = unifiedresources.SnapshotWithoutSources(state, ownedSources)
|
|
}
|
|
rebuilt := readStore.TryReplaceRegistryForRead(snapshotForStore, readPathRegistryFreshness, m.collectSupplementalRecordsBySource)
|
|
if !rebuilt {
|
|
return
|
|
}
|
|
recordSupplementalResourceChanges(store, m.collectSupplementalChanges())
|
|
m.syncUnifiedAgentMetrics(store)
|
|
m.syncUnifiedVMMetrics(store)
|
|
m.syncUnifiedStorageMetrics(store)
|
|
m.syncUnifiedPhysicalDiskMetrics(store)
|
|
m.syncUnifiedAppContainerMetrics(store)
|
|
m.syncUnifiedResourceAlertsToState(store.GetAll())
|
|
}
|
|
|
|
// updateResourceStore populates the canonical resource store from current
|
|
// monitoring state. Callers use it at accepted-ingest boundaries and before
|
|
// broadcast hydration so every ReadState consumer observes the same snapshot.
|
|
func (m *Monitor) updateResourceStore(state models.StateSnapshot) {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
log.Debug().Msg("[Resources] No resource store configured, skipping population")
|
|
return
|
|
}
|
|
|
|
if thresholdStore, ok := store.(StaleThresholdResourceStore); ok {
|
|
thresholdStore.SetStaleThresholds(m.resourceStaleThresholds())
|
|
}
|
|
|
|
log.Debug().
|
|
Int("nodes", len(state.Nodes)).
|
|
Int("vms", len(state.VMs)).
|
|
Int("containers", len(state.Containers)).
|
|
Int("hosts", len(state.Hosts)).
|
|
Int("dockerHosts", len(state.DockerHosts)).
|
|
Msg("[Resources] Populating resource store from state snapshot")
|
|
|
|
snapshotForStore := state
|
|
ownedSources := m.providerOwnedSnapshotSources()
|
|
if len(ownedSources) > 0 {
|
|
snapshotForStore = unifiedresources.SnapshotWithoutSources(state, ownedSources)
|
|
sourceNames := make([]string, 0, len(ownedSources))
|
|
for _, source := range ownedSources {
|
|
sourceNames = append(sourceNames, string(source))
|
|
}
|
|
log.Debug().
|
|
Strs("sources", sourceNames).
|
|
Msg("[Resources] Suppressing legacy snapshot slices for provider-owned sources")
|
|
}
|
|
|
|
recordsBySource := m.collectSupplementalRecordsBySource()
|
|
supplementalChanges := m.collectSupplementalChanges()
|
|
if atomicStore, ok := store.(AtomicSnapshotResourceStore); ok {
|
|
atomicStore.PopulateSnapshotAndSupplemental(snapshotForStore, recordsBySource)
|
|
recordSupplementalResourceChanges(store, supplementalChanges)
|
|
m.syncUnifiedAgentMetrics(store)
|
|
m.syncUnifiedVMMetrics(store)
|
|
m.syncUnifiedStorageMetrics(store)
|
|
m.syncUnifiedPhysicalDiskMetrics(store)
|
|
m.syncUnifiedAppContainerMetrics(store)
|
|
for source, records := range recordsBySource {
|
|
if len(records) == 0 {
|
|
continue
|
|
}
|
|
log.Debug().
|
|
Str("source", string(source)).
|
|
Int("records", len(records)).
|
|
Msg("[Resources] Atomically ingested supplemental records")
|
|
}
|
|
m.syncUnifiedResourceAlertsToState(store.GetAll())
|
|
return
|
|
}
|
|
|
|
store.PopulateFromSnapshot(snapshotForStore)
|
|
|
|
supplementalStore, ok := store.(SupplementalRecordStore)
|
|
if ok {
|
|
for source, records := range recordsBySource {
|
|
if len(records) == 0 {
|
|
continue
|
|
}
|
|
supplementalStore.PopulateSupplementalRecords(source, records)
|
|
log.Debug().
|
|
Str("source", string(source)).
|
|
Int("records", len(records)).
|
|
Msg("[Resources] Ingested supplemental records")
|
|
}
|
|
}
|
|
|
|
recordSupplementalResourceChanges(store, supplementalChanges)
|
|
m.syncUnifiedAgentMetrics(store)
|
|
m.syncUnifiedVMMetrics(store)
|
|
m.syncUnifiedStorageMetrics(store)
|
|
m.syncUnifiedPhysicalDiskMetrics(store)
|
|
m.syncUnifiedAppContainerMetrics(store)
|
|
m.syncUnifiedResourceAlertsToState(store.GetAll())
|
|
}
|
|
|
|
// refreshUnifiedResourceStoreAfterAgentStateChange makes accepted agent
|
|
// ingest and removal immediately visible to canonical ReadState consumers.
|
|
// WebSocket broadcasts may also rebuild the store for their own hydrate path,
|
|
// but client presence must never be the trigger that publishes agent-backed
|
|
// runtime truth or retires removed inventory.
|
|
func (m *Monitor) refreshUnifiedResourceStoreAfterAgentStateChange() {
|
|
if m == nil || m.state == nil {
|
|
return
|
|
}
|
|
m.updateResourceStore(m.GetState())
|
|
}
|
|
|
|
func recordSupplementalResourceChanges(store ResourceStoreInterface, changes []unifiedresources.ResourceChange) {
|
|
if store == nil || len(changes) == 0 {
|
|
return
|
|
}
|
|
|
|
recorder, ok := store.(canonicalResourceChangeRecorder)
|
|
if !ok || recorder == nil {
|
|
return
|
|
}
|
|
|
|
for _, change := range changes {
|
|
if err := recorder.RecordChange(change); err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resource_id", change.ResourceID).
|
|
Str("change_id", change.ID).
|
|
Str("kind", string(change.Kind)).
|
|
Msg("failed to record supplemental canonical resource change")
|
|
}
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedAgentMetrics(store ResourceStoreInterface) {
|
|
if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
storeWrites := make([]metrics.WriteMetric, 0)
|
|
appendStoreWrite := func(resourceType, resourceID, metricType string, value float64) {
|
|
if m.metricsStore == nil {
|
|
return
|
|
}
|
|
storeWrites = append(storeWrites, metrics.WriteMetric{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: value,
|
|
Timestamp: now,
|
|
Tier: metrics.TierRaw,
|
|
})
|
|
}
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypeAgent || resource.Metrics == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceAgent) ||
|
|
monitorHasSource(resource.Sources, unifiedresources.SourceProxmox) ||
|
|
monitorHasSource(resource.Sources, unifiedresources.SourceDocker) {
|
|
continue
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
if target == nil || target.ResourceType != "agent" || strings.TrimSpace(target.ResourceID) == "" {
|
|
continue
|
|
}
|
|
targetID := strings.TrimSpace(target.ResourceID)
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
metricKey := fmt.Sprintf("agent:%s", targetID)
|
|
|
|
if metric := resource.Metrics.CPU; metric != nil {
|
|
value := metric.Percent
|
|
if value == 0 {
|
|
value = metric.Value
|
|
}
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "cpu", value, now)
|
|
}
|
|
appendStoreWrite("agent", targetID, "cpu", value)
|
|
}
|
|
|
|
if metric := resource.Metrics.Memory; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "memory", value, now)
|
|
}
|
|
appendStoreWrite("agent", targetID, "memory", value)
|
|
}
|
|
|
|
if metric := resource.Metrics.Disk; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "disk", value, now)
|
|
}
|
|
appendStoreWrite("agent", targetID, "disk", value)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetIn; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "netin", metric.Value, now)
|
|
}
|
|
appendStoreWrite("agent", targetID, "netin", metric.Value)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetOut; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "netout", metric.Value, now)
|
|
}
|
|
appendStoreWrite("agent", targetID, "netout", metric.Value)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskRead; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "diskread", metric.Value, now)
|
|
}
|
|
appendStoreWrite("agent", targetID, "diskread", metric.Value)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskWrite; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "diskwrite", metric.Value, now)
|
|
}
|
|
appendStoreWrite("agent", targetID, "diskwrite", metric.Value)
|
|
}
|
|
}
|
|
if len(storeWrites) > 0 {
|
|
m.metricsStore.WriteBatchBounded(storeWrites)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedVMMetrics(store ResourceStoreInterface) {
|
|
if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
storeWrites := make([]metrics.WriteMetric, 0)
|
|
appendStoreWrite := func(resourceType, resourceID, metricType string, value float64) {
|
|
if m.metricsStore == nil {
|
|
return
|
|
}
|
|
storeWrites = append(storeWrites, metrics.WriteMetric{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: value,
|
|
Timestamp: now,
|
|
Tier: metrics.TierRaw,
|
|
})
|
|
}
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypeVM || resource.Metrics == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
|
|
hasNativeVMWriter := false
|
|
for _, source := range resource.Sources {
|
|
if source == unifiedresources.SourceProxmox {
|
|
hasNativeVMWriter = true
|
|
break
|
|
}
|
|
}
|
|
if hasNativeVMWriter {
|
|
continue
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
if target == nil || target.ResourceType != "vm" || strings.TrimSpace(target.ResourceID) == "" {
|
|
continue
|
|
}
|
|
targetID := strings.TrimSpace(target.ResourceID)
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
|
|
if metric := resource.Metrics.CPU; metric != nil {
|
|
value := metric.Percent
|
|
if value == 0 {
|
|
value = metric.Value
|
|
}
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "cpu", value, now)
|
|
}
|
|
appendStoreWrite("vm", targetID, "cpu", value)
|
|
}
|
|
|
|
if metric := resource.Metrics.Memory; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "memory", value, now)
|
|
}
|
|
appendStoreWrite("vm", targetID, "memory", value)
|
|
}
|
|
|
|
if metric := resource.Metrics.Disk; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "disk", value, now)
|
|
}
|
|
appendStoreWrite("vm", targetID, "disk", value)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetIn; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "netin", metric.Value, now)
|
|
}
|
|
appendStoreWrite("vm", targetID, "netin", metric.Value)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetOut; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "netout", metric.Value, now)
|
|
}
|
|
appendStoreWrite("vm", targetID, "netout", metric.Value)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskRead; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "diskread", metric.Value, now)
|
|
}
|
|
appendStoreWrite("vm", targetID, "diskread", metric.Value)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskWrite; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "diskwrite", metric.Value, now)
|
|
}
|
|
appendStoreWrite("vm", targetID, "diskwrite", metric.Value)
|
|
}
|
|
}
|
|
if len(storeWrites) > 0 {
|
|
m.metricsStore.WriteBatchBounded(storeWrites)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedStorageMetrics(store ResourceStoreInterface) {
|
|
if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
storeWrites := make([]metrics.WriteMetric, 0)
|
|
appendStoreWrite := func(resourceType, resourceID, metricType string, value float64, observedAt time.Time) {
|
|
if m.metricsStore == nil {
|
|
return
|
|
}
|
|
storeWrites = append(storeWrites, metrics.WriteMetric{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: value,
|
|
Timestamp: observedAt,
|
|
Tier: metrics.TierRaw,
|
|
})
|
|
}
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypeStorage || resource.Metrics == nil || resource.Metrics.Disk == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
|
|
// Native Proxmox storage already writes to history during the storage poller.
|
|
if resource.Storage != nil && resource.Storage.Platform == "" {
|
|
hasProxmoxSource := false
|
|
for _, source := range resource.Sources {
|
|
if source == unifiedresources.SourceProxmox {
|
|
hasProxmoxSource = true
|
|
break
|
|
}
|
|
}
|
|
if hasProxmoxSource {
|
|
continue
|
|
}
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
var targetID string
|
|
if target != nil && target.ResourceType == "storage" && strings.TrimSpace(target.ResourceID) != "" {
|
|
targetID = strings.TrimSpace(target.ResourceID)
|
|
} else {
|
|
targetID = resource.ID
|
|
}
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
|
|
disk := resource.Metrics.Disk
|
|
observedAt := unifiedMetricObservedAt(resource, disk, now)
|
|
usage := disk.Percent
|
|
used := int64(0)
|
|
total := int64(0)
|
|
free := int64(0)
|
|
if disk.Used != nil {
|
|
used = *disk.Used
|
|
}
|
|
if disk.Total != nil {
|
|
total = *disk.Total
|
|
}
|
|
if total > 0 {
|
|
free = total - used
|
|
if usage == 0 && used > 0 {
|
|
usage = (float64(used) / float64(total)) * 100
|
|
}
|
|
}
|
|
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddStorageMetric(targetID, "usage", usage, observedAt)
|
|
if total > 0 {
|
|
m.metricsHistory.AddStorageMetric(targetID, "used", float64(used), observedAt)
|
|
m.metricsHistory.AddStorageMetric(targetID, "total", float64(total), observedAt)
|
|
m.metricsHistory.AddStorageMetric(targetID, "avail", float64(free), observedAt)
|
|
}
|
|
}
|
|
appendStoreWrite("storage", targetID, "usage", usage, observedAt)
|
|
if total > 0 {
|
|
appendStoreWrite("storage", targetID, "used", float64(used), observedAt)
|
|
appendStoreWrite("storage", targetID, "total", float64(total), observedAt)
|
|
appendStoreWrite("storage", targetID, "avail", float64(free), observedAt)
|
|
}
|
|
}
|
|
if len(storeWrites) > 0 {
|
|
m.metricsStore.WriteBatchBounded(storeWrites)
|
|
}
|
|
}
|
|
|
|
// unifiedMetricObservedAt returns the source observation time represented by a
|
|
// canonical metric. Registry rebuilds can happen several times during one
|
|
// polling cycle; using rebuild time here would turn one upstream observation
|
|
// into several distinct history samples.
|
|
func unifiedMetricObservedAt(resource unifiedresources.Resource, metric *unifiedresources.MetricValue, fallback time.Time) time.Time {
|
|
if metric != nil && metric.Source != "" {
|
|
if status, ok := resource.SourceStatus[metric.Source]; ok && !status.LastSeen.IsZero() {
|
|
return status.LastSeen
|
|
}
|
|
}
|
|
if !resource.LastSeen.IsZero() {
|
|
return resource.LastSeen
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedPhysicalDiskMetrics(store ResourceStoreInterface) {
|
|
if store == nil || m.metricsStore == nil {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypePhysicalDisk || resource.PhysicalDisk == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
|
|
hasNativeWriter := false
|
|
for _, source := range resource.Sources {
|
|
if source == unifiedresources.SourceProxmox || source == unifiedresources.SourceAgent {
|
|
hasNativeWriter = true
|
|
break
|
|
}
|
|
}
|
|
if hasNativeWriter {
|
|
continue
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
if target == nil || target.ResourceType != "disk" || strings.TrimSpace(target.ResourceID) == "" {
|
|
continue
|
|
}
|
|
targetID := strings.TrimSpace(target.ResourceID)
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
|
|
nodeName := ""
|
|
for _, hostname := range resource.Identity.Hostnames {
|
|
if hostname = strings.TrimSpace(hostname); hostname != "" {
|
|
nodeName = hostname
|
|
break
|
|
}
|
|
}
|
|
if nodeName == "" {
|
|
nodeName = firstNonEmptyString(strings.TrimSpace(resource.ParentName), strings.TrimSpace(resource.Name))
|
|
}
|
|
|
|
disk := models.PhysicalDisk{
|
|
ID: resource.ID,
|
|
Node: nodeName,
|
|
DevPath: resource.PhysicalDisk.DevPath,
|
|
Model: resource.PhysicalDisk.Model,
|
|
Serial: resource.PhysicalDisk.Serial,
|
|
WWN: resource.PhysicalDisk.WWN,
|
|
Type: resource.PhysicalDisk.DiskType,
|
|
Size: resource.PhysicalDisk.SizeBytes,
|
|
Health: resource.PhysicalDisk.Health,
|
|
Wearout: resource.PhysicalDisk.Wearout,
|
|
Temperature: resource.PhysicalDisk.Temperature,
|
|
RPM: resource.PhysicalDisk.RPM,
|
|
Used: resource.PhysicalDisk.Used,
|
|
SmartAttributes: smartAttributesFromUnifiedMeta(resource.PhysicalDisk.SMART),
|
|
LastChecked: resource.LastSeen,
|
|
}
|
|
if disk.Serial == "" {
|
|
disk.ID = targetID
|
|
}
|
|
m.writeSMARTMetrics(disk, now)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedAppContainerMetrics(store ResourceStoreInterface) {
|
|
if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
storeWrites := make([]metrics.WriteMetric, 0)
|
|
appendStoreWrite := func(resourceType, resourceID, metricType string, value float64) {
|
|
if m.metricsStore == nil {
|
|
return
|
|
}
|
|
storeWrites = append(storeWrites, metrics.WriteMetric{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: value,
|
|
Timestamp: now,
|
|
Tier: metrics.TierRaw,
|
|
})
|
|
}
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypeAppContainer || resource.Metrics == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
hasDockerSource := false
|
|
for _, source := range resource.Sources {
|
|
if source == unifiedresources.SourceDocker {
|
|
hasDockerSource = true
|
|
break
|
|
}
|
|
}
|
|
if hasDockerSource {
|
|
continue
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
if target == nil || target.ResourceType != "app-container" || strings.TrimSpace(target.ResourceID) == "" {
|
|
continue
|
|
}
|
|
targetID := strings.TrimSpace(target.ResourceID)
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
metricKey := fmt.Sprintf("docker:%s", targetID)
|
|
|
|
if metric := resource.Metrics.CPU; metric != nil {
|
|
value := metric.Percent
|
|
if value == 0 {
|
|
value = metric.Value
|
|
}
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "cpu", value, now)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "cpu", value)
|
|
}
|
|
|
|
if metric := resource.Metrics.Memory; metric != nil && (metric.Total != nil || metric.Percent > 0) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "memory", value, now)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "memory", value)
|
|
}
|
|
|
|
if metric := resource.Metrics.Disk; metric != nil && (metric.Total != nil || metric.Percent > 0) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "disk", value, now)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "disk", value)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetIn; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "netin", metric.Value, now)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "netin", metric.Value)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetOut; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "netout", metric.Value, now)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "netout", metric.Value)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskRead; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "diskread", metric.Value, now)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "diskread", metric.Value)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskWrite; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "diskwrite", metric.Value, now)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "diskwrite", metric.Value)
|
|
}
|
|
}
|
|
if len(storeWrites) > 0 {
|
|
m.metricsStore.WriteBatchBounded(storeWrites)
|
|
}
|
|
}
|
|
|
|
func shouldSkipMockOwnedUnifiedMetricSync(resource unifiedresources.Resource) bool {
|
|
if !mock.IsMockEnabled() {
|
|
return false
|
|
}
|
|
|
|
// In mock mode the canonical mock sampler owns chart/history continuity for
|
|
// the entire demo estate. Unified-resource sync must not append a second
|
|
// live timeline on top of seeded mock history for any resource class.
|
|
_ = resource
|
|
return true
|
|
}
|
|
|
|
func shouldSkipNativeMockStateMetricWrites() bool {
|
|
return mock.IsMockEnabled()
|
|
}
|
|
|
|
// getUnifiedResourcesForBroadcast retrieves all resources from the store.
|
|
// Returns nil if no resource store is configured.
|
|
func (m *Monitor) getUnifiedResourcesForBroadcast() []unifiedresources.Resource {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
log.Debug().Msg("[Resources] No store for broadcast")
|
|
return nil
|
|
}
|
|
|
|
allResources := store.GetAll()
|
|
log.Debug().Int("count", len(allResources)).Msg("[Resources] Got resources for broadcast")
|
|
if len(allResources) == 0 {
|
|
return nil
|
|
}
|
|
return allResources
|
|
}
|
|
|
|
// getResourcesForBroadcast retrieves all resources from the store and converts
|
|
// them to frontend format.
|
|
func (m *Monitor) getResourcesForBroadcast() []models.ResourceFrontend {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
resourcesForBroadcast := m.applyPersistedMetadataToUnifiedResources(m.getUnifiedResourcesForBroadcast())
|
|
resourcesForBroadcast = unifiedresources.AttachResourceHealth(
|
|
resourcesForBroadcast,
|
|
resourceHealthAlerts(m.activeAlertsSnapshot()),
|
|
time.Now().UTC(),
|
|
)
|
|
resources, _ := convertResourcesForBroadcast(
|
|
resourcesForBroadcast,
|
|
broadcastMetricsTargetResolver(store),
|
|
)
|
|
return resources
|
|
}
|
|
|
|
func resourceHealthAlerts(active []models.Alert) []unifiedresources.ResourceHealthAlert {
|
|
if len(active) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]unifiedresources.ResourceHealthAlert, 0, len(active))
|
|
for _, alert := range active {
|
|
out = append(out, unifiedresources.ResourceHealthAlert{
|
|
ResourceID: alert.ResourceID,
|
|
Level: alert.Level,
|
|
Type: alert.Type,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (m *Monitor) applyPersistedMetadataToUnifiedResources(resources []unifiedresources.Resource) []unifiedresources.Resource {
|
|
if len(resources) == 0 || m == nil {
|
|
return resources
|
|
}
|
|
|
|
out := make([]unifiedresources.Resource, len(resources))
|
|
copy(out, resources)
|
|
for i := range out {
|
|
resource := &out[i]
|
|
|
|
switch unifiedresources.ContractResourceType(*resource) {
|
|
case unifiedresources.ResourceTypeAppContainer:
|
|
if resource.Docker == nil {
|
|
continue
|
|
}
|
|
hostID := strings.TrimSpace(resource.Docker.HostSourceID)
|
|
containerID := strings.TrimSpace(resource.Docker.ContainerID)
|
|
if hostID == "" {
|
|
continue
|
|
}
|
|
if customURL, ok := m.dockerAppContainerCustomURL(*resource, hostID, containerID); ok {
|
|
// The metadata record is authoritative even when empty: an
|
|
// explicit clear must remove a stale URL still carried by an
|
|
// older unified-resource snapshot.
|
|
resource.CustomURL = strings.TrimSpace(customURL)
|
|
}
|
|
case unifiedresources.ResourceTypePod,
|
|
unifiedresources.ResourceTypeK8sDeployment,
|
|
unifiedresources.ResourceTypeK8sService:
|
|
if customURL, ok := m.kubernetesWorkloadCustomURL(*resource); ok {
|
|
resource.CustomURL = strings.TrimSpace(customURL)
|
|
}
|
|
case unifiedresources.ResourceTypeAgent,
|
|
unifiedresources.ResourceType("docker-host"),
|
|
unifiedresources.ResourceTypePBS,
|
|
unifiedresources.ResourceTypePMG,
|
|
unifiedresources.ResourceTypeK8sCluster,
|
|
unifiedresources.ResourceTypeK8sNode:
|
|
if customURL, ok := m.hostResourceCustomURL(*resource); ok {
|
|
resource.CustomURL = customURL
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func appendUniqueMetadataCandidate(candidates []string, seen map[string]struct{}, value string) []string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return candidates
|
|
}
|
|
if _, exists := seen[value]; exists {
|
|
return candidates
|
|
}
|
|
seen[value] = struct{}{}
|
|
return append(candidates, value)
|
|
}
|
|
|
|
func hostResourceMetadataCandidates(resource unifiedresources.Resource) []string {
|
|
candidates := make([]string, 0, 20)
|
|
seen := make(map[string]struct{}, 20)
|
|
add := func(value string) {
|
|
candidates = appendUniqueMetadataCandidate(candidates, seen, value)
|
|
}
|
|
|
|
if resource.Docker != nil {
|
|
add(resource.Docker.HostSourceID)
|
|
add(resource.Docker.AgentID)
|
|
}
|
|
if resource.Agent != nil {
|
|
add(resource.Agent.AgentID)
|
|
}
|
|
if resource.PBS != nil {
|
|
add(resource.PBS.InstanceID)
|
|
}
|
|
if resource.PMG != nil {
|
|
add(resource.PMG.InstanceID)
|
|
}
|
|
if resource.Kubernetes != nil {
|
|
if unifiedresources.ContractResourceType(resource) == unifiedresources.ResourceTypeK8sNode {
|
|
clusterID := strings.TrimSpace(resource.Kubernetes.ClusterID)
|
|
nodeName := strings.TrimSpace(resource.Kubernetes.NodeName)
|
|
if nodeName == "" {
|
|
nodeName = strings.TrimSpace(resource.Name)
|
|
}
|
|
if clusterID != "" && nodeName != "" {
|
|
add(clusterID + ":node:" + nodeName)
|
|
}
|
|
add(resource.Kubernetes.NodeUID)
|
|
}
|
|
add(resource.Kubernetes.ClusterID)
|
|
add(resource.Kubernetes.AgentID)
|
|
}
|
|
if resource.Proxmox != nil {
|
|
add(resource.Proxmox.SourceID)
|
|
add(resource.Proxmox.NodeName)
|
|
}
|
|
if resource.DiscoveryTarget != nil {
|
|
add(resource.DiscoveryTarget.AgentID)
|
|
add(resource.DiscoveryTarget.ResourceID)
|
|
}
|
|
if resource.MetricsTarget != nil {
|
|
add(resource.MetricsTarget.ResourceID)
|
|
}
|
|
if resource.Canonical != nil {
|
|
add(resource.Canonical.PrimaryID)
|
|
if prefixAt := strings.IndexByte(resource.Canonical.PrimaryID, ':'); prefixAt >= 0 {
|
|
add(resource.Canonical.PrimaryID[prefixAt+1:])
|
|
}
|
|
for _, alias := range resource.Canonical.Aliases {
|
|
add(alias)
|
|
}
|
|
for _, supersededID := range resource.Canonical.SupersededIDs {
|
|
add(supersededID)
|
|
}
|
|
}
|
|
add(resource.Identity.MachineID)
|
|
add(resource.ID)
|
|
add(resource.Name)
|
|
return candidates
|
|
}
|
|
|
|
func (m *Monitor) hostResourceCustomURL(resource unifiedresources.Resource) (string, bool) {
|
|
if m == nil {
|
|
return "", false
|
|
}
|
|
|
|
candidates := hostResourceMetadataCandidates(resource)
|
|
if unifiedresources.ContractResourceType(resource) == unifiedresources.ResourceType("docker-host") &&
|
|
m.dockerMetadataStore != nil {
|
|
for _, candidate := range candidates {
|
|
if meta := m.dockerMetadataStore.GetHostMetadata(candidate); meta != nil {
|
|
if customURL := strings.TrimSpace(meta.CustomURL); customURL != "" {
|
|
return customURL, true
|
|
}
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
if m.hostMetadataStore != nil {
|
|
for _, candidate := range candidates {
|
|
if meta := m.hostMetadataStore.Get(candidate); meta != nil {
|
|
if customURL := strings.TrimSpace(meta.CustomURL); customURL != "" {
|
|
return customURL, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func (m *Monitor) dockerAppContainerCustomURL(
|
|
resource unifiedresources.Resource,
|
|
hostID,
|
|
containerID string,
|
|
) (string, bool) {
|
|
if m == nil {
|
|
return "", false
|
|
}
|
|
|
|
if m.guestMetadataStore != nil {
|
|
if stableKey := dockerAppContainerMetadataKey(hostID, resource.Name); stableKey != "" {
|
|
if meta := m.guestMetadataStore.Get(stableKey); meta != nil {
|
|
return strings.TrimSpace(meta.CustomURL), true
|
|
}
|
|
}
|
|
for _, key := range append(
|
|
[]string{strings.TrimSpace(resource.ID)},
|
|
dockerAppContainerGuestMetadataLegacyKeys(hostID, containerID)...,
|
|
) {
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if meta := m.guestMetadataStore.Get(key); meta != nil {
|
|
return strings.TrimSpace(meta.CustomURL), true
|
|
}
|
|
}
|
|
}
|
|
|
|
if m.dockerMetadataStore != nil {
|
|
// The runtime key is where drawer saves used to land, so it is always at
|
|
// least as fresh as the stable name key, which only receives
|
|
// copy-if-missing snapshots of it. Prefer the runtime record while it
|
|
// exists; the stable name key covers recreated containers whose old
|
|
// runtime key no longer matches.
|
|
if meta := m.dockerMetadataStore.Get(dockerContainerRuntimeMetadataKey(hostID, containerID)); meta != nil {
|
|
return strings.TrimSpace(meta.CustomURL), true
|
|
}
|
|
if stableKey := dockerContainerNameMetadataKey(hostID, resource.Name); stableKey != "" {
|
|
if meta := m.dockerMetadataStore.Get(stableKey); meta != nil {
|
|
return strings.TrimSpace(meta.CustomURL), true
|
|
}
|
|
}
|
|
}
|
|
|
|
return "", false
|
|
}
|
|
|
|
// broadcastResourceCatalogs carries the content-addressed dedupe catalogs the
|
|
// broadcast conversion extracts from per-resource static metadata.
|
|
type broadcastResourceCatalogs struct {
|
|
capabilities map[string]json.RawMessage
|
|
policies map[string]json.RawMessage
|
|
aiSafeSummaries map[string]string
|
|
}
|
|
|
|
// convertResourcesForBroadcast converts unified resources into the frontend
|
|
// payload shape. Distinct capability blobs are deduped into the returned
|
|
// catalog and referenced per resource via capabilitiesRef.
|
|
func convertResourcesForBroadcast(
|
|
allResources []unifiedresources.Resource,
|
|
metricsTargetResolvers ...MetricsTargetResourceStore,
|
|
) ([]models.ResourceFrontend, broadcastResourceCatalogs) {
|
|
if len(allResources) == 0 {
|
|
return []models.ResourceFrontend{}, broadcastResourceCatalogs{}
|
|
}
|
|
allResources = attachBroadcastMetricsTargets(
|
|
allResources,
|
|
firstBroadcastMetricsTargetResolver(metricsTargetResolvers),
|
|
)
|
|
allResources = unifiedresources.CoalescePresentationHostResources(allResources)
|
|
type broadcastResource struct {
|
|
input models.ResourceConvertInput
|
|
sortKey string
|
|
resourceID string
|
|
}
|
|
|
|
converted := make([]broadcastResource, 0, len(allResources))
|
|
catalogs := broadcastResourceCatalogs{
|
|
capabilities: make(map[string]json.RawMessage),
|
|
policies: make(map[string]json.RawMessage),
|
|
aiSafeSummaries: make(map[string]string),
|
|
}
|
|
for _, r := range allResources {
|
|
input := monitorResourceToConvertInput(r)
|
|
if len(input.Capabilities) > 0 {
|
|
id := capabilityCatalogID(input.Capabilities)
|
|
catalogs.capabilities[id] = input.Capabilities
|
|
input.CapabilitiesRef = id
|
|
input.Capabilities = nil
|
|
}
|
|
// Non-default policies and AI-safe summaries dedupe the same way:
|
|
// estates carry a handful of distinct postures and templated summary
|
|
// strings, so refs replace per-resource inline duplication.
|
|
if len(input.Policy) > 0 {
|
|
id := capabilityCatalogID(input.Policy)
|
|
catalogs.policies[id] = input.Policy
|
|
input.PolicyRef = id
|
|
input.Policy = nil
|
|
}
|
|
if input.AISafeSummary != "" {
|
|
id := capabilityCatalogID(json.RawMessage(input.AISafeSummary))
|
|
catalogs.aiSafeSummaries[id] = input.AISafeSummary
|
|
input.AISafeSummaryRef = id
|
|
input.AISafeSummary = ""
|
|
}
|
|
sortKey := strings.ToLower(input.DisplayName)
|
|
if sortKey == "" {
|
|
sortKey = strings.ToLower(input.Name)
|
|
}
|
|
converted = append(converted, broadcastResource{
|
|
input: input,
|
|
sortKey: sortKey,
|
|
resourceID: input.ID,
|
|
})
|
|
}
|
|
|
|
sort.Slice(converted, func(i, j int) bool {
|
|
if converted[i].sortKey == converted[j].sortKey {
|
|
return converted[i].resourceID < converted[j].resourceID
|
|
}
|
|
return converted[i].sortKey < converted[j].sortKey
|
|
})
|
|
|
|
result := make([]models.ResourceFrontend, len(converted))
|
|
for i, resource := range converted {
|
|
result[i] = models.ConvertResourceToFrontend(resource.input)
|
|
}
|
|
if len(catalogs.capabilities) == 0 {
|
|
catalogs.capabilities = nil
|
|
}
|
|
if len(catalogs.policies) == 0 {
|
|
catalogs.policies = nil
|
|
}
|
|
if len(catalogs.aiSafeSummaries) == 0 {
|
|
catalogs.aiSafeSummaries = nil
|
|
}
|
|
return result, catalogs
|
|
}
|
|
|
|
func broadcastMetricsTargetResolver(source interface{}) MetricsTargetResourceStore {
|
|
resolver, ok := source.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return resolver
|
|
}
|
|
|
|
func firstBroadcastMetricsTargetResolver(
|
|
resolvers []MetricsTargetResourceStore,
|
|
) MetricsTargetResourceStore {
|
|
for _, resolver := range resolvers {
|
|
if resolver != nil {
|
|
return resolver
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func attachBroadcastMetricsTargets(
|
|
resources []unifiedresources.Resource,
|
|
resolver MetricsTargetResourceStore,
|
|
) []unifiedresources.Resource {
|
|
if len(resources) == 0 || resolver == nil {
|
|
return resources
|
|
}
|
|
|
|
out := make([]unifiedresources.Resource, len(resources))
|
|
for i, resource := range resources {
|
|
if resource.MetricsTarget == nil {
|
|
resource.MetricsTarget = validBroadcastMetricsTarget(
|
|
resolver.MetricsTargetForResource(resource.ID),
|
|
)
|
|
}
|
|
out[i] = resource
|
|
}
|
|
return out
|
|
}
|
|
|
|
func validBroadcastMetricsTarget(
|
|
target *unifiedresources.MetricsTarget,
|
|
) *unifiedresources.MetricsTarget {
|
|
if target == nil {
|
|
return nil
|
|
}
|
|
resourceType := strings.TrimSpace(target.ResourceType)
|
|
resourceID := strings.TrimSpace(target.ResourceID)
|
|
if resourceType == "" || resourceID == "" {
|
|
return nil
|
|
}
|
|
return &unifiedresources.MetricsTarget{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
}
|
|
}
|
|
|
|
func monitorResourceToConvertInput(resource unifiedresources.Resource) models.ResourceConvertInput {
|
|
resourceType := monitorFrontendResourceType(resource)
|
|
if resource.DiscoveryTarget == nil {
|
|
resource.DiscoveryTarget = monitorDiscoveryTarget(resource, resourceType)
|
|
}
|
|
if resource.MetricsTarget == nil {
|
|
resource.MetricsTarget = monitorMetricsTarget(resource, resourceType)
|
|
}
|
|
unifiedresources.RefreshCanonicalMetadata(&resource)
|
|
slimResourceForBroadcast(&resource)
|
|
name, displayName := monitorFrontendNames(resource, resourceType)
|
|
platformID := monitorPlatformID(resource, resourceType)
|
|
|
|
input := models.ResourceConvertInput{
|
|
ID: resource.ID,
|
|
Type: resourceType,
|
|
Technology: resource.Technology,
|
|
Name: name,
|
|
DisplayName: displayName,
|
|
PlatformID: platformID,
|
|
PlatformType: monitorPlatformType(resource, resourceType),
|
|
SourceType: monitorSourceType(resource.Sources),
|
|
Sources: monitorSourceKeys(resource.Sources),
|
|
ParentID: monitorStringValue(resource.ParentID),
|
|
ParentName: resource.ParentName,
|
|
ChildCount: resource.ChildCount,
|
|
ClusterID: monitorClusterID(resource),
|
|
Status: monitorFrontendStatus(resource, resourceType),
|
|
CPU: monitorMetricInput(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.CPU })),
|
|
Memory: monitorMetricInput(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Memory })),
|
|
Disk: monitorMetricInput(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Disk })),
|
|
Temperature: monitorTemperature(resource),
|
|
Uptime: monitorUptime(resource),
|
|
Tags: append([]string(nil), resource.Tags...),
|
|
Labels: monitorLabels(resource),
|
|
CustomURL: strings.TrimSpace(resource.CustomURL),
|
|
LastSeenUnix: monitorLastSeenUnix(resource.LastSeen),
|
|
Health: monitorRawJSON(resource.Health),
|
|
IncidentCount: resource.IncidentCount,
|
|
IncidentCode: resource.IncidentCode,
|
|
IncidentSeverity: string(resource.IncidentSeverity),
|
|
IncidentSummary: resource.IncidentSummary,
|
|
IncidentCategory: resource.IncidentCategory,
|
|
IncidentLabel: resource.IncidentLabel,
|
|
IncidentPriority: resource.IncidentPriority,
|
|
IncidentImpactSummary: resource.IncidentImpactSummary,
|
|
IncidentUrgency: resource.IncidentUrgency,
|
|
IncidentAction: resource.IncidentAction,
|
|
Identity: monitorIdentity(resource, name),
|
|
DiscoveryTarget: monitorRawJSON(resource.DiscoveryTarget),
|
|
MetricsTarget: monitorRawJSON(resource.MetricsTarget),
|
|
Canonical: monitorRawJSON(resource.Canonical),
|
|
Policy: monitorRawJSON(resource.Policy),
|
|
AISafeSummary: resource.AISafeSummary,
|
|
Capabilities: monitorRawJSON(resource.Capabilities),
|
|
Relationships: monitorRawJSON(resource.Relationships),
|
|
RecentChanges: monitorRawJSON(resource.RecentChanges),
|
|
FacetCounts: monitorRawJSON(resource.FacetCounts),
|
|
Incidents: monitorRawJSON(resource.Incidents),
|
|
Proxmox: monitorRawJSON(resource.Proxmox),
|
|
Storage: monitorRawJSON(resource.Storage),
|
|
Agent: monitorRawJSON(resource.Agent),
|
|
Docker: monitorRawJSON(resource.Docker),
|
|
PBS: monitorRawJSON(resource.PBS),
|
|
PMG: monitorRawJSON(resource.PMG),
|
|
Kubernetes: monitorRawJSON(resource.Kubernetes),
|
|
PhysicalDisk: monitorRawJSON(resource.PhysicalDisk),
|
|
Ceph: monitorRawJSON(resource.Ceph),
|
|
TrueNAS: monitorRawJSON(resource.TrueNAS),
|
|
VMware: monitorRawJSON(resource.VMware),
|
|
Availability: monitorRawJSON(resource.Availability),
|
|
AvailabilityChecks: monitorRawJSON(resource.AvailabilityChecks),
|
|
PlatformData: monitorPlatformData(resource, resourceType, platformID),
|
|
}
|
|
|
|
hasNetwork, rx, tx := monitorNetworkMetricInput(resource.Metrics)
|
|
input.HasNetwork = hasNetwork
|
|
input.NetworkRX = rx
|
|
input.NetworkTX = tx
|
|
input.DiskReadRate, input.DiskWriteRate = monitorDiskIOMetricInput(resource.Metrics)
|
|
|
|
return input
|
|
}
|
|
|
|
func monitorRawJSON(value interface{}) json.RawMessage {
|
|
if value == nil {
|
|
return nil
|
|
}
|
|
encoded, err := json.Marshal(value)
|
|
if err != nil || string(encoded) == "null" {
|
|
return nil
|
|
}
|
|
return encoded
|
|
}
|
|
|
|
// slimResourceForBroadcast trims static metadata the client-facing stream was
|
|
// re-shipping on every resource (governed gap resource-payload-static-metadata).
|
|
// It only edits the freshly refreshed per-broadcast copy, never stored state:
|
|
// - Aliases duplicating superseded canonical ids are dropped; the ids remain
|
|
// available in canonicalIdentity.supersededIds, which identity consumers
|
|
// also resolve against.
|
|
// - Default-posture policy and its AI-safe summary are omitted; consumers
|
|
// treat a missing policy as the default posture, and the summary is only
|
|
// rendered for non-default postures.
|
|
func slimResourceForBroadcast(resource *unifiedresources.Resource) {
|
|
if resource == nil {
|
|
return
|
|
}
|
|
if canonical := resource.Canonical; canonical != nil && len(canonical.Aliases) > 0 {
|
|
duplicated := make(map[string]struct{}, len(canonical.SupersededIDs)+2)
|
|
for _, id := range canonical.SupersededIDs {
|
|
duplicated[strings.ToLower(strings.TrimSpace(id))] = struct{}{}
|
|
}
|
|
// The row's own wire id is a self-reference: every identity consumer
|
|
// unions resource.id beside the alias spread, so the wire never needs
|
|
// it inside aliases. The primary id stays: aliases remain the complete
|
|
// live vocabulary a host is reachable by (see
|
|
// TestBroadcastSlimmingKeepsHostAgentSupersededIdentityResolvable).
|
|
if resource.ID != "" {
|
|
duplicated[strings.ToLower(strings.TrimSpace(resource.ID))] = struct{}{}
|
|
}
|
|
aliases := make([]string, 0, len(canonical.Aliases))
|
|
for _, alias := range canonical.Aliases {
|
|
if _, dup := duplicated[strings.ToLower(strings.TrimSpace(alias))]; dup {
|
|
continue
|
|
}
|
|
aliases = append(aliases, alias)
|
|
}
|
|
canonical.Aliases = aliases
|
|
}
|
|
if isDefaultBroadcastResourcePolicy(resource.Policy) {
|
|
resource.Policy = nil
|
|
resource.AISafeSummary = ""
|
|
}
|
|
}
|
|
|
|
// isDefaultBroadcastResourcePolicy mirrors the frontend's
|
|
// hasDefaultResourcePolicyPosture: internal sensitivity, cloud-summary
|
|
// routing, and no redactions is the posture consumers assume when the
|
|
// broadcast omits the policy entirely.
|
|
func isDefaultBroadcastResourcePolicy(policy *unifiedresources.ResourcePolicy) bool {
|
|
return policy != nil &&
|
|
policy.Sensitivity == unifiedresources.ResourceSensitivityInternal &&
|
|
policy.Routing.Scope == unifiedresources.ResourceRoutingScopeCloudSummary &&
|
|
len(policy.Routing.Redact) == 0
|
|
}
|
|
|
|
// capabilityCatalogID derives the stable catalog id for one marshaled
|
|
// capabilities blob. Content-addressed ids keep the catalog and refs stable
|
|
// across restarts and reconnects, so they never churn websocket deltas.
|
|
func capabilityCatalogID(encoded json.RawMessage) string {
|
|
digest := sha256.Sum256(encoded)
|
|
return hex.EncodeToString(digest[:6])
|
|
}
|
|
|
|
func monitorDiscoveryTarget(resource unifiedresources.Resource, resourceType string) *unifiedresources.DiscoveryTarget {
|
|
switch resourceType {
|
|
case "agent", "docker-host":
|
|
agentID := monitorAgentTargetID(resource)
|
|
if agentID == "" {
|
|
return nil
|
|
}
|
|
return &unifiedresources.DiscoveryTarget{
|
|
ResourceType: string(unifiedresources.ResourceTypeAgent),
|
|
AgentID: agentID,
|
|
ResourceID: agentID,
|
|
Hostname: monitorTargetHostname(resource),
|
|
}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func monitorMetricsTarget(resource unifiedresources.Resource, resourceType string) *unifiedresources.MetricsTarget {
|
|
switch resourceType {
|
|
case "agent", "docker-host":
|
|
agentID := monitorAgentTargetID(resource)
|
|
if agentID == "" {
|
|
return nil
|
|
}
|
|
return &unifiedresources.MetricsTarget{
|
|
ResourceType: string(unifiedresources.ResourceTypeAgent),
|
|
ResourceID: agentID,
|
|
}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func monitorAgentTargetID(resource unifiedresources.Resource) string {
|
|
if resource.Agent != nil {
|
|
if id := strings.TrimSpace(resource.Agent.AgentID); id != "" {
|
|
return id
|
|
}
|
|
}
|
|
if resource.Docker != nil {
|
|
if id := strings.TrimSpace(resource.Docker.AgentID); id != "" {
|
|
return id
|
|
}
|
|
if id := strings.TrimSpace(resource.Docker.HostSourceID); id != "" {
|
|
return id
|
|
}
|
|
}
|
|
if resource.Kubernetes != nil {
|
|
if id := strings.TrimSpace(resource.Kubernetes.AgentID); id != "" {
|
|
return id
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func monitorTargetHostname(resource unifiedresources.Resource) string {
|
|
if resource.Agent != nil {
|
|
if hostname := strings.TrimSpace(resource.Agent.Hostname); hostname != "" {
|
|
return hostname
|
|
}
|
|
}
|
|
if resource.Docker != nil {
|
|
if hostname := strings.TrimSpace(resource.Docker.Hostname); hostname != "" {
|
|
return hostname
|
|
}
|
|
}
|
|
if resource.PBS != nil {
|
|
if hostname := strings.TrimSpace(resource.PBS.Hostname); hostname != "" {
|
|
return hostname
|
|
}
|
|
}
|
|
if resource.PMG != nil {
|
|
if hostname := strings.TrimSpace(resource.PMG.Hostname); hostname != "" {
|
|
return hostname
|
|
}
|
|
}
|
|
return strings.TrimSpace(resource.Name)
|
|
}
|
|
|
|
func monitorFrontendResourceType(resource unifiedresources.Resource) string {
|
|
return string(unifiedresources.ContractResourceType(resource))
|
|
}
|
|
|
|
func monitorFrontendNames(resource unifiedresources.Resource, resourceType string) (string, string) {
|
|
name := strings.TrimSpace(unifiedresources.ResourceDisplayName(resource))
|
|
if name == "" {
|
|
name = resource.ID
|
|
}
|
|
return name, name
|
|
}
|
|
|
|
func monitorPlatformType(resource unifiedresources.Resource, resourceType string) string {
|
|
if resource.Proxmox != nil {
|
|
return "proxmox-pve"
|
|
}
|
|
if resource.VMware != nil {
|
|
return "vmware-vsphere"
|
|
}
|
|
if resource.TrueNAS != nil {
|
|
return "truenas"
|
|
}
|
|
if resource.Availability != nil {
|
|
return "generic"
|
|
}
|
|
if storagePlatform := monitorStoragePlatformType(resource.Storage, resource.Sources); storagePlatform != "" {
|
|
return storagePlatform
|
|
}
|
|
switch resourceType {
|
|
case "vm", "system-container":
|
|
return "proxmox-pve"
|
|
case "storage", "pool":
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceProxmox) {
|
|
return "proxmox-pve"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourcePBS) {
|
|
return "proxmox-pbs"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceTrueNAS) {
|
|
return "truenas"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceVMware) {
|
|
return "vmware-vsphere"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceAgent) {
|
|
return "agent"
|
|
}
|
|
return "generic"
|
|
case "docker-host", "app-container":
|
|
return "docker"
|
|
case "k8s-cluster", "k8s-node", "pod", "k8s-deployment":
|
|
return "kubernetes"
|
|
case "pbs":
|
|
return "proxmox-pbs"
|
|
case "pmg":
|
|
return "proxmox-pmg"
|
|
case "agent":
|
|
return "agent"
|
|
default:
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceK8s) {
|
|
return "kubernetes"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceDocker) {
|
|
return "docker"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourcePBS) {
|
|
return "proxmox-pbs"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourcePMG) {
|
|
return "proxmox-pmg"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceAgent) {
|
|
return "agent"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceProxmox) {
|
|
return "proxmox-pve"
|
|
}
|
|
for _, source := range resource.Sources {
|
|
if candidate := strings.TrimSpace(string(source)); candidate != "" {
|
|
return candidate
|
|
}
|
|
}
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
func monitorStoragePlatformType(storage *unifiedresources.StorageMeta, sources []unifiedresources.DataSource) string {
|
|
if storage == nil {
|
|
return ""
|
|
}
|
|
switch strings.ToLower(strings.TrimSpace(storage.Platform)) {
|
|
case "proxmox", "proxmox-pve", "pve":
|
|
return "proxmox-pve"
|
|
case "pbs", "proxmox-pbs":
|
|
return "proxmox-pbs"
|
|
case "truenas":
|
|
return "truenas"
|
|
case "vmware", "vmware-vsphere", "vsphere":
|
|
return "vmware-vsphere"
|
|
case "docker", "podman":
|
|
return "docker"
|
|
case "kubernetes", "k8s":
|
|
return "kubernetes"
|
|
case "unraid":
|
|
if monitorHasSource(sources, unifiedresources.SourceAgent) {
|
|
return "agent"
|
|
}
|
|
return "generic"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func monitorPlatformID(resource unifiedresources.Resource, resourceType string) string {
|
|
switch resourceType {
|
|
case "node", "vm", "system-container":
|
|
if resource.Proxmox != nil && strings.TrimSpace(resource.Proxmox.Instance) != "" {
|
|
return strings.TrimSpace(resource.Proxmox.Instance)
|
|
}
|
|
case "agent":
|
|
if resource.Agent != nil && strings.TrimSpace(resource.Agent.AgentID) != "" {
|
|
return strings.TrimSpace(resource.Agent.AgentID)
|
|
}
|
|
case "docker-host":
|
|
if resource.Docker != nil && strings.TrimSpace(resource.Docker.Hostname) != "" {
|
|
return strings.TrimSpace(resource.Docker.Hostname)
|
|
}
|
|
case "app-container":
|
|
if resource.Docker != nil && strings.TrimSpace(resource.Docker.Hostname) != "" {
|
|
return strings.TrimSpace(resource.Docker.Hostname)
|
|
}
|
|
if resource.ParentID != nil {
|
|
return strings.TrimSpace(*resource.ParentID)
|
|
}
|
|
case "k8s-cluster", "k8s-node", "pod", "k8s-deployment":
|
|
if resource.Kubernetes != nil && strings.TrimSpace(resource.Kubernetes.AgentID) != "" {
|
|
return strings.TrimSpace(resource.Kubernetes.AgentID)
|
|
}
|
|
case "pbs":
|
|
if resource.PBS != nil && strings.TrimSpace(resource.PBS.Hostname) != "" {
|
|
return strings.TrimSpace(resource.PBS.Hostname)
|
|
}
|
|
case "pmg":
|
|
if resource.PMG != nil && strings.TrimSpace(resource.PMG.Hostname) != "" {
|
|
return strings.TrimSpace(resource.PMG.Hostname)
|
|
}
|
|
case "network-endpoint":
|
|
if resource.Availability != nil {
|
|
if targetID := strings.TrimSpace(resource.Availability.TargetID); targetID != "" {
|
|
return targetID
|
|
}
|
|
if address := strings.TrimSpace(resource.Availability.Address); address != "" {
|
|
return address
|
|
}
|
|
}
|
|
}
|
|
return resource.ID
|
|
}
|
|
|
|
func monitorFrontendStatus(resource unifiedresources.Resource, resourceType string) string {
|
|
switch resourceType {
|
|
case "app-container":
|
|
switch resource.Status {
|
|
case unifiedresources.StatusOnline:
|
|
return "running"
|
|
case unifiedresources.StatusOffline:
|
|
return "stopped"
|
|
case unifiedresources.StatusWarning:
|
|
return "degraded"
|
|
}
|
|
case "pod":
|
|
if resource.Kubernetes != nil {
|
|
phase := strings.ToLower(strings.TrimSpace(resource.Kubernetes.PodPhase))
|
|
switch phase {
|
|
case "pending", "unknown":
|
|
return "degraded"
|
|
case "succeeded", "failed":
|
|
return "stopped"
|
|
}
|
|
// Phase=Running is not enough — a pod can have Phase=Running with
|
|
// CrashLoopBackOff containers, which statusFromKubernetesPod
|
|
// correctly classifies as StatusWarning. Fall through to the
|
|
// unified-status mapping below so that surfaces as "degraded".
|
|
}
|
|
}
|
|
|
|
switch resource.Status {
|
|
case unifiedresources.StatusOnline:
|
|
if monitorIsWorkloadType(resourceType) || resourceType == "pod" {
|
|
return "running"
|
|
}
|
|
return "online"
|
|
case unifiedresources.StatusOffline:
|
|
if monitorIsWorkloadType(resourceType) || resourceType == "pod" {
|
|
return "stopped"
|
|
}
|
|
return "offline"
|
|
case unifiedresources.StatusWarning:
|
|
return "degraded"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
func monitorIsWorkloadType(resourceType string) bool {
|
|
switch resourceType {
|
|
case "app-container", "system-container", "vm", "oci-container":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func monitorClusterID(resource unifiedresources.Resource) string {
|
|
if clusterID := strings.TrimSpace(unifiedresources.ResourceClusterName(resource)); clusterID != "" {
|
|
return clusterID
|
|
}
|
|
|
|
if resource.Docker != nil && resource.Docker.Swarm != nil {
|
|
if name := strings.TrimSpace(resource.Docker.Swarm.ClusterName); name != "" {
|
|
return name
|
|
}
|
|
if id := strings.TrimSpace(resource.Docker.Swarm.ClusterID); id != "" {
|
|
return id
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func monitorMetricInput(metric *unifiedresources.MetricValue) *models.ResourceMetricInput {
|
|
if metric == nil {
|
|
return nil
|
|
}
|
|
|
|
current := metric.Percent
|
|
if current == 0 {
|
|
current = metric.Value
|
|
}
|
|
if metric.Percent != 0 && metric.Value != 0 {
|
|
current = math.Max(metric.Percent, metric.Value)
|
|
}
|
|
|
|
result := &models.ResourceMetricInput{Current: current}
|
|
if metric.Total != nil {
|
|
total := *metric.Total
|
|
result.Total = &total
|
|
}
|
|
if metric.Used != nil {
|
|
used := *metric.Used
|
|
result.Used = &used
|
|
}
|
|
if result.Total != nil && result.Used != nil {
|
|
free := *result.Total - *result.Used
|
|
result.Free = &free
|
|
}
|
|
return result
|
|
}
|
|
|
|
func monitorNetworkMetricInput(metrics *unifiedresources.ResourceMetrics) (bool, int64, int64) {
|
|
if metrics == nil || (metrics.NetIn == nil && metrics.NetOut == nil) {
|
|
return false, 0, 0
|
|
}
|
|
|
|
var rx int64
|
|
var tx int64
|
|
if metrics.NetIn != nil {
|
|
rx = int64(math.Round(metrics.NetIn.Value))
|
|
}
|
|
if metrics.NetOut != nil {
|
|
tx = int64(math.Round(metrics.NetOut.Value))
|
|
}
|
|
return true, rx, tx
|
|
}
|
|
|
|
func monitorDiskIOMetricInput(metrics *unifiedresources.ResourceMetrics) (*int64, *int64) {
|
|
if metrics == nil || (metrics.DiskRead == nil && metrics.DiskWrite == nil) {
|
|
return nil, nil
|
|
}
|
|
var read, write *int64
|
|
if metrics.DiskRead != nil {
|
|
value := int64(math.Round(metrics.DiskRead.Value))
|
|
read = &value
|
|
}
|
|
if metrics.DiskWrite != nil {
|
|
value := int64(math.Round(metrics.DiskWrite.Value))
|
|
write = &value
|
|
}
|
|
return read, write
|
|
}
|
|
|
|
func monitorTemperature(resource unifiedresources.Resource) *float64 {
|
|
if resource.Agent != nil && resource.Agent.Temperature != nil {
|
|
value := *resource.Agent.Temperature
|
|
return &value
|
|
}
|
|
if resource.Proxmox != nil && resource.Proxmox.Temperature != nil {
|
|
value := *resource.Proxmox.Temperature
|
|
return &value
|
|
}
|
|
if resource.Docker != nil && resource.Docker.Temperature != nil {
|
|
value := *resource.Docker.Temperature
|
|
return &value
|
|
}
|
|
if resource.Kubernetes != nil && resource.Kubernetes.Temperature != nil {
|
|
value := *resource.Kubernetes.Temperature
|
|
return &value
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func monitorUptime(resource unifiedresources.Resource) *int64 {
|
|
if resource.Agent != nil && resource.Agent.UptimeSeconds > 0 {
|
|
value := resource.Agent.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.Proxmox != nil && resource.Proxmox.Uptime > 0 {
|
|
value := resource.Proxmox.Uptime
|
|
return &value
|
|
}
|
|
if resource.Docker != nil && resource.Docker.UptimeSeconds > 0 {
|
|
value := resource.Docker.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.Kubernetes != nil && resource.Kubernetes.UptimeSeconds > 0 {
|
|
value := resource.Kubernetes.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.PBS != nil && resource.PBS.UptimeSeconds > 0 {
|
|
value := resource.PBS.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.PMG != nil && resource.PMG.UptimeSeconds > 0 {
|
|
value := resource.PMG.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.TrueNAS != nil && resource.TrueNAS.UptimeSeconds > 0 {
|
|
value := resource.TrueNAS.UptimeSeconds
|
|
return &value
|
|
}
|
|
// Canonical Resource.Uptime is the universal fallback. The vSphere
|
|
// adapter populates only this field for ESXi hosts and VMs (no
|
|
// vmware-specific UptimeSeconds carve-out), so without this fall the
|
|
// websocket broadcast layer would drop uptime for VMware-backed rows
|
|
// even though the REST `/api/resources` payload exposes it on
|
|
// `resource.uptime` directly.
|
|
if resource.Uptime > 0 {
|
|
value := resource.Uptime
|
|
return &value
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func monitorLabels(resource unifiedresources.Resource) map[string]string {
|
|
if resource.Kubernetes == nil || len(resource.Kubernetes.Labels) == 0 {
|
|
return nil
|
|
}
|
|
labels := make(map[string]string, len(resource.Kubernetes.Labels))
|
|
for key, value := range resource.Kubernetes.Labels {
|
|
labels[key] = value
|
|
}
|
|
return labels
|
|
}
|
|
|
|
func monitorIdentity(resource unifiedresources.Resource, fallbackName string) *models.ResourceIdentityInput {
|
|
hostname := ""
|
|
if resource.Agent != nil {
|
|
hostname = strings.TrimSpace(resource.Agent.Hostname)
|
|
}
|
|
if hostname == "" && resource.Docker != nil {
|
|
hostname = strings.TrimSpace(resource.Docker.Hostname)
|
|
}
|
|
if hostname == "" && resource.Proxmox != nil {
|
|
hostname = strings.TrimSpace(resource.Proxmox.NodeName)
|
|
}
|
|
if hostname == "" && resource.Availability != nil {
|
|
hostname = strings.TrimSpace(resource.Availability.Address)
|
|
}
|
|
if hostname == "" {
|
|
for _, candidate := range resource.Identity.Hostnames {
|
|
if trimmed := strings.TrimSpace(candidate); trimmed != "" {
|
|
hostname = trimmed
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if hostname == "" {
|
|
hostname = fallbackName
|
|
}
|
|
|
|
ips := make([]string, 0, len(resource.Identity.IPAddresses))
|
|
for _, ip := range resource.Identity.IPAddresses {
|
|
trimmed := strings.TrimSpace(ip)
|
|
if trimmed == "" {
|
|
continue
|
|
}
|
|
ips = append(ips, trimmed)
|
|
}
|
|
|
|
machineID := strings.TrimSpace(resource.Identity.MachineID)
|
|
if hostname == "" && machineID == "" && len(ips) == 0 {
|
|
return nil
|
|
}
|
|
|
|
return &models.ResourceIdentityInput{
|
|
Hostname: hostname,
|
|
MachineID: machineID,
|
|
IPs: ips,
|
|
}
|
|
}
|
|
|
|
func monitorPlatformData(resource unifiedresources.Resource, resourceType string, platformID string) json.RawMessage {
|
|
var payload interface{}
|
|
|
|
switch resourceType {
|
|
case "node":
|
|
if resource.Proxmox != nil {
|
|
payload = map[string]interface{}{
|
|
"instance": resource.Proxmox.Instance,
|
|
"host": "",
|
|
"guestURL": "",
|
|
"pveVersion": resource.Proxmox.PVEVersion,
|
|
"kernelVersion": resource.Proxmox.KernelVersion,
|
|
"cpuInfo": resource.Proxmox.CPUInfo,
|
|
"loadAverage": []float64{},
|
|
"isClusterMember": resource.Proxmox.ClusterName != "",
|
|
"clusterName": resource.Proxmox.ClusterName,
|
|
"connectionHealth": monitorSourceStatus(resource.SourceStatus, unifiedresources.SourceProxmox),
|
|
}
|
|
}
|
|
case "vm":
|
|
payload = buildProxmoxVMPayload(resource)
|
|
case "system-container", "oci-container":
|
|
payload = buildProxmoxVMPayload(resource)
|
|
case "agent":
|
|
if resource.Proxmox != nil {
|
|
payload = map[string]interface{}{
|
|
"instance": resource.Proxmox.Instance,
|
|
"host": "",
|
|
"guestURL": "",
|
|
"pveVersion": resource.Proxmox.PVEVersion,
|
|
"kernelVersion": resource.Proxmox.KernelVersion,
|
|
"cpuInfo": resource.Proxmox.CPUInfo,
|
|
"loadAverage": []float64{},
|
|
"isClusterMember": resource.Proxmox.ClusterName != "",
|
|
"clusterName": resource.Proxmox.ClusterName,
|
|
"connectionHealth": monitorSourceStatus(resource.SourceStatus, unifiedresources.SourceProxmox),
|
|
}
|
|
} else if resource.Agent != nil {
|
|
payload = map[string]interface{}{
|
|
"platform": resource.Agent.Platform,
|
|
"osName": resource.Agent.OSName,
|
|
"osVersion": resource.Agent.OSVersion,
|
|
"kernelVersion": resource.Agent.KernelVersion,
|
|
"architecture": resource.Agent.Architecture,
|
|
"agentVersion": resource.Agent.AgentVersion,
|
|
"interfaces": resource.Agent.NetworkInterfaces,
|
|
"disks": resource.Agent.Disks,
|
|
"memory": resource.Agent.Memory,
|
|
}
|
|
}
|
|
case "docker-host":
|
|
if resource.Docker != nil {
|
|
payload = map[string]interface{}{
|
|
"agentId": platformID,
|
|
"runtime": resource.Docker.Runtime,
|
|
"runtimeVersion": resource.Docker.RuntimeVersion,
|
|
"dockerVersion": resource.Docker.DockerVersion,
|
|
"os": resource.Docker.OS,
|
|
"kernelVersion": resource.Docker.KernelVersion,
|
|
"architecture": resource.Docker.Architecture,
|
|
"agentVersion": resource.Docker.AgentVersion,
|
|
"swarm": resource.Docker.Swarm,
|
|
"interfaces": resource.Docker.NetworkInterfaces,
|
|
"disks": resource.Docker.Disks,
|
|
}
|
|
}
|
|
case "app-container":
|
|
if resource.Docker != nil {
|
|
payload = map[string]interface{}{
|
|
"hostId": monitorStringValue(resource.ParentID),
|
|
"hostName": resource.Docker.Hostname,
|
|
"image": resource.Docker.Image,
|
|
"state": strings.ToLower(string(resource.Status)),
|
|
"status": strings.ToLower(string(resource.Status)),
|
|
"health": "",
|
|
"createdAt": time.Time{},
|
|
}
|
|
}
|
|
case "k8s-cluster":
|
|
if resource.Kubernetes != nil {
|
|
payload = map[string]interface{}{
|
|
"agentId": resource.Kubernetes.AgentID,
|
|
"server": resource.Kubernetes.Server,
|
|
"context": resource.Kubernetes.Context,
|
|
"version": resource.Kubernetes.Version,
|
|
"customDisplayName": "",
|
|
"hidden": false,
|
|
"pendingUninstall": resource.Kubernetes.PendingUninstall,
|
|
"nodeCount": resource.ChildCount,
|
|
}
|
|
}
|
|
case "k8s-node":
|
|
if resource.Kubernetes != nil {
|
|
payload = map[string]interface{}{
|
|
"clusterId": resource.Kubernetes.ClusterID,
|
|
"ready": resource.Kubernetes.Ready,
|
|
"unschedulable": resource.Kubernetes.Unschedulable,
|
|
"kubeletVersion": resource.Kubernetes.KubeletVersion,
|
|
"containerRuntimeVersion": resource.Kubernetes.ContainerRuntimeVersion,
|
|
"osImage": resource.Kubernetes.OSImage,
|
|
"kernelVersion": resource.Kubernetes.KernelVersion,
|
|
"architecture": resource.Kubernetes.Architecture,
|
|
"capacityCpuCores": resource.Kubernetes.CapacityCPU,
|
|
"capacityMemoryBytes": resource.Kubernetes.CapacityMemoryBytes,
|
|
"capacityPods": resource.Kubernetes.CapacityPods,
|
|
"allocatableCpuCores": resource.Kubernetes.AllocCPU,
|
|
"allocatableMemoryBytes": resource.Kubernetes.AllocMemoryBytes,
|
|
"allocatablePods": resource.Kubernetes.AllocPods,
|
|
"roles": append([]string(nil), resource.Kubernetes.Roles...),
|
|
}
|
|
}
|
|
case "pod":
|
|
if resource.Kubernetes != nil {
|
|
payload = map[string]interface{}{
|
|
"clusterId": resource.Kubernetes.ClusterID,
|
|
"namespace": resource.Kubernetes.Namespace,
|
|
"nodeName": resource.Kubernetes.NodeName,
|
|
"phase": resource.Kubernetes.PodPhase,
|
|
"restarts": resource.Kubernetes.Restarts,
|
|
"ownerKind": resource.Kubernetes.OwnerKind,
|
|
"ownerName": resource.Kubernetes.OwnerName,
|
|
}
|
|
}
|
|
case "k8s-deployment":
|
|
if resource.Kubernetes != nil {
|
|
payload = map[string]interface{}{
|
|
"clusterId": resource.Kubernetes.ClusterID,
|
|
"namespace": resource.Kubernetes.Namespace,
|
|
"desiredReplicas": resource.Kubernetes.DesiredReplicas,
|
|
"updatedReplicas": resource.Kubernetes.UpdatedReplicas,
|
|
"readyReplicas": resource.Kubernetes.ReadyReplicas,
|
|
"availableReplicas": resource.Kubernetes.AvailableReplicas,
|
|
}
|
|
}
|
|
case "pbs":
|
|
if resource.PBS != nil {
|
|
payload = map[string]interface{}{
|
|
"host": resource.PBS.Hostname,
|
|
"version": resource.PBS.Version,
|
|
"connectionHealth": resource.PBS.ConnectionHealth,
|
|
"memoryUsed": monitorMetricUsed(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Memory })),
|
|
"memoryTotal": monitorMetricTotal(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Memory })),
|
|
"numDatastores": resource.PBS.DatastoreCount,
|
|
}
|
|
}
|
|
case "pmg":
|
|
if resource.PMG != nil {
|
|
payload = map[string]interface{}{
|
|
"host": resource.PMG.Hostname,
|
|
"version": resource.PMG.Version,
|
|
"connectionHealth": resource.PMG.ConnectionHealth,
|
|
"nodeCount": resource.PMG.NodeCount,
|
|
"queueActive": resource.PMG.QueueActive,
|
|
"queueDeferred": resource.PMG.QueueDeferred,
|
|
"queueHold": resource.PMG.QueueHold,
|
|
"queueIncoming": resource.PMG.QueueIncoming,
|
|
"queueTotal": resource.PMG.QueueTotal,
|
|
"mailCountTotal": resource.PMG.MailCountTotal,
|
|
"spamIn": resource.PMG.SpamIn,
|
|
"virusIn": resource.PMG.VirusIn,
|
|
"lastUpdated": resource.PMG.LastUpdated,
|
|
}
|
|
}
|
|
case "storage", "pool":
|
|
payload = monitorStoragePlatformData(resource, platformID)
|
|
case "network-endpoint":
|
|
if resource.Availability != nil {
|
|
payload = map[string]interface{}{
|
|
"targetId": resource.Availability.TargetID,
|
|
"address": resource.Availability.Address,
|
|
"protocol": resource.Availability.Protocol,
|
|
"port": resource.Availability.Port,
|
|
"path": resource.Availability.Path,
|
|
"enabled": resource.Availability.Enabled,
|
|
"available": resource.Availability.Available,
|
|
"lastChecked": resource.Availability.LastChecked,
|
|
"lastSuccess": resource.Availability.LastSuccess,
|
|
"latencyMillis": resource.Availability.LatencyMillis,
|
|
"consecutiveFailures": resource.Availability.ConsecutiveFailures,
|
|
"lastError": resource.Availability.LastError,
|
|
"failureThreshold": resource.Availability.FailureThreshold,
|
|
"pollIntervalSeconds": resource.Availability.PollIntervalSeconds,
|
|
"timeoutMillis": resource.Availability.TimeoutMillis,
|
|
}
|
|
}
|
|
}
|
|
|
|
if payload == nil {
|
|
return nil
|
|
}
|
|
payload = monitorAttachSourceKeys(payload, resource.Sources)
|
|
if payload == nil {
|
|
return nil
|
|
}
|
|
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return encoded
|
|
}
|
|
|
|
func monitorAttachSourceKeys(payload interface{}, sources []unifiedresources.DataSource) interface{} {
|
|
sourceKeys := monitorSourceKeys(sources)
|
|
if len(sourceKeys) == 0 {
|
|
return payload
|
|
}
|
|
if payloadMap, ok := payload.(map[string]interface{}); ok {
|
|
if payloadMap == nil {
|
|
return nil
|
|
}
|
|
payloadMap["sources"] = sourceKeys
|
|
return payloadMap
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func monitorStoragePlatformData(resource unifiedresources.Resource, platformID string) map[string]interface{} {
|
|
nodeLabel := resource.ParentName
|
|
if nodeLabel == "" {
|
|
nodeLabel = monitorStringValue(resource.ParentID)
|
|
}
|
|
payload := map[string]interface{}{
|
|
"instance": platformID,
|
|
"node": nodeLabel,
|
|
"active": resource.Status == unifiedresources.StatusOnline,
|
|
}
|
|
|
|
if resource.Proxmox != nil {
|
|
if strings.TrimSpace(resource.Proxmox.Instance) != "" {
|
|
payload["instance"] = strings.TrimSpace(resource.Proxmox.Instance)
|
|
}
|
|
if strings.TrimSpace(resource.Proxmox.NodeName) != "" {
|
|
payload["node"] = strings.TrimSpace(resource.Proxmox.NodeName)
|
|
}
|
|
}
|
|
|
|
if resource.Storage == nil {
|
|
payload["type"] = ""
|
|
payload["content"] = ""
|
|
payload["shared"] = false
|
|
payload["enabled"] = true
|
|
return payload
|
|
}
|
|
|
|
storage := resource.Storage
|
|
payload["type"] = storage.Type
|
|
payload["content"] = storage.Content
|
|
payload["contentTypes"] = append([]string(nil), storage.ContentTypes...)
|
|
payload["shared"] = storage.Shared
|
|
payload["enabled"] = storage.Enabled
|
|
payload["active"] = storage.Active
|
|
payload["isCeph"] = storage.IsCeph
|
|
payload["isZfs"] = storage.IsZFS
|
|
payload["platform"] = storage.Platform
|
|
payload["topology"] = storage.Topology
|
|
payload["protection"] = storage.Protection
|
|
payload["pool"] = storage.Pool
|
|
payload["path"] = storage.Path
|
|
payload["nodes"] = append([]string(nil), storage.Nodes...)
|
|
payload["risk"] = storage.Risk
|
|
payload["riskSummary"] = storage.RiskSummary
|
|
payload["postureSummary"] = storage.PostureSummary
|
|
payload["protectionReduced"] = storage.ProtectionReduced
|
|
payload["protectionSummary"] = storage.ProtectionSummary
|
|
payload["rebuildInProgress"] = storage.RebuildInProgress
|
|
payload["rebuildSummary"] = storage.RebuildSummary
|
|
payload["arrayState"] = storage.ArrayState
|
|
payload["syncAction"] = storage.SyncAction
|
|
payload["syncProgress"] = storage.SyncProgress
|
|
payload["numProtected"] = storage.NumProtected
|
|
payload["numDisabled"] = storage.NumDisabled
|
|
payload["numInvalid"] = storage.NumInvalid
|
|
payload["numMissing"] = storage.NumMissing
|
|
payload["zfsPoolState"] = storage.ZFSPoolState
|
|
payload["zfsReadErrors"] = storage.ZFSReadErrors
|
|
payload["zfsWriteErrors"] = storage.ZFSWriteErrors
|
|
payload["zfsChecksumErrors"] = storage.ZFSChecksumErrors
|
|
if storage.ZFSPool != nil {
|
|
payload["zfsPool"] = storage.ZFSPool
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func convertProxmoxDisks(disks []unifiedresources.DiskInfo) []map[string]interface{} {
|
|
if len(disks) == 0 {
|
|
return nil
|
|
}
|
|
|
|
out := make([]map[string]interface{}, 0, len(disks))
|
|
for _, d := range disks {
|
|
usage := float64(0)
|
|
if d.Total > 0 {
|
|
usage = float64(d.Used) / float64(d.Total) * 100
|
|
}
|
|
out = append(out, map[string]interface{}{
|
|
"total": d.Total,
|
|
"used": d.Used,
|
|
"free": d.Free,
|
|
"usage": usage,
|
|
"mountpoint": d.Mountpoint,
|
|
"type": d.Filesystem,
|
|
"device": d.Device,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func buildProxmoxVMPayload(resource unifiedresources.Resource) map[string]interface{} {
|
|
if resource.Proxmox == nil {
|
|
return nil
|
|
}
|
|
return map[string]interface{}{
|
|
"vmid": resource.Proxmox.VMID,
|
|
"node": resource.Proxmox.NodeName,
|
|
"instance": resource.Proxmox.Instance,
|
|
"cpus": resource.Proxmox.CPUs,
|
|
"template": resource.Proxmox.Template,
|
|
"networkIn": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.NetIn }),
|
|
"networkOut": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.NetOut }),
|
|
"diskRead": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.DiskRead }),
|
|
"diskWrite": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue {
|
|
return metrics.DiskWrite
|
|
}),
|
|
"disks": convertProxmoxDisks(resource.Proxmox.Disks),
|
|
"swapUsed": resource.Proxmox.SwapUsed,
|
|
"swapTotal": resource.Proxmox.SwapTotal,
|
|
"balloon": resource.Proxmox.Balloon,
|
|
"lastBackup": resource.Proxmox.LastBackup,
|
|
// A backup of this guest is running right now; lastBackup still
|
|
// points at the latest completed backup.
|
|
"backupInProgress": resource.Proxmox.BackupInProgress,
|
|
"ipAddresses": append([]string(nil), resource.Identity.IPAddresses...),
|
|
}
|
|
}
|
|
|
|
func monitorMetricValue(metrics *unifiedresources.ResourceMetrics, pick func(*unifiedresources.ResourceMetrics) *unifiedresources.MetricValue) *unifiedresources.MetricValue {
|
|
if metrics == nil {
|
|
return nil
|
|
}
|
|
return pick(metrics)
|
|
}
|
|
|
|
func monitorMetricInt64(metrics *unifiedresources.ResourceMetrics, pick func(*unifiedresources.ResourceMetrics) *unifiedresources.MetricValue) int64 {
|
|
metric := monitorMetricValue(metrics, pick)
|
|
if metric == nil {
|
|
return 0
|
|
}
|
|
return int64(math.Round(metric.Value))
|
|
}
|
|
|
|
func monitorMetricUsed(metric *unifiedresources.MetricValue) int64 {
|
|
if metric == nil || metric.Used == nil {
|
|
return 0
|
|
}
|
|
return *metric.Used
|
|
}
|
|
|
|
func monitorMetricTotal(metric *unifiedresources.MetricValue) int64 {
|
|
if metric == nil || metric.Total == nil {
|
|
return 0
|
|
}
|
|
return *metric.Total
|
|
}
|
|
|
|
func monitorSourceStatus(statuses map[unifiedresources.DataSource]unifiedresources.SourceStatus, source unifiedresources.DataSource) string {
|
|
if statuses == nil {
|
|
return ""
|
|
}
|
|
status, ok := statuses[source]
|
|
if !ok {
|
|
return ""
|
|
}
|
|
return status.Status
|
|
}
|
|
|
|
func monitorHasSource(sources []unifiedresources.DataSource, source unifiedresources.DataSource) bool {
|
|
for _, candidate := range sources {
|
|
if candidate == source {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func monitorSourceKeys(sources []unifiedresources.DataSource) []string {
|
|
if len(sources) == 0 {
|
|
return nil
|
|
}
|
|
keys := make([]string, 0, len(sources))
|
|
seen := make(map[string]struct{}, len(sources))
|
|
for _, source := range sources {
|
|
key := strings.TrimSpace(string(source))
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[key]; ok {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
keys = append(keys, key)
|
|
}
|
|
return keys
|
|
}
|
|
|
|
func monitorSourceType(sources []unifiedresources.DataSource) string {
|
|
if len(sources) > 1 {
|
|
return "hybrid"
|
|
}
|
|
if len(sources) == 1 {
|
|
switch sources[0] {
|
|
case unifiedresources.SourceAgent, unifiedresources.SourceDocker, unifiedresources.SourceK8s:
|
|
return "agent"
|
|
default:
|
|
return "api"
|
|
}
|
|
}
|
|
return "api"
|
|
}
|
|
|
|
func monitorStringValue(value *string) string {
|
|
if value == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(*value)
|
|
}
|
|
|
|
func monitorLastSeenUnix(value time.Time) int64 {
|
|
// Zero means the resource has never actually been sighted (synthesized
|
|
// offline placeholders). Report 0 so the frontend renders "never" instead
|
|
// of fabricating a fresh sighting.
|
|
if value.IsZero() {
|
|
return 0
|
|
}
|
|
return value.UnixMilli()
|
|
}
|
|
|
|
// pollStorageBackupsWithNodes polls backups using a provided nodes list to avoid duplicate GetNodes calls
|
|
// Stop gracefully stops the monitor
|
|
// guestMetadataDrainTimeout bounds how long Stop waits for queued guest
|
|
// metadata writes. It matches tenantMonitorShutdownTimeout so a wedged store
|
|
// cannot hold up tenant teardown indefinitely.
|
|
const guestMetadataDrainTimeout = 2 * time.Second
|
|
|
|
func (m *Monitor) Stop() {
|
|
log.Info().Msg("stopping monitor")
|
|
|
|
if m.deadMan != nil {
|
|
m.deadMan.stop(time.Now().UTC(), m.alertManager)
|
|
}
|
|
|
|
// Stop the alert manager to save history
|
|
if m.alertManager != nil {
|
|
m.alertManager.Stop()
|
|
}
|
|
|
|
// Stop notification manager
|
|
if m.notificationMgr != nil {
|
|
m.notificationMgr.Stop()
|
|
}
|
|
|
|
// Drain background guest-metadata writes before the data directory can be
|
|
// torn down. Without this a queued write lands after shutdown and leaves a
|
|
// stray guest_metadata.json.tmp behind.
|
|
if m.guestMetadataStore != nil {
|
|
m.guestMetadataStore.WaitForPendingWrites(guestMetadataDrainTimeout)
|
|
}
|
|
|
|
// Close persistent metrics store (flushes buffered data)
|
|
if m.metricsStore != nil {
|
|
if err := m.metricsStore.Close(); err != nil {
|
|
log.Error().Err(err).Msg("failed to close metrics store")
|
|
} else {
|
|
log.Info().Msg("metrics store closed successfully")
|
|
}
|
|
}
|
|
|
|
log.Info().Msg("monitor stopped")
|
|
}
|
|
|
|
// recordAuthFailure records an authentication failure for a node
|