package monitoring import ( "context" "encoding/json" stderrors "errors" "fmt" "math" "math/rand" "os" "path/filepath" "reflect" "sort" "strings" "sync" "sync/atomic" "time" "github.com/rcourtman/pulse-go-rewrite/internal/ai/memory" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/discovery" "github.com/rcourtman/pulse-go-rewrite/internal/logging" "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors" "github.com/rcourtman/pulse-go-rewrite/internal/notifications" recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" "github.com/rcourtman/pulse-go-rewrite/internal/system" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/internal/websocket" "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory" "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" "github.com/rcourtman/pulse-go-rewrite/pkg/pbs" "github.com/rcourtman/pulse-go-rewrite/pkg/pmg" "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) const ( defaultTaskTimeout = 90 * time.Second minTaskTimeout = 30 * time.Second maxTaskTimeout = 3 * time.Minute ) const mockKeepRealPollingEnv = "PULSE_MOCK_KEEP_REAL_POLLING" func keepRealPollingInMockMode() bool { raw := strings.TrimSpace(strings.ToLower(os.Getenv(mockKeepRealPollingEnv))) switch raw { case "1", "true", "yes", "on": return true case "", "0", "false", "no", "off": return false default: return false } } // alertManagerRestoreOptions keeps mock mode a clean room across process // restarts. SetMockMode clears active alerts when the toggle flips, but a // process that boots with mock mode already enabled never runs that path, so // alerts raised against real infrastructure are restored from disk and served // beside fixture data until something clears them. func alertManagerRestoreOptions() []alerts.ManagerOption { if mock.IsMockEnabled() { return []alerts.ManagerOption{alerts.WithoutPersistedAlertRestore()} } return nil } // newProxmoxClientFunc is a variable that holds the function to create a new Proxmox client. // It is used to allow mocking the client creation in tests. var newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) { return proxmox.NewClient(cfg) } // PVEClientInterface defines the interface for PVE clients (both regular and cluster) type PVEClientInterface interface { GetNodes(ctx context.Context) ([]proxmox.Node, error) GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) GetContainers(ctx context.Context, node string) ([]proxmox.Container, error) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) GetAllStorage(ctx context.Context) ([]proxmox.Storage, error) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) GetTaskLog(ctx context.Context, node, upid string) ([]proxmox.TaskLogLine, error) GetReplicationStatus(ctx context.Context) ([]proxmox.ReplicationJob, error) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) GetVMSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) GetVMStatus(ctx context.Context, node string, vmid int) (*proxmox.VMStatus, error) GetContainerStatus(ctx context.Context, node string, vmid int) (*proxmox.Container, error) GetContainerConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error) GetContainerInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.ContainerInterface, error) GetClusterResources(ctx context.Context, resourceType string) ([]proxmox.ClusterResource, error) IsClusterMember(ctx context.Context) (bool, error) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error) GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error) GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) GetNodePendingUpdates(ctx context.Context, node string) ([]proxmox.AptPackage, error) GetCephStatus(ctx context.Context) (*proxmox.CephStatus, error) GetCephDF(ctx context.Context) (*proxmox.CephDF, error) } // ResourceStoreInterface provides methods for polling optimization and resource access. // When an agent is monitoring a node, we can reduce API polling for that node. type ResourceStoreInterface interface { // ShouldSkipAPIPolling returns true if API polling should be skipped for the hostname // because an agent is providing richer data. ShouldSkipAPIPolling(hostname string) bool // GetPollingRecommendations returns a map of hostname -> polling multiplier. // 0 = skip entirely, 0.5 = half frequency, 1 = normal GetPollingRecommendations() map[string]float64 // GetAll returns all resources in the store (for WebSocket broadcasts) GetAll() []unifiedresources.Resource // PopulateFromSnapshot updates the store with data from a StateSnapshot PopulateFromSnapshot(snapshot models.StateSnapshot) } // StaleThresholdResourceStore is implemented by stores that evaluate source // freshness while ingesting monitor snapshots. type StaleThresholdResourceStore interface { SetStaleThresholds(thresholds map[unifiedresources.DataSource]time.Duration) } // SupplementalRecordStore is an optional extension for resource stores that can // ingest source-native unified records in addition to legacy snapshots. type SupplementalRecordStore interface { PopulateSupplementalRecords(source unifiedresources.DataSource, records []unifiedresources.IngestRecord) } // AtomicSnapshotResourceStore is an optional extension for stores that can // atomically replace their canonical registry from a snapshot plus // supplemental records in a single swap. type AtomicSnapshotResourceStore interface { PopulateSnapshotAndSupplemental(snapshot models.StateSnapshot, recordsBySource map[unifiedresources.DataSource][]unifiedresources.IngestRecord) } // MetricsTargetResourceStore optionally resolves the history/metrics target for // a canonical resource in the live unified store. type MetricsTargetResourceStore interface { MetricsTargetForResource(resourceID string) *unifiedresources.MetricsTarget } // UnifiedResourceFreshnessStore is an optional extension for stores that track // their own canonical-resource freshness independent of state.LastUpdate. type UnifiedResourceFreshnessStore interface { UnifiedResourceFreshness() time.Time } // MonitorSupplementalRecordsProvider emits source-native records outside the // poll-provider scheduling path (for example, dedicated background pollers). type MonitorSupplementalRecordsProvider interface { SupplementalRecords(m *Monitor, orgID string) []unifiedresources.IngestRecord } // MonitorSupplementalChangesProvider optionally emits canonical resource // timeline changes alongside supplemental records. type MonitorSupplementalChangesProvider interface { SupplementalChanges(m *Monitor, orgID string) []unifiedresources.ResourceChange } // MonitorSupplementalInventoryReadinessProvider optionally reports when a // supplemental provider's current org-scoped inventory is settled enough to be // consumed by monitored-system grouping and support boundaries. // // Providers that suppress snapshot-owned sources must implement this contract // so the monitor can fail closed until the canonical store has been rebuilt // from a settled provider baseline. type MonitorSupplementalInventoryReadinessProvider interface { SupplementalInventoryReadyAt(m *Monitor, orgID string) (time.Time, bool) } // MonitorPhysicalDiskTemperatureHistoryProvider optionally exposes source-native // physical-disk temperature history through the canonical monitoring chart // boundary when Pulse's own stored history is shallow. type MonitorPhysicalDiskTemperatureHistoryProvider interface { PhysicalDiskTemperatureHistory(m *Monitor, orgID string, duration time.Duration) map[string][]MetricPoint } func getNodeDisplayName(instance *config.PVEInstance, nodeName string) string { baseName := strings.TrimSpace(nodeName) if baseName == "" { baseName = "unknown-node" } if instance == nil { return baseName } if instance.IsCluster { _, displayName := config.PVEClusterNodePresentation(instance, baseName) if displayName != "" { return displayName } return baseName } friendly := strings.TrimSpace(instance.Name) if friendly != "" { return friendly } if baseName != "" && baseName != "unknown-node" { return baseName } if label := normalizeEndpointHost(instance.Host); label != "" && !isLikelyIPAddress(label) { return label } return baseName } func (m *Monitor) getInstanceConfig(instanceName string) *config.PVEInstance { if m == nil { return nil } m.mu.RLock() defer m.mu.RUnlock() if m.config == nil { return nil } for i := range m.config.PVEInstances { if strings.EqualFold(m.config.PVEInstances[i].Name, instanceName) { instanceCopy := m.config.PVEInstances[i] return &instanceCopy } } return nil } func (m *Monitor) totalClientCount() int { if m == nil { return 0 } total := 0 for _, provider := range m.pollProviderSnapshotWithBuiltins() { if provider == nil { continue } total += len(provider.ListInstances(m)) } return total } func (m *Monitor) getPVEClient(name string) (PVEClientInterface, bool) { if m == nil { return nil, false } m.mu.RLock() defer m.mu.RUnlock() client, ok := m.pveClients[name] return client, ok } func (m *Monitor) getPBSClient(name string) (*pbs.Client, bool) { if m == nil { return nil, false } m.mu.RLock() defer m.mu.RUnlock() client, ok := m.pbsClients[name] return client, ok } func (m *Monitor) getPMGClient(name string) (*pmg.Client, bool) { if m == nil { return nil, false } m.mu.RLock() defer m.mu.RUnlock() client, ok := m.pmgClients[name] return client, ok } // normalizeSMARTDeviceIdentifier strips a trailing type annotation (e.g. // "sdd [scsi]" -> "sdd") and the /dev/ prefix so SMART temperatures match disks // by device path even when smartctl reports a typed label. func normalizeSMARTDeviceIdentifier(device string) string { normalized := strings.TrimSpace(device) if normalized == "" { return "" } if base, suffix, found := strings.Cut(normalized, " ["); found && strings.HasSuffix(suffix, "]") { normalized = base } normalized = strings.TrimSpace(normalized) return strings.TrimPrefix(normalized, "/dev/") } func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) []models.PhysicalDisk { if len(disks) == 0 || len(nodes) == 0 { return disks } // Build temperature maps by node for both SMART and legacy NVMe data smartTempsByNode := make(map[string][]models.DiskTemp) nvmeTempsByNode := make(map[string][]models.NVMeTemp) for _, node := range nodes { log.Debug(). Str("nodeName", node.Name). Bool("hasTemp", node.Temperature != nil). Bool("tempAvailable", node.Temperature != nil && node.Temperature.Available). Int("smartCount", func() int { if node.Temperature != nil { return len(node.Temperature.SMART) } return 0 }()). Msg("mergeNVMeTempsIntoDisks: checking node temperature") if node.Temperature == nil || !node.Temperature.Available { continue } // Collect SMART temps (preferred source) if len(node.Temperature.SMART) > 0 { temps := make([]models.DiskTemp, len(node.Temperature.SMART)) copy(temps, node.Temperature.SMART) smartTempsByNode[node.Name] = temps log.Debug(). Str("nodeName", node.Name). Int("smartTempCount", len(temps)). Msg("mergeNVMeTempsIntoDisks: collected SMART temps for node") } // Collect legacy NVMe temps as fallback if len(node.Temperature.NVMe) > 0 { temps := make([]models.NVMeTemp, len(node.Temperature.NVMe)) copy(temps, node.Temperature.NVMe) sort.Slice(temps, func(i, j int) bool { return temps[i].Device < temps[j].Device }) nvmeTempsByNode[node.Name] = temps } } if len(smartTempsByNode) == 0 && len(nvmeTempsByNode) == 0 { log.Debug(). Int("diskCount", len(disks)). Msg("mergeNVMeTempsIntoDisks: no SMART or NVMe temperature data available") return disks } log.Debug(). Int("smartNodeCount", len(smartTempsByNode)). Int("nvmeNodeCount", len(nvmeTempsByNode)). Int("diskCount", len(disks)). Msg("mergeNVMeTempsIntoDisks: starting disk temperature merge") updated := make([]models.PhysicalDisk, len(disks)) copy(updated, disks) // Process SMART temperatures first (preferred method) for i := range updated { smartTemps, ok := smartTempsByNode[updated[i].Node] log.Debug(). Str("diskDevPath", updated[i].DevPath). Str("diskNode", updated[i].Node). Bool("hasSMARTData", ok). Int("smartTempCount", len(smartTemps)). Msg("mergeNVMeTempsIntoDisks: checking disk for SMART temp match") if !ok || len(smartTemps) == 0 { continue } // Try to match by WWN (most reliable), but never guess when an // identity appears more than once in the node snapshot. if diskinventory.IsUsableHardwareID(updated[i].WWN) { if temp := uniqueDiskTemperatureMatch(smartTemps, func(candidate models.DiskTemp) bool { return diskinventory.IsUsableHardwareID(candidate.WWN) && strings.EqualFold(candidate.WWN, updated[i].WWN) }); temp != nil && temp.Temperature > 0 && !temp.StandbySkipped { setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart") log.Debug(). Str("disk", updated[i].DevPath). Str("wwn", updated[i].WWN). Int("temp", temp.Temperature). Msg("Matched SMART temperature by WWN") } } // Fall back to serial number match (case-insensitive) if diskinventory.IsUsableHardwareID(updated[i].Serial) && updated[i].Temperature == 0 { if temp := uniqueDiskTemperatureMatch(smartTemps, func(candidate models.DiskTemp) bool { return diskinventory.IsUsableHardwareID(candidate.Serial) && strings.EqualFold(candidate.Serial, updated[i].Serial) }); temp != nil && temp.Temperature > 0 && !temp.StandbySkipped { setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart") log.Debug(). Str("disk", updated[i].DevPath). Str("serial", updated[i].Serial). Int("temp", temp.Temperature). Msg("Matched SMART temperature by serial") } } // Last resort: match by device path (normalized) if updated[i].Temperature == 0 { normalizedDevPath := normalizeSMARTDeviceIdentifier(updated[i].DevPath) if temp := uniqueDiskTemperatureMatch(smartTemps, func(candidate models.DiskTemp) bool { normalizedTempDev := normalizeSMARTDeviceIdentifier(candidate.Device) return normalizedTempDev != "" && normalizedTempDev == normalizedDevPath }); temp != nil && temp.Temperature > 0 && !temp.StandbySkipped { setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart") log.Debug(). Str("disk", updated[i].DevPath). Int("temp", temp.Temperature). Msg("Matched SMART temperature by device path") } } } // Process legacy NVMe temperatures for disks that didn't get SMART data disksByNode := make(map[string][]int) for i := range updated { if strings.EqualFold(updated[i].Type, "nvme") && updated[i].Temperature == 0 { disksByNode[updated[i].Node] = append(disksByNode[updated[i].Node], i) } } for nodeName, diskIndexes := range disksByNode { temps, ok := nvmeTempsByNode[nodeName] if !ok || len(temps) == 0 { continue } sort.Slice(diskIndexes, func(i, j int) bool { return updated[diskIndexes[i]].DevPath < updated[diskIndexes[j]].DevPath }) for idx, diskIdx := range diskIndexes { if idx >= len(temps) { break } tempVal := temps[idx].Temp if tempVal <= 0 || math.IsNaN(tempVal) { continue } setPhysicalDiskTemperature(&updated[diskIdx], int(math.Round(tempVal)), "proxmox_node_nvme") log.Debug(). Str("disk", updated[diskIdx].DevPath). Int("temp", updated[diskIdx].Temperature). Msg("Matched legacy NVMe temperature by index") } } return updated } func uniqueDiskTemperatureMatch(temperatures []models.DiskTemp, predicate func(models.DiskTemp) bool) *models.DiskTemp { matchIndex := -1 for index := range temperatures { if !predicate(temperatures[index]) { continue } if matchIndex >= 0 { return nil } matchIndex = index } if matchIndex < 0 { return nil } return &temperatures[matchIndex] } func setPhysicalDiskTemperature(disk *models.PhysicalDisk, temperature int, source string) { if disk == nil || temperature <= 0 { return } disk.Temperature = temperature if disk.Collection == nil { disk.Collection = &diskinventory.CollectionStatus{} } disk.Collection.Temperature = diskinventory.Available(source) } // mergeHostAgentSMARTIntoDisks merges SMART temperature data from linked host agents // into physical disks for Proxmox nodes. This allows disk temps collected by the // pulse-agent running on a PVE node to populate the Physical Disks view. func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.Node, hosts []models.Host) []models.PhysicalDisk { if len(disks) == 0 || len(nodes) == 0 || len(hosts) == 0 { return disks } // Build a map of host ID to host for quick lookup hostByID := make(map[string]*models.Host, len(hosts)) for i := range hosts { hostByID[hosts[i].ID] = &hosts[i] } // Build a map of node name to linked host's SMART data smartByNodeName := make(map[string][]models.HostDiskSMART) for _, node := range nodes { if node.LinkedAgentID == "" { continue } host, ok := hostByID[node.LinkedAgentID] if !ok || len(host.Sensors.SMART) == 0 { continue } smartByNodeName[node.Name] = host.Sensors.SMART log.Debug(). Str("nodeName", node.Name). Str("hostAgentID", node.LinkedAgentID). Int("smartDiskCount", len(host.Sensors.SMART)). Msg("mergeHostAgentSMARTIntoDisks: found linked host agent with SMART data") } if len(smartByNodeName) == 0 { return disks } updated := make([]models.PhysicalDisk, len(disks)) copy(updated, disks) for i := range updated { smartData, ok := smartByNodeName[updated[i].Node] if !ok || len(smartData) == 0 { continue } // Find matching SMART entry by WWN, serial, or topology-scoped device path. var matched *models.HostDiskSMART uniqueMatch := func(predicate func(models.HostDiskSMART) bool) *models.HostDiskSMART { matchIndex := -1 for index := range smartData { if !predicate(smartData[index]) { continue } if matchIndex >= 0 { return nil } matchIndex = index } if matchIndex < 0 { return nil } return &smartData[matchIndex] } // Try to match by WWN (most reliable) if diskinventory.IsUsableHardwareID(updated[i].WWN) { matched = uniqueMatch(func(candidate models.HostDiskSMART) bool { return diskinventory.IsUsableHardwareID(candidate.WWN) && strings.EqualFold(candidate.WWN, updated[i].WWN) && diskTopologyCompatible(updated[i].Controller, updated[i].Target, candidate.Controller, candidate.Target) }) } // Fall back to serial number match if matched == nil && diskinventory.IsUsableHardwareID(updated[i].Serial) { matched = uniqueMatch(func(candidate models.HostDiskSMART) bool { return diskinventory.IsUsableHardwareID(candidate.Serial) && strings.EqualFold(candidate.Serial, updated[i].Serial) && diskTopologyCompatible(updated[i].Controller, updated[i].Target, candidate.Controller, candidate.Target) }) } // Last resort: match by device path if matched == nil { normalizedDevPath := normalizeSMARTDeviceIdentifier(updated[i].DevPath) matched = uniqueMatch(func(candidate models.HostDiskSMART) bool { normalizedDiskDev := normalizeSMARTDeviceIdentifier(candidate.Device) return normalizedDiskDev != "" && normalizedDiskDev == normalizedDevPath && diskTopologyCompatible(updated[i].Controller, updated[i].Target, candidate.Controller, candidate.Target) }) } if matched == nil { continue } if strings.TrimSpace(updated[i].Model) == "" && strings.TrimSpace(matched.Model) != "" { updated[i].Model = strings.TrimSpace(matched.Model) } if strings.TrimSpace(matched.Serial) != "" && (strings.TrimSpace(updated[i].Serial) == "" || shouldPromoteHostAgentSerial(updated[i], *matched)) { updated[i].Serial = strings.TrimSpace(matched.Serial) } if strings.TrimSpace(updated[i].WWN) == "" && strings.TrimSpace(matched.WWN) != "" { updated[i].WWN = strings.TrimSpace(matched.WWN) } if shouldPromoteHostAgentDiskType(updated[i].Type, matched.Type) { updated[i].Type = strings.TrimSpace(matched.Type) } if strings.TrimSpace(updated[i].Controller) == "" && strings.TrimSpace(matched.Controller) != "" { updated[i].Controller = strings.TrimSpace(matched.Controller) } if strings.TrimSpace(updated[i].Target) == "" && strings.TrimSpace(matched.Target) != "" { updated[i].Target = strings.TrimSpace(matched.Target) } if updated[i].Size <= 0 && matched.SizeBytes > 0 { updated[i].Size = matched.SizeBytes } if strings.TrimSpace(updated[i].StorageGroup) == "" && strings.TrimSpace(matched.Pool) != "" { updated[i].StorageGroup = strings.TrimSpace(matched.Pool) } // Merge temperature if not already set if !matched.Standby && updated[i].Temperature == 0 && matched.Temperature > 0 { updated[i].Temperature = matched.Temperature log.Debug(). Str("device", updated[i].DevPath). Int("temp", matched.Temperature). Msg("Matched host agent SMART temperature") } // Always merge SMART attributes from host agent if matched.Attributes != nil { updated[i].SmartAttributes = smartAttributesCopy(matched.Attributes) if derivedWearout := deriveWearoutFromSMARTAttributes(matched.Attributes); derivedWearout >= 0 { updated[i].Wearout = derivedWearout } } if matched.IO != nil { ioCopy := *matched.IO updated[i].IO = &ioCopy } updated[i].Collection = diskinventory.MergeStatus(updated[i].Collection, matched.Collection) if shouldUseHostAgentPhysicalDiskHealth(updated[i].Health, matched.Health) { updated[i].Health = matched.Health } } return updated } func shouldUseHostAgentPhysicalDiskHealth(existing, incoming string) bool { incoming = strings.ToUpper(strings.TrimSpace(incoming)) if incoming == "" || incoming == "UNKNOWN" { return false } existing = strings.ToUpper(strings.TrimSpace(existing)) if incoming == "FAILED" { return true } return existing == "" || existing == "UNKNOWN" } func diskTopologyCompatible(leftController, leftTarget, rightController, rightTarget string) bool { leftController = strings.TrimSpace(leftController) leftTarget = strings.TrimSpace(leftTarget) rightController = strings.TrimSpace(rightController) rightTarget = strings.TrimSpace(rightTarget) if leftController != "" && rightController != "" && !strings.EqualFold(leftController, rightController) { return false } if leftTarget != "" && rightTarget != "" && !strings.EqualFold(leftTarget, rightTarget) { return false } return true } func shouldPromoteHostAgentSerial(disk models.PhysicalDisk, smart models.HostDiskSMART) bool { if smart.Collection != nil && strings.EqualFold(strings.TrimSpace(smart.Type), "sas") && smart.Collection.Serial.State == diskinventory.FieldAvailable && strings.HasPrefix(strings.ToLower(strings.TrimSpace(smart.Collection.Serial.Source)), "smartctl") { return true } // Older agents did not carry collection provenance. Proxmox commonly puts // a 64-bit SAS address in its serial field; a non-address smartctl serial is // the actual drive identity and must replace that transport address. return strings.EqualFold(strings.TrimSpace(disk.Type), "sas") && strings.EqualFold(strings.TrimSpace(smart.Type), "sas") && looksLikeSASAddress(disk.Serial) && !looksLikeSASAddress(smart.Serial) } func looksLikeSASAddress(value string) bool { value = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(value)), "0x") if len(value) != 16 { return false } for _, char := range value { if (char < '0' || char > '9') && (char < 'a' || char > 'f') { return false } } return true } func shouldPromoteHostAgentDiskType(existing, incoming string) bool { incoming = strings.ToLower(strings.TrimSpace(incoming)) if incoming == "" { return false } existing = strings.ToLower(strings.TrimSpace(existing)) if existing == "" || existing == "unknown" || existing == "scsi" { return true } // Proxmox reports coarse form-factor types (hdd/ssd) or misreads SAS // members as sata; the agent's smartctl transport evidence is // authoritative for the link type. if incoming == "sas" && (existing == "sata" || existing == "hdd" || existing == "ssd") { return true } return false } func deriveWearoutFromSMARTAttributes(attrs *models.SMARTAttributes) int { if attrs == nil || attrs.PercentageUsed == nil { return -1 } return storagehealth.RemainingLifeFromPercentageUsed(*attrs.PercentageUsed) } func physicalDiskFromReadStateView(view *unifiedresources.PhysicalDiskView) models.PhysicalDisk { if view == nil { // Wearout 0 means no endurance remaining, so an absent view has to // carry the unreported sentinel rather than the struct zero value. return models.PhysicalDisk{Wearout: unifiedresources.WearoutUnreported} } return models.PhysicalDisk{ ID: view.ID(), Node: view.Node(), Instance: view.Instance(), DevPath: view.DevPath(), Model: view.Model(), Serial: view.Serial(), WWN: view.WWN(), Type: view.DiskType(), Controller: view.Controller(), Target: view.Target(), Size: view.SizeBytes(), Health: view.Health(), Wearout: view.Wearout(), Temperature: view.Temperature(), RPM: view.RPM(), Used: view.Used(), StorageGroup: view.StorageGroup(), SmartAttributes: smartAttributesFromUnifiedMeta(view.SMART()), IO: physicalDiskIOFromUnifiedMeta(view.IO()), Collection: diskinventory.CloneStatus(view.Collection()), LastChecked: view.LastSeen(), } } func physicalDiskIOFromUnifiedMeta(in *unifiedresources.PhysicalDiskIOMeta) *models.DiskIO { if in == nil { return nil } return &models.DiskIO{ Device: in.Device, ReadBytes: in.ReadBytes, WriteBytes: in.WriteBytes, ReadOps: in.ReadOps, WriteOps: in.WriteOps, ReadTime: in.ReadTimeMs, WriteTime: in.WriteTimeMs, IOTime: in.IOTimeMs, } } func smartAttributesFromUnifiedMeta(in *unifiedresources.SMARTMeta) *models.SMARTAttributes { if in == nil { return nil } out := &models.SMARTAttributes{ PowerOnHours: cloneInt64Pointer(in.PowerOnHours), PowerCycles: cloneInt64Pointer(in.PowerCycles), ReallocatedSectors: cloneInt64Pointer(in.ReallocatedSectors), PendingSectors: cloneInt64Pointer(in.PendingSectors), OfflineUncorrectable: cloneInt64Pointer(in.OfflineUncorrectable), UDMACRCErrors: cloneInt64Pointer(in.UDMACRCErrors), PercentageUsed: cloneIntPointer(in.PercentageUsed), AvailableSpare: cloneIntPointer(in.AvailableSpare), MediaErrors: cloneInt64Pointer(in.MediaErrors), UnsafeShutdowns: cloneInt64Pointer(in.UnsafeShutdowns), } if out.PowerOnHours == nil && out.PowerCycles == nil && out.ReallocatedSectors == nil && out.PendingSectors == nil && out.OfflineUncorrectable == nil && out.UDMACRCErrors == nil && out.PercentageUsed == nil && out.AvailableSpare == nil && out.MediaErrors == nil && out.UnsafeShutdowns == nil { return nil } return out } func cloneIntPointer(in *int) *int { if in == nil { return nil } value := *in return &value } func cloneInt64Pointer(in *int64) *int64 { if in == nil { return nil } value := *in return &value } func physicalDisksForInstanceFromReadState(readState unifiedresources.ReadState, instance string) []models.PhysicalDisk { if readState == nil { return nil } out := make([]models.PhysicalDisk, 0) for _, disk := range readState.PhysicalDisks() { if disk == nil || disk.Instance() != instance { continue } out = append(out, physicalDiskFromReadStateView(disk)) } return out } func nodesForInstanceFromReadState(readState unifiedresources.ReadState, instance string) []models.Node { if readState == nil { return nil } out := make([]models.Node, 0) for _, node := range readState.Nodes() { if node == nil || node.Instance() != instance { continue } out = append(out, nodeFromReadStateView(node)) } return out } func hostsFromReadState(readState unifiedresources.ReadState) []models.Host { if readState == nil { return nil } out := make([]models.Host, 0) for _, host := range readState.Hosts() { if host == nil { continue } out = append(out, hostFromReadStateView(host)) } return out } // writeSMARTMetrics writes SMART temperature history to the in-memory chart // buffer and persists SMART attributes when the metrics store is enabled. func (m *Monitor) writeSMARTMetrics(disk models.PhysicalDisk, now time.Time) { if shouldSkipNativeMockStateMetricWrites() { return } resourceID := unifiedresources.PhysicalDiskMetricID(disk) if resourceID == "" { return } // Temperature (always write if > 0) if disk.Temperature > 0 { if m.metricsHistory != nil { m.metricsHistory.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now) } if m.metricsStore != nil { m.metricsStore.Write("disk", resourceID, "smart_temp", float64(disk.Temperature), now) } } attrs := disk.SmartAttributes if attrs == nil || m.metricsStore == nil { return } // Common if attrs.PowerOnHours != nil { m.metricsStore.Write("disk", resourceID, "smart_power_on_hours", float64(*attrs.PowerOnHours), now) } if attrs.PowerCycles != nil { m.metricsStore.Write("disk", resourceID, "smart_power_cycles", float64(*attrs.PowerCycles), now) } // SATA-specific if attrs.ReallocatedSectors != nil { m.metricsStore.Write("disk", resourceID, "smart_reallocated_sectors", float64(*attrs.ReallocatedSectors), now) } if attrs.PendingSectors != nil { m.metricsStore.Write("disk", resourceID, "smart_pending_sectors", float64(*attrs.PendingSectors), now) } if attrs.OfflineUncorrectable != nil { m.metricsStore.Write("disk", resourceID, "smart_offline_uncorrectable", float64(*attrs.OfflineUncorrectable), now) } if attrs.UDMACRCErrors != nil { m.metricsStore.Write("disk", resourceID, "smart_crc_errors", float64(*attrs.UDMACRCErrors), now) } // NVMe-specific if attrs.PercentageUsed != nil { m.metricsStore.Write("disk", resourceID, "smart_percentage_used", float64(*attrs.PercentageUsed), now) } if attrs.AvailableSpare != nil { m.metricsStore.Write("disk", resourceID, "smart_available_spare", float64(*attrs.AvailableSpare), now) } if attrs.MediaErrors != nil { m.metricsStore.Write("disk", resourceID, "smart_media_errors", float64(*attrs.MediaErrors), now) } if attrs.UnsafeShutdowns != nil { m.metricsStore.Write("disk", resourceID, "smart_unsafe_shutdowns", float64(*attrs.UnsafeShutdowns), now) } } // PollExecutor defines the contract for executing polling tasks. type PollExecutor interface { Execute(ctx context.Context, task PollTask) } type realExecutor struct { monitor *Monitor } func newRealExecutor(m *Monitor) PollExecutor { return &realExecutor{monitor: m} } func (r *realExecutor) Execute(ctx context.Context, task PollTask) { if r == nil || r.monitor == nil { return } if task.Run != nil { task.Run(ctx) return } switch strings.ToLower(task.InstanceType) { case "pve": if task.PVEClient == nil { log.Warn(). Str("instance", task.InstanceName). Msg("PollExecutor received nil PVE client") return } r.monitor.pollPVEInstance(ctx, task.InstanceName, task.PVEClient) case "pbs": if task.PBSClient == nil { log.Warn(). Str("instance", task.InstanceName). Msg("PollExecutor received nil PBS client") return } r.monitor.pollPBSInstance(ctx, task.InstanceName, task.PBSClient) case "pmg": if task.PMGClient == nil { log.Warn(). Str("instance", task.InstanceName). Msg("PollExecutor received nil PMG client") return } r.monitor.pollPMGInstance(ctx, task.InstanceName, task.PMGClient) default: if logging.IsLevelEnabled(zerolog.DebugLevel) { log.Debug(). Str("instance", task.InstanceName). Str("type", task.InstanceType). Msg("PollExecutor received unsupported task type") } } } type instanceInfo struct { Key string Type InstanceType DisplayName string Connection string Metadata map[string]string } type pollStatus struct { LastSuccess time.Time LastErrorAt time.Time LastErrorMessage string LastErrorCategory string ConsecutiveFailures int FirstFailureAt time.Time } type dlqInsight struct { Reason string FirstAttempt time.Time LastAttempt time.Time RetryCount int NextRetry time.Time } type ErrorDetail struct { At time.Time `json:"at"` Message string `json:"message"` Category string `json:"category"` } type InstancePollStatus struct { LastSuccess *time.Time `json:"lastSuccess,omitempty"` LastError *ErrorDetail `json:"lastError,omitempty"` ConsecutiveFailures int `json:"consecutiveFailures"` FirstFailureAt *time.Time `json:"firstFailureAt,omitempty"` } type InstanceBreaker struct { State string `json:"state"` Since *time.Time `json:"since,omitempty"` LastTransition *time.Time `json:"lastTransition,omitempty"` RetryAt *time.Time `json:"retryAt,omitempty"` FailureCount int `json:"failureCount"` } type InstanceDLQ struct { Present bool `json:"present"` Reason string `json:"reason,omitempty"` FirstAttempt *time.Time `json:"firstAttempt,omitempty"` LastAttempt *time.Time `json:"lastAttempt,omitempty"` RetryCount int `json:"retryCount,omitempty"` NextRetry *time.Time `json:"nextRetry,omitempty"` } type InstanceHealth struct { Key string `json:"key"` Type string `json:"type"` DisplayName string `json:"displayName"` Instance string `json:"instance"` Connection string `json:"connection"` PollStatus InstancePollStatus `json:"pollStatus"` Breaker InstanceBreaker `json:"breaker"` DeadLetter InstanceDLQ `json:"deadLetter"` Warnings []string `json:"warnings"` } func (h InstanceHealth) NormalizeCollections() InstanceHealth { if h.Warnings == nil { h.Warnings = []string{} } return h } // Monitor handles all monitoring operations type Monitor struct { config *config.Config state *models.State orgID string // Organization ID for tenant isolation (empty = default/legacy) mockUnifiedViewMu sync.Mutex mockUnifiedView monitorUnifiedStateView mockUnifiedViewVersion uint64 mockUnifiedViewValid bool pveClients map[string]PVEClientInterface pbsClients map[string]*pbs.Client pmgClients map[string]*pmg.Client availabilityStatuses map[string]AvailabilityProbeStatus availabilityProbeTrackers map[string]availabilityProbeAssignmentTracker pollProviders map[InstanceType]PollProvider pollMetrics *PollMetrics scheduler *AdaptiveScheduler stalenessTracker *StalenessTracker taskQueue *TaskQueue pollTimeout time.Duration circuitBreakers map[string]*circuitBreaker deadLetterQueue *TaskQueue failureCounts map[string]int lastOutcome map[string]taskOutcome backoffCfg backoffConfig rng *rand.Rand maxRetryAttempts int tempCollector *TemperatureCollector // SSH-based temperature collector guestMetadataStore *config.GuestMetadataStore dockerMetadataStore *config.DockerMetadataStore hostMetadataStore *config.HostMetadataStore hostContinuityStore *config.HostContinuityStore hostAgentLifecycleMu sync.RWMutex mu sync.RWMutex startTime time.Time rateTracker *RateTracker metricsHistory *MetricsHistory metricsStore *metrics.Store // Persistent SQLite metrics storage alertManager *alerts.Manager alertResolvedAICallback func(*alerts.Alert) alertTriggeredAICallback func(*alerts.Alert) alertPushCallback func(*alerts.Alert) connectionsSnapshotLister func() []alerts.ConnectionSnapshot // returns platform connection snapshots for the connection-degraded check incidentStore *memory.IncidentStore notificationMgr *notifications.NotificationManager configPersist *config.ConfigPersistence discoveryService *discovery.Service // Background discovery service activePollCount int32 // Number of active polling operations pollCounter int64 // Counter for polling cycles authFailures map[string]int // Track consecutive auth failures per node lastAuthAttempt map[string]time.Time // Track last auth attempt time lastClusterCheck map[string]time.Time // Track last cluster check for standalone nodes lastPhysicalDiskPoll map[string]time.Time // Track last physical disk poll time per instance lastPVEBackupPoll map[string]time.Time // Track last PVE backup poll per instance vzdumpJobTaskCache map[string]vzdumpJobTaskCacheEntry // Cache synthesized per-guest tasks for multi-guest vzdump job runs, keyed by instance|UPID lastPBSBackupPoll map[string]time.Time // Track last PBS backup poll per instance pveBackupInventoryReady map[string]map[string]bool // Track PVE guest inventory readiness for backup orphan detection pveBackupTemplateSubjects map[string]map[string]struct{} // Track template VMIDs excluded from runtime workloads but valid for backups backupPermissionWarnings map[string]string // Track backup permission issues per instance (instance -> warning message) persistence *config.ConfigPersistence // Add persistence for saving updated configs pbsBackupPollers map[string]bool // Track PBS backup polling goroutines per instance pbsBackupCacheTime map[string]map[pbsBackupGroupKey]time.Time // Track when each PBS backup group was last fetched runtimePollingMu sync.RWMutex // Guards runtimePollingOverride; polling goroutines read it every cycle runtimePollingOverride runtimePollingOverrides // Runtime polling-cadence overrides pushed by the settings API (#1619) runtimeCtx context.Context // Context used while monitor is running wsHub *websocket.Hub // Hub used for broadcasting state diagMu sync.RWMutex // Protects diagnostic snapshot maps nodeSnapshots map[string]NodeMemorySnapshot guestSnapshots map[string]GuestMemorySnapshot rrdCacheMu sync.RWMutex // Protects short-lived guest memory caches. nodeRRDMemCache map[string]rrdMemCacheEntry vmAgentMemCache map[string]agentMemCacheEntry removedDockerHosts map[string]time.Time // Track deliberately removed Docker hosts (ID -> removal time) dockerTokenBindings map[string]string // Track token ID -> Docker host identity bindings to enforce uniqueness dockerIdentityFlaps map[string]*identityFlapTracker // Track per-host identity flapping (cloned VMs sharing machine-id) removedKubernetesClusters map[string]time.Time // Track deliberately removed Kubernetes clusters (ID -> removal time) kubernetesTokenBindings map[string]string // Track token ID -> agent ID bindings to enforce uniqueness removedHostAgents map[string]time.Time // Track deliberately removed host agents (ID -> removal time) hostTokenBindings map[string]string // Track tokenID:hostname -> host identity bindings hostIdentityFlaps map[string]*identityFlapTracker // Track per-host-agent identity flapping (cloned machines sharing machine-id) hostReportApplyLocksMu sync.Mutex hostReportApplyLocks map[string]*hostReportApplyLock hostReportOrderMu sync.Mutex hostReportOrders map[string]hostReportOrder dockerCommands map[string]*dockerHostCommand dockerCommandIndex map[string]string guestMetadataMu sync.RWMutex guestMetadataCache map[string]guestMetadataCacheEntry guestMetadataLimiterMu sync.Mutex guestMetadataLimiter map[string]time.Time guestMetadataSlots chan struct{} guestMetadataMinRefresh time.Duration guestMetadataRefreshJitter time.Duration guestMetadataRetryBackoff time.Duration guestMetadataHoldDuration time.Duration guestAgentWorkSlots chan struct{} // Configurable guest agent timeouts (refs #592) guestAgentFSInfoTimeout time.Duration guestAgentNetworkTimeout time.Duration guestAgentOSInfoTimeout time.Duration guestAgentVersionTimeout time.Duration guestAgentRetries int guestAgentVMBudget time.Duration executor PollExecutor breakerBaseRetry time.Duration breakerMaxDelay time.Duration breakerHalfOpenWindow time.Duration instanceInfoCache map[string]*instanceInfo pollStatusMap map[string]*pollStatus dlqInsightMap map[string]*dlqInsight nodeLastOnline map[string]time.Time // Track last time each node was seen online (for grace period) nodePendingUpdatesCache map[string]pendingUpdatesCache // Cache pending updates per node (checked every 30 min) pveMembershipMisses map[string]int // Consecutive authoritative membership absences by instance/node resourceStore ResourceStoreInterface // Optional unified resource store for polling optimization supplementalProviders map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider recoveryManager *recoverymanager.Manager // Optional recovery store manager for backup rollups recoveryIngestMu sync.Mutex recoveryIngestRunning bool recoveryIngestPending []recoveryIngestBatch mockMetricsCancel context.CancelFunc mockMetricsWg sync.WaitGroup dockerChecker DockerChecker // Optional Docker checker for LXC containers dockerCheckerConfiguredAt time.Time // Last time the Docker checker was configured dockerInventoryCollector DockerInventoryCollector // Optional Docker inventory collector for LXC containers // Agent profile cache to avoid disk I/O on every report (refs #1094) agentProfileCacheMu sync.RWMutex agentProfileCache *agentProfileCacheEntry // Proxmox LXC filesystem cache: bounded pct df readings reported by an // agent securely linked to the node that owns each running container. proxmoxLXCFilesystemsMu sync.RWMutex proxmoxLXCFilesystemsCache map[string]agentLXCFilesystemCacheEntry // Cluster sensor cache: temperature data collected by an agent on one Proxmox // cluster node via SSH to its siblings. Keyed by lowercase node name. clusterSensorsMu sync.RWMutex clusterSensorsCache map[string]clusterSensorsCacheEntry mockChartCacheMu sync.RWMutex mockChartMapCache map[mockChartMetricMapCacheKey]map[string][]MetricPoint // Commercial feature gate for monitoring-owned Pro behaviour (external // availability probes). Guarded by its own mutex so read-time derivations // that already hold m.mu can consult it without lock re-entrancy. licenseCheckerMu sync.RWMutex licenseChecker func(feature string) bool } func (m *Monitor) setRuntimeContext(ctx context.Context, hub *websocket.Hub) { m.mu.Lock() defer m.mu.Unlock() m.runtimeCtx = ctx m.wsHub = hub } func (m *Monitor) getRuntimeContext() context.Context { m.mu.RLock() defer m.mu.RUnlock() return m.runtimeCtx } // clusterSensorsCacheEntry stores temperature data collected by a sibling agent via SSH. type clusterSensorsCacheEntry struct { sensors models.HostSensorSummary updatedAt time.Time } type rrdMemCacheEntry struct { available uint64 used uint64 total uint64 hasAvail bool hasUsed bool hasTotal bool netIn float64 netOut float64 hasNetIn bool hasNetOut bool fetchedAt time.Time } // pendingUpdatesCache caches apt pending updates count per node type pendingUpdatesCache struct { count int checkedAt time.Time } // TTL for pending updates cache (30 minutes - balance between freshness and API load) const pendingUpdatesCacheTTL = 30 * time.Minute // agentProfileCacheEntry caches agent profiles and assignments to avoid disk I/O on every agent report. // TTL is 60 seconds to balance freshness with performance. type agentProfileCacheEntry struct { profiles []models.AgentProfile assignments []models.AgentProfileAssignment loadedAt time.Time } const agentProfileCacheTTL = 60 * time.Second // shouldRunBackupPoll determines whether a backup polling cycle should execute. // Returns whether polling should run, a human-readable skip reason, and the timestamp to record. func (m *Monitor) shouldRunBackupPoll(last time.Time, now time.Time) (bool, string, time.Time) { if m == nil || m.config == nil { return false, "configuration unavailable", last } if !m.backupPollingEnabledSetting() { return false, "backup polling globally disabled", last } interval := m.backupPollingIntervalSetting() if interval > 0 { if !last.IsZero() && now.Sub(last) < interval { next := last.Add(interval) return false, fmt.Sprintf("next run scheduled for %s", next.Format(time.RFC3339)), last } return true, "", now } backupCycles := m.config.BackupPollingCycles if backupCycles <= 0 { backupCycles = 10 } if m.pollCounter%int64(backupCycles) == 0 || m.pollCounter == 1 { return true, "", now } remaining := int64(backupCycles) - (m.pollCounter % int64(backupCycles)) return false, fmt.Sprintf("next run in %d polling cycles", remaining), last } const ( dockerConnectionPrefix = "docker-" kubernetesConnectionPrefix = "kubernetes-" hostConnectionPrefix = "host-" dockerOfflineGraceMultiplier = 4 dockerMinimumHealthWindow = 30 * time.Second dockerMaximumHealthWindow = 10 * time.Minute kubernetesOfflineGraceMultiplier = 4 kubernetesMinimumHealthWindow = 30 * time.Second kubernetesMaximumHealthWindow = 10 * time.Minute hostOfflineGraceMultiplier = 6 hostMinimumHealthWindow = 60 * time.Second hostMaximumHealthWindow = 10 * time.Minute nodeOfflineGracePeriod = 60 * time.Second // Grace period before marking Proxmox nodes offline nodeRRDCacheTTL = 30 * time.Second nodeRRDRequestTimeout = 2 * time.Second ) type taskOutcome struct { success bool transient bool err error recordedAt time.Time } func nodeRRDCacheKey(instanceName, nodeName string) string { return instanceName + "/" + nodeName } func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterface, instanceName, nodeName string) (rrdMemCacheEntry, error) { if client == nil || nodeName == "" { return rrdMemCacheEntry{}, fmt.Errorf("invalid arguments for RRD lookup") } now := time.Now() cacheKey := nodeRRDCacheKey(instanceName, nodeName) m.rrdCacheMu.RLock() if entry, ok := m.nodeRRDMemCache[cacheKey]; ok && now.Sub(entry.fetchedAt) < nodeRRDCacheTTL { m.rrdCacheMu.RUnlock() return entry, nil } m.rrdCacheMu.RUnlock() requestCtx, cancel := context.WithTimeout(ctx, nodeRRDRequestTimeout) defer cancel() points, err := client.GetNodeRRDData(requestCtx, nodeName, "hour", "AVERAGE", []string{"memavailable", "memused", "memtotal", "netin", "netout"}) if err != nil { return rrdMemCacheEntry{}, err } var memAvailable uint64 var memUsed uint64 var memTotal uint64 var hasAvail bool var hasUsed bool var hasTotal bool var netIn float64 var netOut float64 var hasNetIn bool var hasNetOut bool for i := len(points) - 1; i >= 0; i-- { point := points[i] if !hasTotal && point.MemTotal != nil && !math.IsNaN(*point.MemTotal) && !math.IsInf(*point.MemTotal, 0) && *point.MemTotal > 0 && *point.MemTotal <= math.MaxUint64 { memTotal = uint64(math.Round(*point.MemTotal)) hasTotal = true } if !hasAvail && point.MemAvailable != nil && !math.IsNaN(*point.MemAvailable) && !math.IsInf(*point.MemAvailable, 0) && *point.MemAvailable >= 0 && *point.MemAvailable <= math.MaxUint64 { memAvailable = uint64(math.Round(*point.MemAvailable)) hasAvail = true } if !hasUsed && point.MemUsed != nil && !math.IsNaN(*point.MemUsed) && !math.IsInf(*point.MemUsed, 0) && *point.MemUsed >= 0 && *point.MemUsed <= math.MaxUint64 { memUsed = uint64(math.Round(*point.MemUsed)) hasUsed = true } if !hasNetIn && point.NetIn != nil && !math.IsNaN(*point.NetIn) && !math.IsInf(*point.NetIn, 0) { netIn = *point.NetIn hasNetIn = true } if !hasNetOut && point.NetOut != nil && !math.IsNaN(*point.NetOut) && !math.IsInf(*point.NetOut, 0) { netOut = *point.NetOut hasNetOut = true } } if memTotal > 0 { if memAvailable > memTotal { memAvailable = 0 hasAvail = false } if memUsed > memTotal { memUsed = 0 hasUsed = false } } if !hasAvail && !hasUsed && !hasNetIn && !hasNetOut { return rrdMemCacheEntry{}, fmt.Errorf("rrd node metrics not present") } entry := rrdMemCacheEntry{ available: memAvailable, used: memUsed, total: memTotal, hasAvail: hasAvail, hasUsed: hasUsed, hasTotal: hasTotal, netIn: netIn, netOut: netOut, hasNetIn: hasNetIn, hasNetOut: hasNetOut, fetchedAt: now, } m.rrdCacheMu.Lock() if m.nodeRRDMemCache == nil { m.nodeRRDMemCache = make(map[string]rrdMemCacheEntry) } m.nodeRRDMemCache[cacheKey] = entry m.rrdCacheMu.Unlock() return entry, nil } // RemoveDockerHost removes a docker host from the shared state and clears related alerts. func (m *Monitor) GetConnectionStatuses() map[string]bool { if m == nil { return map[string]bool{} } if mock.IsMockEnabled() { statuses := make(map[string]bool) state := mock.CurrentFixtureGraph().State for _, node := range state.Nodes { key := "pve-" + node.Name statuses[key] = strings.ToLower(node.Status) == "online" if node.Host != "" { statuses[node.Host] = strings.ToLower(node.Status) == "online" } } for _, pbsInst := range state.PBSInstances { key := "pbs-" + pbsInst.Name statuses[key] = strings.ToLower(pbsInst.Status) != "offline" if pbsInst.Host != "" { statuses[pbsInst.Host] = strings.ToLower(pbsInst.Status) != "offline" } } for _, pmgInst := range state.PMGInstances { key := "pmg-" + pmgInst.Name statuses[key] = strings.ToLower(pmgInst.Status) != "offline" if pmgInst.Host != "" { statuses[pmgInst.Host] = strings.ToLower(pmgInst.Status) != "offline" } } for _, dockerHost := range state.DockerHosts { key := dockerConnectionPrefix + dockerHost.ID statuses[key] = strings.ToLower(dockerHost.Status) == "online" } return statuses } statuses := make(map[string]bool) for _, provider := range m.pollProviderSnapshotWithBuiltins() { for key, connected := range m.providerConnectionStatuses(provider) { if strings.TrimSpace(key) == "" { continue } statuses[key] = connected } } return statuses } // checkContainerizedTempMonitoring logs a security warning if Pulse is running // in a container with SSH-based temperature monitoring enabled func checkContainerizedTempMonitoring() { // Check if running in container isContainer := os.Getenv("PULSE_DOCKER") == "true" || system.InContainer() if !isContainer { return } // Check if SSH keys exist (indicates temperature monitoring is configured) homeDir := os.Getenv("HOME") if homeDir == "" { homeDir = "/home/pulse" } sshKeyPath := homeDir + "/.ssh/id_ed25519" if _, err := os.Stat(sshKeyPath); err != nil { // No SSH key found, temperature monitoring not configured return } // Log warning log.Warn(). Msg("SECURITY NOTICE: Pulse is running in a container with SSH-based temperature monitoring enabled. " + "SSH private keys are stored inside the container, which could be a security risk if the container is compromised. " + "Future versions will use agent-based architecture for better security. " + "See documentation for hardening recommendations.") } // New creates a new Monitor instance func New(cfg *config.Config) (*Monitor, error) { if cfg == nil { return nil, fmt.Errorf("config cannot be nil") } hostContinuityStore := config.NewHostContinuityStore(cfg.DataPath, nil) if err := hostContinuityStore.LoadError(); err != nil { return nil, fmt.Errorf("load host continuity lifecycle journal: %w", err) } // Initialize temperature collector with sensors SSH key // Will use root user for now - can be made configurable later homeDir := os.Getenv("HOME") if homeDir == "" { homeDir = "/home/pulse" } sshKeyPath := filepath.Join(homeDir, ".ssh/id_ed25519_sensors") tempCollector := NewTemperatureCollectorWithPort("root", sshKeyPath, cfg.SSHPort) // Security warning if running in container with SSH temperature monitoring checkContainerizedTempMonitoring() stalenessTracker := NewStalenessTracker(getPollMetrics()) stalenessTracker.SetBounds(cfg.AdaptivePollingBaseInterval, cfg.AdaptivePollingMaxInterval) taskQueue := NewTaskQueue() deadLetterQueue := NewTaskQueue() breakers := make(map[string]*circuitBreaker) failureCounts := make(map[string]int) lastOutcome := make(map[string]taskOutcome) backoff := backoffConfig{ Initial: 5 * time.Second, Multiplier: 2, Jitter: 0.2, Max: 5 * time.Minute, } if cfg.AdaptivePollingEnabled && cfg.AdaptivePollingMaxInterval > 0 && cfg.AdaptivePollingMaxInterval <= 15*time.Second { backoff.Initial = 750 * time.Millisecond backoff.Max = 6 * time.Second } var scheduler *AdaptiveScheduler if cfg.AdaptivePollingEnabled { scheduler = NewAdaptiveScheduler(SchedulerConfig{ BaseInterval: cfg.AdaptivePollingBaseInterval, MinInterval: cfg.AdaptivePollingMinInterval, MaxInterval: cfg.AdaptivePollingMaxInterval, }, stalenessTracker, nil, nil) } minRefresh := cfg.GuestMetadataMinRefreshInterval if minRefresh <= 0 { minRefresh = config.DefaultGuestMetadataMinRefresh } jitter := cfg.GuestMetadataRefreshJitter if jitter < 0 { jitter = 0 } retryBackoff := cfg.GuestMetadataRetryBackoff if retryBackoff <= 0 { retryBackoff = config.DefaultGuestMetadataRetryBackoff } concurrency := cfg.GuestMetadataMaxConcurrent if concurrency <= 0 { concurrency = config.DefaultGuestMetadataMaxConcurrent } holdDuration := defaultGuestMetadataHold // Load guest agent timeout configuration from environment variables (refs #592) guestAgentFSInfoTimeout := parsePositiveDurationEnv("GUEST_AGENT_FSINFO_TIMEOUT", defaultGuestAgentFSInfoTimeout) guestAgentNetworkTimeout := parsePositiveDurationEnv("GUEST_AGENT_NETWORK_TIMEOUT", defaultGuestAgentNetworkTimeout) guestAgentOSInfoTimeout := parsePositiveDurationEnv("GUEST_AGENT_OSINFO_TIMEOUT", defaultGuestAgentOSInfoTimeout) guestAgentVersionTimeout := parsePositiveDurationEnv("GUEST_AGENT_VERSION_TIMEOUT", defaultGuestAgentVersionTimeout) guestAgentRetries := parseNonNegativeIntEnv("GUEST_AGENT_RETRIES", defaultGuestAgentRetries) guestAgentVMBudget := parseDurationEnv("GUEST_AGENT_VM_BUDGET", 0) guestAgentVMMaxConcurrent := parseNonNegativeIntEnv("GUEST_AGENT_VM_MAX_CONCURRENT", defaultGuestAgentVMMaxConcurrent) // Initialize persistent metrics store (SQLite) with configurable retention var metricsStore *metrics.Store metricsStoreConfig := metrics.DefaultConfig(cfg.DataPath) if strings.TrimSpace(cfg.MetricsDBPath) != "" { metricsStoreConfig.DBPath = cfg.MetricsDBPath } if cfg.MetricsRollupInterval > 0 { metricsStoreConfig.RollupInterval = cfg.MetricsRollupInterval } // Override retention settings from config (allows tier-based pricing in future) if cfg.MetricsRetentionRawHours > 0 { metricsStoreConfig.RetentionRaw = time.Duration(cfg.MetricsRetentionRawHours) * time.Hour } if cfg.MetricsRetentionMinuteHours > 0 { metricsStoreConfig.RetentionMinute = time.Duration(cfg.MetricsRetentionMinuteHours) * time.Hour } if cfg.MetricsRetentionHourlyDays > 0 { metricsStoreConfig.RetentionHourly = time.Duration(cfg.MetricsRetentionHourlyDays) * 24 * time.Hour } if cfg.MetricsRetentionDailyDays > 0 { metricsStoreConfig.RetentionDaily = time.Duration(cfg.MetricsRetentionDailyDays) * 24 * time.Hour } // In mock mode, extend hourly/daily retention to 90 days so seeded mock // backfill (daily tier 30d, hourly tier 7d — see mockStoreSeedPlans) // outlives its windows. Raw and minute tiers keep production defaults — // the minute seed window (24h) matches the default retention, and short // raw retention keeps live mock ticks from bloating the DB (the old // 90-day raw retention caused metrics.db to grow to ~2 GB). if mock.IsMockEnabled() { metricsStoreConfig.WriteBufferSize = 2000 metricsStoreConfig.RetentionHourly = 90 * 24 * time.Hour metricsStoreConfig.RetentionDaily = 90 * 24 * time.Hour } ms, err := metrics.NewStore(metricsStoreConfig) if err != nil { // Do not automatically delete the DB on error, as it causes data loss on transient errors (e.g. locks). // If the DB is truly corrupted, the user should manually remove it. log.Error().Err(err).Msg("failed to initialize persistent metrics store - continuing without metrics persistence") } else { if mock.IsMockEnabled() { ms.SetMaxOpenConns(10) } metricsStore = ms log.Info(). Str("path", metricsStoreConfig.DBPath). Dur("retentionRaw", metricsStoreConfig.RetentionRaw). Dur("retentionMinute", metricsStoreConfig.RetentionMinute). Dur("retentionHourly", metricsStoreConfig.RetentionHourly). Dur("retentionDaily", metricsStoreConfig.RetentionDaily). Msg("Persistent metrics store initialized with configurable retention") } incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{ DataDir: cfg.DataPath, }) m := &Monitor{ config: cfg, state: models.NewState(), pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), pmgClients: make(map[string]*pmg.Client), availabilityStatuses: make(map[string]AvailabilityProbeStatus), availabilityProbeTrackers: make(map[string]availabilityProbeAssignmentTracker), pollProviders: make(map[InstanceType]PollProvider), pollMetrics: getPollMetrics(), scheduler: scheduler, stalenessTracker: stalenessTracker, taskQueue: taskQueue, pollTimeout: derivePollTimeout(cfg), deadLetterQueue: deadLetterQueue, circuitBreakers: breakers, failureCounts: failureCounts, lastOutcome: lastOutcome, backoffCfg: backoff, rng: rand.New(rand.NewSource(time.Now().UnixNano())), maxRetryAttempts: 5, tempCollector: tempCollector, guestMetadataStore: config.NewGuestMetadataStore(cfg.DataPath, nil), dockerMetadataStore: config.NewDockerMetadataStore(cfg.DataPath, nil), hostMetadataStore: config.NewHostMetadataStore(cfg.DataPath, nil), hostContinuityStore: hostContinuityStore, startTime: time.Now(), rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points (~8h @ 30s) metricsStore: metricsStore, // Persistent SQLite storage alertManager: alerts.NewManagerWithDataDir(cfg.DataPath, alertManagerRestoreOptions()...), incidentStore: incidentStore, notificationMgr: notifications.NewNotificationManagerWithDataDir(cfg.PublicURL, cfg.DataPath), configPersist: config.NewConfigPersistence(cfg.DataPath), discoveryService: nil, // Will be initialized in Start() authFailures: make(map[string]int), lastAuthAttempt: make(map[string]time.Time), lastClusterCheck: make(map[string]time.Time), lastPhysicalDiskPoll: make(map[string]time.Time), lastPVEBackupPoll: make(map[string]time.Time), vzdumpJobTaskCache: make(map[string]vzdumpJobTaskCacheEntry), lastPBSBackupPoll: make(map[string]time.Time), pveBackupInventoryReady: make(map[string]map[string]bool), pveBackupTemplateSubjects: make(map[string]map[string]struct{}), backupPermissionWarnings: make(map[string]string), persistence: config.NewConfigPersistence(cfg.DataPath), pbsBackupPollers: make(map[string]bool), pbsBackupCacheTime: make(map[string]map[pbsBackupGroupKey]time.Time), nodeSnapshots: make(map[string]NodeMemorySnapshot), guestSnapshots: make(map[string]GuestMemorySnapshot), nodeRRDMemCache: make(map[string]rrdMemCacheEntry), vmAgentMemCache: make(map[string]agentMemCacheEntry), removedDockerHosts: make(map[string]time.Time), dockerTokenBindings: make(map[string]string), dockerIdentityFlaps: make(map[string]*identityFlapTracker), removedKubernetesClusters: make(map[string]time.Time), kubernetesTokenBindings: make(map[string]string), removedHostAgents: make(map[string]time.Time), hostTokenBindings: make(map[string]string), hostIdentityFlaps: make(map[string]*identityFlapTracker), hostReportApplyLocks: make(map[string]*hostReportApplyLock), hostReportOrders: make(map[string]hostReportOrder), clusterSensorsCache: make(map[string]clusterSensorsCacheEntry), proxmoxLXCFilesystemsCache: make(map[string]agentLXCFilesystemCacheEntry), dockerCommands: make(map[string]*dockerHostCommand), dockerCommandIndex: make(map[string]string), guestMetadataCache: make(map[string]guestMetadataCacheEntry), guestMetadataLimiter: make(map[string]time.Time), guestMetadataMinRefresh: minRefresh, guestMetadataRefreshJitter: jitter, guestMetadataRetryBackoff: retryBackoff, guestMetadataHoldDuration: holdDuration, guestAgentFSInfoTimeout: guestAgentFSInfoTimeout, guestAgentNetworkTimeout: guestAgentNetworkTimeout, guestAgentOSInfoTimeout: guestAgentOSInfoTimeout, guestAgentVersionTimeout: guestAgentVersionTimeout, guestAgentRetries: guestAgentRetries, guestAgentVMBudget: guestAgentVMBudget, instanceInfoCache: make(map[string]*instanceInfo), pollStatusMap: make(map[string]*pollStatus), dlqInsightMap: make(map[string]*dlqInsight), nodeLastOnline: make(map[string]time.Time), nodePendingUpdatesCache: make(map[string]pendingUpdatesCache), pveMembershipMisses: make(map[string]int), supplementalProviders: make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider), } m.breakerBaseRetry = 5 * time.Second m.breakerMaxDelay = 5 * time.Minute m.breakerHalfOpenWindow = 30 * time.Second if cfg.AdaptivePollingEnabled && cfg.AdaptivePollingMaxInterval > 0 && cfg.AdaptivePollingMaxInterval <= 15*time.Second { m.breakerBaseRetry = 2 * time.Second m.breakerMaxDelay = 10 * time.Second m.breakerHalfOpenWindow = 2 * time.Second } m.executor = newRealExecutor(m) m.hydrateRemovedHostAgents(time.Now().UTC()) m.alertManager.SetBackupIntentContextResolver(m.resolveBackupIntentContext) m.registerBuiltInPollProviders() m.buildInstanceInfoCache(cfg) // Initialize state with config values m.state.TemperatureMonitoringEnabled = cfg.TemperatureMonitoringEnabled if m.pollMetrics != nil { m.pollMetrics.ResetQueueDepth(0) } // Load saved configurations if alertConfig, err := m.configPersist.LoadAlertConfig(); err == nil { m.alertManager.UpdateConfig(*alertConfig) // Apply schedule settings to notification manager m.notificationMgr.SetEnabled(alertConfig.Enabled && alertConfig.ActivationState == alerts.ActivationActive) m.notificationMgr.SetCooldown(alertConfig.Schedule.Cooldown) m.notificationMgr.SetGroupingConfig( alertConfig.Schedule.Grouping.Enabled, alertConfig.Schedule.Grouping.Window, alertConfig.Schedule.Grouping.ByNode, alertConfig.Schedule.Grouping.ByGuest, ) m.notificationMgr.SetInitialNotifyTarget(alertConfig.Schedule.InitialNotify) m.notificationMgr.SetNotifyOnResolve(alertConfig.Schedule.NotifyOnResolve) } else { log.Warn().Err(err).Msg("failed to load alert configuration") } if intentPolicies, err := m.configPersist.LoadAlertIntentPolicies(); err == nil { if err := m.alertManager.LoadIntentPolicies(*intentPolicies); err != nil { log.Warn().Err(err).Msg("failed to install alert intent policies") } } else { log.Warn().Err(err).Msg("failed to load alert intent policies") } if emailConfig, err := m.configPersist.LoadEmailConfig(); err == nil { m.notificationMgr.SetEmailConfig(*emailConfig) } else { log.Warn().Err(err).Msg("failed to load email configuration") } if concurrency > 0 { m.guestMetadataSlots = make(chan struct{}, concurrency) } if guestAgentVMMaxConcurrent > 0 { m.guestAgentWorkSlots = make(chan struct{}, guestAgentVMMaxConcurrent) } if appriseConfig, err := m.configPersist.LoadAppriseConfig(); err == nil { m.notificationMgr.SetAppriseConfig(*appriseConfig) } else { log.Warn().Err(err).Msg("failed to load Apprise configuration") } // Migrate webhooks if needed (from unencrypted to encrypted) if err := m.configPersist.MigrateWebhooksIfNeeded(); err != nil { log.Warn().Err(err).Msg("failed to migrate webhooks") } if webhooks, err := m.configPersist.LoadWebhooks(); err == nil { for _, webhook := range webhooks { m.notificationMgr.AddWebhook(webhook) } } else { log.Warn().Err(err).Msg("failed to load webhook configuration") } // In mock mode the canonical sampler owns demo chart history by default. // Support-only hybrid runs can opt back into real client initialization. mockEnabled := mock.IsMockEnabled() if mockEnabled && !keepRealPollingInMockMode() { log.Info().Msg("mock mode enabled - real client initialization disabled") } else { m.initPVEClients(cfg) m.initPBSClients(cfg) m.initPMGClients(cfg) } // Initialize state stats m.state.Stats = models.Stats{ StartTime: m.startTime, Version: "2.0.0-go", } return m, nil } // SetExecutor allows tests to override the poll executor; passing nil restores the default executor. func (m *Monitor) SetExecutor(exec PollExecutor) { if m == nil { return } m.mu.Lock() defer m.mu.Unlock() if exec == nil { m.executor = newRealExecutor(m) return } m.executor = exec } func (m *Monitor) buildInstanceInfoCache(cfg *config.Config) { _ = cfg m.refreshInstanceInfoCacheFromProviders() } func (m *Monitor) getExecutor() PollExecutor { m.mu.RLock() exec := m.executor m.mu.RUnlock() return exec } func clampInterval(value, min, max time.Duration) time.Duration { if value <= 0 { return min } if min > 0 && value < min { return min } if max > 0 && value > max { return max } return value } func (m *Monitor) effectivePVEPollingInterval() time.Duration { if m == nil { return effectivePVEPollingIntervalForConfig(nil) } return effectivePVEPollingIntervalForConfig(m.config) } func (m *Monitor) fixedIntervalForInstance(instanceType InstanceType, instanceName string) time.Duration { if provider, ok := m.getPollProvider(instanceType).(FixedIntervalPollProvider); ok && provider != nil { return provider.FixedInstanceInterval(m, instanceName) } return 0 } func (m *Monitor) baseIntervalForInstanceType(instanceType InstanceType) time.Duration { if provider := m.getPollProvider(instanceType); provider != nil { if interval := provider.BaseInterval(m); interval > 0 { return interval } } if m == nil || m.config == nil { return DefaultSchedulerConfig().BaseInterval } switch instanceType { case InstanceTypePVE: return m.effectivePVEPollingInterval() case InstanceTypePBS: return clampInterval(m.pbsPollingIntervalSetting(), 10*time.Second, time.Hour) case InstanceTypePMG: return clampInterval(m.pmgPollingIntervalSetting(), 10*time.Second, time.Hour) default: base := m.config.AdaptivePollingBaseInterval if base <= 0 { base = DefaultSchedulerConfig().BaseInterval } return clampInterval(base, time.Second, 0) } } // Start begins the monitoring loop func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) { // Consolidate any duplicate cluster instances before starting // This fixes the case where multiple agents registered from the same cluster m.consolidateDuplicateClusters() pollingInterval := m.effectivePVEPollingInterval() log.Info(). Dur("pollingInterval", pollingInterval). Msg("Starting monitoring loop") m.setRuntimeContext(ctx, wsHub) defer m.stopMockMetricsSampler() // Best-effort startup cleanup: when direct PBS is configured, remove legacy // PVE-proxied PBS backup points to prevent duplicate recovery entries. m.purgeStalePVEPBSBackupsBestEffort(ctx) if mock.IsMockEnabled() { m.startMockMetricsSampler(ctx) } // Initialize and start discovery service if enabled if mock.IsMockEnabled() { log.Info().Msg("mock mode enabled - skipping discovery service") m.discoveryService = nil } else if m.config.DiscoveryEnabled { discoverySubnet := m.config.DiscoverySubnet if discoverySubnet == "" { discoverySubnet = "auto" } cfgProvider := func() config.DiscoveryConfig { return m.discoveryConfigSnapshot() } m.discoveryService = discovery.NewService(wsHub, 5*time.Minute, discoverySubnet, cfgProvider) if m.discoveryService != nil { m.discoveryService.Start(ctx) log.Info().Msg("discovery service initialized and started") } else { log.Error().Msg("failed to initialize discovery service") } } else { log.Info().Msg("discovery service disabled by configuration") m.discoveryService = nil } // Set up alert callbacks m.alertManager.SetAlertCallback(func(alert *alerts.Alert) { m.handleAlertFired(alert) }) // Set up AI analysis callback - this bypasses activation state and other notification suppression // so AI can analyze alerts even during pending_review setup phase m.alertManager.SetAlertForAICallback(func(alert *alerts.Alert) { log.Debug().Str("alertID", alert.ID).Msg("AI alert callback invoked (bypassing notification suppression)") if m.alertTriggeredAICallback != nil { m.alertTriggeredAICallback(alert) } }) m.alertManager.SetResolvedCallback(func(alertID string) { m.handleAlertResolved(alertID) // Don't broadcast full state here - it causes a cascade with many guests. // The frontend will get the updated alerts through the regular broadcast ticker. }) m.alertManager.SetAcknowledgedCallback(func(alert *alerts.Alert, user string) { m.handleAlertAcknowledged(alert, user) }) m.alertManager.SetUnacknowledgedCallback(func(alert *alerts.Alert, user string) { m.handleAlertUnacknowledged(alert, user) }) m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) { m.handleAlertEscalated(wsHub, alert, level) }) // Create separate tickers for polling and broadcasting using the configured cadence workerCount := m.totalClientCount() m.startTaskWorkers(ctx, workerCount) pollTicker := time.NewTicker(pollingInterval) defer pollTicker.Stop() broadcastTicker := time.NewTicker(pollingInterval) defer broadcastTicker.Stop() keepRealPolling := keepRealPollingInMockMode() // Start connection retry mechanism for failed clients // This handles cases where network/Proxmox isn't ready on initial startup if !mock.IsMockEnabled() || keepRealPolling { go m.retryFailedConnections(ctx) } // Do an immediate poll on start. if mock.IsMockEnabled() { if keepRealPolling { log.Info().Msg("mock mode enabled - running mock alerts and real metric polling") go m.checkMockAlerts() go m.poll(ctx, wsHub) } else { log.Info().Msg("mock mode enabled - skipping real node polling") go m.checkMockAlerts() } } else { go m.poll(ctx, wsHub) } for { select { case <-pollTicker.C: now := time.Now() m.evaluateDockerAgents(now) m.evaluateKubernetesAgents(now) m.evaluateHostAgents(now) m.cleanupRemovedDockerHosts(now) m.cleanupRemovedKubernetesClusters(now) m.cleanupRemovedHostAgents(now) m.cleanupGuestMetadataCache(now) m.cleanupDiagnosticSnapshots(now) m.cleanupRRDCache(now) m.cleanupTrackingMaps(now) m.cleanupMetricsHistory() m.cleanupRateTracker(now) if mock.IsMockEnabled() { // In mock mode, keep synthetic alerts fresh go m.checkMockAlerts() if keepRealPolling { // Keep real metrics flowing while mock UI mode is active. go m.poll(ctx, wsHub) } } else { // Poll real infrastructure go m.poll(ctx, wsHub) } // Connection-degraded alerts derive from the unified connections // ledger, which lives behind the api layer. We invoke the // registered lister here so a wedged PVE/PBS/PMG/VMware/TrueNAS // connection escalates into the top-nav alert stream. go m.checkConnectionAlerts() case <-broadcastTicker.C: // Broadcast current state regardless of polling status // Use GetState() instead of m.state.GetSnapshot() to respect mock mode state := m.GetState() log.Info(). Int("nodes", len(state.Nodes)). Int("vms", len(state.VMs)). Int("containers", len(state.Containers)). Int("hosts", len(state.Hosts)). Int("pbs", len(state.PBSInstances)). Int("pbsBackups", len(state.Backups.PBS)). Int("physicalDisks", len(state.PhysicalDisks)). Msg("Broadcasting state update (ticker)") if !currentStateBroadcasterHasSubscribers(wsHub, m.GetOrgID()) { continue } m.broadcastCurrentState(wsHub) case <-ctx.Done(): log.Info().Msg("monitoring loop stopped") return } } } // poll fetches data from all configured instances func (m *Monitor) poll(_ context.Context, wsHub *websocket.Hub) { defer recoverFromPanic("poll") // Limit concurrent polls to 2 to prevent resource exhaustion currentCount := atomic.AddInt32(&m.activePollCount, 1) if currentCount > 2 { atomic.AddInt32(&m.activePollCount, -1) if logging.IsLevelEnabled(zerolog.DebugLevel) { log.Debug().Int32("activePolls", currentCount-1).Msg("too many concurrent polls, skipping") } return } defer atomic.AddInt32(&m.activePollCount, -1) if logging.IsLevelEnabled(zerolog.DebugLevel) { log.Debug().Msg("starting polling cycle") } startTime := time.Now() now := startTime plannedTasks := m.buildScheduledTasks(now) for _, task := range plannedTasks { m.taskQueue.Upsert(task) } m.updateQueueDepthMetric() // Update performance metrics atomically to prevent data races when // multiple poll() goroutines run concurrently (e.g. mock mode transitions). wsClients := 0 if wsHub != nil { wsClients = wsHub.GetClientCount() } m.state.UpdatePollStats( time.Since(startTime).Seconds(), int64(time.Since(m.startTime).Seconds()), wsClients, ) // Sync alert state so broadcasts include the latest acknowledgement data m.syncAlertsToState() // Increment poll counter m.mu.Lock() m.pollCounter++ m.mu.Unlock() if logging.IsLevelEnabled(zerolog.DebugLevel) { log.Debug().Dur("duration", time.Since(startTime)).Msg("polling cycle completed") } // Broadcasting is now handled by the timer in Start() } func (m *Monitor) startTaskWorkers(ctx context.Context, workers int) { if m.taskQueue == nil { return } if workers < 1 { workers = 1 } if workers > 10 { workers = 10 } for i := 0; i < workers; i++ { go m.taskWorker(ctx, i) } } func (m *Monitor) taskWorker(ctx context.Context, id int) { defer recoverFromPanic(fmt.Sprintf("taskWorker-%d", id)) if logging.IsLevelEnabled(zerolog.DebugLevel) { log.Debug().Int("worker", id).Msg("task worker started") } for { task, ok := m.taskQueue.WaitNext(ctx) if !ok { if logging.IsLevelEnabled(zerolog.DebugLevel) { log.Debug().Int("worker", id).Msg("task worker stopping") } return } m.executeScheduledTask(ctx, task) m.rescheduleTask(task) m.updateQueueDepthMetric() } } func derivePollTimeout(cfg *config.Config) time.Duration { timeout := defaultTaskTimeout if cfg != nil && cfg.ConnectionTimeout > 0 { // ConnectionTimeout is the budget for one provider request, while a // poll cycle may enumerate hundreds of guests across many cluster // nodes. A short per-request timeout must not shrink the whole-cycle // budget below the default; that cancelled otherwise healthy large // PVE generations at exactly 30 seconds (#1437). if requestScaledTimeout := cfg.ConnectionTimeout * 2; requestScaledTimeout > timeout { timeout = requestScaledTimeout } } if timeout < minTaskTimeout { timeout = minTaskTimeout } // Use configurable max timeout from config (set via MAX_POLL_TIMEOUT env var) // Falls back to hardcoded maxTaskTimeout if config is nil or MaxPollTimeout not set maxTimeout := maxTaskTimeout if cfg != nil && cfg.MaxPollTimeout > 0 { maxTimeout = cfg.MaxPollTimeout } if timeout > maxTimeout { timeout = maxTimeout } return timeout } func (m *Monitor) taskExecutionTimeout(_ InstanceType) time.Duration { if m == nil { return defaultTaskTimeout } timeout := m.pollTimeout if timeout <= 0 { timeout = defaultTaskTimeout } return timeout } func (m *Monitor) executeScheduledTask(ctx context.Context, task ScheduledTask) { if !m.allowExecution(task) { if logging.IsLevelEnabled(zerolog.DebugLevel) { log.Debug(). Str("instance", task.InstanceName). Str("type", string(task.InstanceType)). Msg("Task blocked by circuit breaker") } return } if m.pollMetrics != nil { wait := time.Duration(0) if !task.NextRun.IsZero() { wait = time.Since(task.NextRun) if wait < 0 { wait = 0 } } instanceType := string(task.InstanceType) if strings.TrimSpace(instanceType) == "" { instanceType = "unknown" } m.pollMetrics.RecordQueueWait(instanceType, wait) } executor := m.getExecutor() if executor == nil { log.Error(). Str("instance", task.InstanceName). Str("type", string(task.InstanceType)). Msg("No poll executor configured; skipping task") return } pollTask, ok := m.buildPollTask(task) if !ok { return } taskCtx := ctx var cancel context.CancelFunc timeout := m.taskExecutionTimeout(task.InstanceType) if timeout > 0 { taskCtx, cancel = context.WithTimeout(ctx, timeout) defer cancel() } executor.Execute(taskCtx, pollTask) if timeout > 0 && stderrors.Is(taskCtx.Err(), context.DeadlineExceeded) { log.Warn(). Str("instance", task.InstanceName). Str("type", string(task.InstanceType)). Dur("timeout", timeout). Msg("Polling task timed out; rescheduling with fresh worker") } } func (m *Monitor) buildPollTask(task ScheduledTask) (PollTask, bool) { provider := m.getPollProvider(task.InstanceType) if provider == nil { log.Debug(). Str("instance", task.InstanceName). Str("type", string(task.InstanceType)). Msg("Skipping unsupported task type") return PollTask{}, false } pollTask, err := provider.BuildPollTask(m, task.InstanceName) if err != nil { log.Warn(). Str("instance", task.InstanceName). Str("type", string(task.InstanceType)). Err(err). Msg("Skipping scheduled task") return PollTask{}, false } if strings.TrimSpace(pollTask.InstanceName) == "" { pollTask.InstanceName = task.InstanceName } if strings.TrimSpace(pollTask.InstanceType) == "" { pollTask.InstanceType = string(task.InstanceType) } return pollTask, true } func (m *Monitor) rescheduleTask(task ScheduledTask) { if m.taskQueue == nil { return } key := schedulerKey(task.InstanceType, task.InstanceName) m.mu.Lock() outcome, hasOutcome := m.lastOutcome[key] failureCount := m.failureCounts[key] m.mu.Unlock() if hasOutcome && !outcome.success { if !outcome.transient || failureCount >= m.maxRetryAttempts { m.sendToDeadLetter(task, outcome.err) return } delay := m.backoffCfg.nextDelay(failureCount-1, m.randomFloat()) if delay <= 0 { delay = 5 * time.Second } if m.config != nil && m.config.AdaptivePollingEnabled && m.config.AdaptivePollingMaxInterval > 0 && m.config.AdaptivePollingMaxInterval <= 15*time.Second { maxDelay := 4 * time.Second if delay > maxDelay { delay = maxDelay } } next := task next.Interval = delay next.NextRun = time.Now().Add(delay) m.taskQueue.Upsert(next) return } fixedInterval := m.fixedIntervalForInstance(task.InstanceType, task.InstanceName) if m.scheduler == nil { nextInterval := fixedInterval if nextInterval <= 0 { nextInterval = task.Interval } if nextInterval <= 0 { nextInterval = m.baseIntervalForInstanceType(task.InstanceType) } if nextInterval <= 0 { nextInterval = DefaultSchedulerConfig().BaseInterval } next := task next.NextRun = time.Now().Add(nextInterval) next.Interval = nextInterval m.taskQueue.Upsert(next) return } desc := InstanceDescriptor{ Name: task.InstanceName, Type: task.InstanceType, LastInterval: task.Interval, LastScheduled: task.NextRun, FixedInterval: fixedInterval, } if m.stalenessTracker != nil { if snap, ok := m.stalenessTracker.snapshot(task.InstanceType, task.InstanceName); ok { desc.LastSuccess = snap.LastSuccess desc.LastFailure = snap.LastError if snap.ChangeHash != "" { desc.Metadata = TaskMetadata{ChangeHash: snap.ChangeHash} } } } tasks := m.scheduler.BuildPlan(time.Now(), []InstanceDescriptor{desc}, m.taskQueue.Size()) if len(tasks) == 0 { next := task nextInterval := task.Interval if nextInterval <= 0 && m.config != nil { nextInterval = m.config.AdaptivePollingBaseInterval } if nextInterval <= 0 { nextInterval = DefaultSchedulerConfig().BaseInterval } next.Interval = nextInterval next.NextRun = time.Now().Add(nextInterval) m.taskQueue.Upsert(next) return } for _, next := range tasks { m.taskQueue.Upsert(next) } } func (m *Monitor) sendToDeadLetter(task ScheduledTask, err error) { if m.deadLetterQueue == nil { log.Error(). Str("instance", task.InstanceName). Str("type", string(task.InstanceType)). Err(err). Msg("Dead-letter queue unavailable; dropping task") return } log.Error(). Str("instance", task.InstanceName). Str("type", string(task.InstanceType)). Err(err). Msg("Routing task to dead-letter queue after repeated failures") next := task next.Interval = 30 * time.Minute next.NextRun = time.Now().Add(next.Interval) m.deadLetterQueue.Upsert(next) m.updateDeadLetterMetrics() key := schedulerKey(task.InstanceType, task.InstanceName) now := time.Now() m.mu.Lock() if m.dlqInsightMap == nil { m.dlqInsightMap = make(map[string]*dlqInsight) } info, ok := m.dlqInsightMap[key] if !ok { info = &dlqInsight{} m.dlqInsightMap[key] = info } if info.FirstAttempt.IsZero() { info.FirstAttempt = now } info.LastAttempt = now info.RetryCount++ info.NextRetry = next.NextRun if err != nil { info.Reason = classifyDLQReason(err) } m.mu.Unlock() } func classifyDLQReason(err error) string { if err == nil { return "" } if errors.IsRetryableError(err) { return "max_retry_attempts" } return "permanent_failure" } func (m *Monitor) updateDeadLetterMetrics() { if m.pollMetrics == nil || m.deadLetterQueue == nil { return } size := m.deadLetterQueue.Size() if size <= 0 { m.pollMetrics.UpdateDeadLetterCounts(nil) return } tasks := m.deadLetterQueue.PeekAll(size) m.pollMetrics.UpdateDeadLetterCounts(tasks) } func (m *Monitor) updateBreakerMetric(instanceType InstanceType, instance string, breaker *circuitBreaker) { if m.pollMetrics == nil || breaker == nil { return } state, failures, retryAt, _, _ := breaker.stateDetails() m.pollMetrics.SetBreakerState(string(instanceType), instance, state, failures, retryAt) } func (m *Monitor) randomFloat() float64 { if m.rng == nil { m.rng = rand.New(rand.NewSource(time.Now().UnixNano())) } return m.rng.Float64() } func (m *Monitor) updateQueueDepthMetric() { if m.pollMetrics == nil || m.taskQueue == nil { return } snapshot := m.taskQueue.Snapshot() m.pollMetrics.SetQueueDepth(snapshot.Depth) m.pollMetrics.UpdateQueueSnapshot(snapshot) } func (m *Monitor) allowExecution(task ScheduledTask) bool { if m.circuitBreakers == nil { return true } key := schedulerKey(task.InstanceType, task.InstanceName) breaker := m.ensureBreaker(key) allowed := breaker.allow(time.Now()) m.updateBreakerMetric(task.InstanceType, task.InstanceName, breaker) return allowed } func (m *Monitor) ensureBreaker(key string) *circuitBreaker { m.mu.Lock() defer m.mu.Unlock() if m.circuitBreakers == nil { m.circuitBreakers = make(map[string]*circuitBreaker) } if breaker, ok := m.circuitBreakers[key]; ok { return breaker } baseRetry := m.breakerBaseRetry if baseRetry <= 0 { baseRetry = 5 * time.Second } maxDelay := m.breakerMaxDelay if maxDelay <= 0 { maxDelay = 5 * time.Minute } halfOpen := m.breakerHalfOpenWindow if halfOpen <= 0 { halfOpen = 30 * time.Second } breaker := newCircuitBreaker(3, baseRetry, maxDelay, halfOpen) m.circuitBreakers[key] = breaker return breaker } func (m *Monitor) recordTaskResult(instanceType InstanceType, instance string, pollErr error) { if m == nil { return } key := schedulerKey(instanceType, instance) now := time.Now() breaker := m.ensureBreaker(key) m.mu.Lock() status, ok := m.pollStatusMap[key] if !ok { status = &pollStatus{} m.pollStatusMap[key] = status } if pollErr == nil { if m.failureCounts != nil { m.failureCounts[key] = 0 } if m.lastOutcome != nil { m.lastOutcome[key] = taskOutcome{ success: true, transient: true, err: nil, recordedAt: now, } } status.LastSuccess = now status.ConsecutiveFailures = 0 status.FirstFailureAt = time.Time{} // A recorded error means "current outstanding failure", not "last error // ever". Clearing it on success keeps downstream consumers honest: the // connections aggregator surfaces LastError as a live error banner and // derives Unauthorized state from it, so a stale entry would show a // red error on a healthy connection forever (#1493). status.LastErrorAt = time.Time{} status.LastErrorMessage = "" status.LastErrorCategory = "" m.mu.Unlock() if breaker != nil { breaker.recordSuccess() m.updateBreakerMetric(instanceType, instance, breaker) } return } transient := isTransientError(pollErr) category := "permanent" if transient { category = "transient" } if m.failureCounts != nil { m.failureCounts[key] = m.failureCounts[key] + 1 } if m.lastOutcome != nil { m.lastOutcome[key] = taskOutcome{ success: false, transient: transient, err: pollErr, recordedAt: now, } } status.LastErrorAt = now status.LastErrorMessage = pollErr.Error() status.LastErrorCategory = category status.ConsecutiveFailures++ if status.ConsecutiveFailures == 1 { status.FirstFailureAt = now } m.mu.Unlock() if breaker != nil { breaker.recordFailure(now) m.updateBreakerMetric(instanceType, instance, breaker) } } // SchedulerHealthResponse contains complete scheduler health data for API exposure. type SchedulerHealthResponse struct { UpdatedAt time.Time `json:"updatedAt"` Enabled bool `json:"enabled"` Queue QueueSnapshot `json:"queue"` DeadLetter DeadLetterSnapshot `json:"deadLetter"` Breakers []BreakerSnapshot `json:"breakers"` Staleness []StalenessSnapshot `json:"staleness"` Instances []InstanceHealth `json:"instances"` } // DeadLetterSnapshot contains dead-letter queue data. type DeadLetterSnapshot struct { Count int `json:"count"` Tasks []DeadLetterTask `json:"tasks"` } func emptyDeadLetterSnapshot() DeadLetterSnapshot { return DeadLetterSnapshot{ Tasks: []DeadLetterTask{}, } } func emptySchedulerHealthResponse(enabled bool) SchedulerHealthResponse { return SchedulerHealthResponse{ UpdatedAt: time.Now(), Enabled: enabled, Queue: emptyQueueSnapshot(), DeadLetter: emptyDeadLetterSnapshot(), Breakers: []BreakerSnapshot{}, Staleness: []StalenessSnapshot{}, Instances: []InstanceHealth{}, } } // SchedulerHealth returns a complete snapshot of scheduler health for API exposure. // DeadLetterCount reports the number of dead-lettered poll tasks using only // the queue's own short-lived mutex. Liveness probes (/api/health) use this // instead of SchedulerHealth, which refreshes provider caches under the // monitor's write lock on every call — far more work and lock surface than a // watchdog-polled liveness check should pay for. func (m *Monitor) DeadLetterCount() int { if m == nil || m.deadLetterQueue == nil { return 0 } return m.deadLetterQueue.Size() } func (m *Monitor) SchedulerHealth() SchedulerHealthResponse { response := emptySchedulerHealthResponse(m.config != nil && m.config.AdaptivePollingEnabled) m.refreshInstanceInfoCacheFromProviders() // Queue snapshot if m.taskQueue != nil { response.Queue = m.taskQueue.Snapshot() if m.pollMetrics != nil { m.pollMetrics.UpdateQueueSnapshot(response.Queue) } } // Dead-letter queue snapshot if m.deadLetterQueue != nil { deadLetterTasks := m.deadLetterQueue.PeekAll(25) // limit to top 25 m.mu.RLock() for i := range deadLetterTasks { key := schedulerKey(InstanceType(deadLetterTasks[i].Type), deadLetterTasks[i].Instance) if outcome, ok := m.lastOutcome[key]; ok && outcome.err != nil { deadLetterTasks[i].LastError = outcome.err.Error() } if count, ok := m.failureCounts[key]; ok { deadLetterTasks[i].Failures = count } } m.mu.RUnlock() response.DeadLetter = DeadLetterSnapshot{ Count: m.deadLetterQueue.Size(), Tasks: deadLetterTasks, } m.updateDeadLetterMetrics() } // Circuit breaker snapshots m.mu.RLock() breakerSnapshots := make([]BreakerSnapshot, 0, len(m.circuitBreakers)) for key, breaker := range m.circuitBreakers { state, failures, retryAt := breaker.State() // Only include breakers that are not in default closed state with 0 failures if state != "closed" || failures > 0 { // Parse instance type and name from key parts := strings.SplitN(key, "::", 2) instanceType, instanceName := "unknown", key if len(parts) == 2 { instanceType, instanceName = parts[0], parts[1] } breakerSnapshots = append(breakerSnapshots, BreakerSnapshot{ Instance: instanceName, Type: instanceType, State: state, Failures: failures, RetryAt: retryAt, }) } } m.mu.RUnlock() response.Breakers = breakerSnapshots // Staleness snapshots if m.stalenessTracker != nil { response.Staleness = m.stalenessTracker.Snapshot() } instanceInfos := make(map[string]*instanceInfo) pollStatuses := make(map[string]pollStatus) dlqInsights := make(map[string]dlqInsight) breakerRefs := make(map[string]*circuitBreaker) m.mu.RLock() for k, v := range m.instanceInfoCache { if v == nil { continue } copyVal := *v instanceInfos[k] = ©Val } for k, v := range m.pollStatusMap { if v == nil { continue } pollStatuses[k] = *v } for k, v := range m.dlqInsightMap { if v == nil { continue } dlqInsights[k] = *v } for k, v := range m.circuitBreakers { if v != nil { breakerRefs[k] = v } } m.mu.RUnlock() for key, breaker := range breakerRefs { instanceType := InstanceType("unknown") instanceName := key if parts := strings.SplitN(key, "::", 2); len(parts) == 2 { if parts[0] != "" { instanceType = InstanceType(parts[0]) } if parts[1] != "" { instanceName = parts[1] } } m.updateBreakerMetric(instanceType, instanceName, breaker) } keySet := make(map[string]struct{}) for k := range instanceInfos { if k != "" { keySet[k] = struct{}{} } } for k := range pollStatuses { if k != "" { keySet[k] = struct{}{} } } for k := range dlqInsights { if k != "" { keySet[k] = struct{}{} } } for k := range breakerRefs { if k != "" { keySet[k] = struct{}{} } } for _, task := range response.DeadLetter.Tasks { if task.Instance == "" { continue } keySet[schedulerKey(InstanceType(task.Type), task.Instance)] = struct{}{} } for _, snap := range response.Staleness { if snap.Instance == "" { continue } keySet[schedulerKey(InstanceType(snap.Type), snap.Instance)] = struct{}{} } if len(keySet) > 0 { keys := make([]string, 0, len(keySet)) for k := range keySet { keys = append(keys, k) } sort.Strings(keys) instances := make([]InstanceHealth, 0, len(keys)) for _, key := range keys { instType := "unknown" instName := key if parts := strings.SplitN(key, "::", 2); len(parts) == 2 { if parts[0] != "" { instType = parts[0] } if parts[1] != "" { instName = parts[1] } } instType = strings.TrimSpace(instType) instName = strings.TrimSpace(instName) info := instanceInfos[key] display := instName connection := "" if info != nil { if instType == "unknown" || instType == "" { if info.Type != "" { instType = string(info.Type) } } if strings.Contains(info.Key, "::") { if parts := strings.SplitN(info.Key, "::", 2); len(parts) == 2 { if instName == key { instName = parts[1] } if (instType == "" || instType == "unknown") && parts[0] != "" { instType = parts[0] } } } if info.DisplayName != "" { display = info.DisplayName } if info.Connection != "" { connection = info.Connection } } display = strings.TrimSpace(display) connection = strings.TrimSpace(connection) if display == "" { display = instName } if display == "" { display = connection } if instType == "" { instType = "unknown" } if instName == "" { instName = key } status, hasStatus := pollStatuses[key] instanceStatus := InstancePollStatus{} if hasStatus { instanceStatus.ConsecutiveFailures = status.ConsecutiveFailures instanceStatus.LastSuccess = timePtr(status.LastSuccess) if !status.FirstFailureAt.IsZero() { instanceStatus.FirstFailureAt = timePtr(status.FirstFailureAt) } if !status.LastErrorAt.IsZero() && status.LastErrorMessage != "" { instanceStatus.LastError = &ErrorDetail{ At: status.LastErrorAt, Message: status.LastErrorMessage, Category: status.LastErrorCategory, } } } breakerInfo := InstanceBreaker{ State: "closed", FailureCount: 0, } if br, ok := breakerRefs[key]; ok && br != nil { state, failures, retryAt, since, lastTransition := br.stateDetails() if state != "" { breakerInfo.State = state } breakerInfo.FailureCount = failures breakerInfo.RetryAt = timePtr(retryAt) breakerInfo.Since = timePtr(since) breakerInfo.LastTransition = timePtr(lastTransition) } dlqInfo := InstanceDLQ{Present: false} if dlq, ok := dlqInsights[key]; ok { dlqInfo.Present = true dlqInfo.Reason = dlq.Reason dlqInfo.FirstAttempt = timePtr(dlq.FirstAttempt) dlqInfo.LastAttempt = timePtr(dlq.LastAttempt) dlqInfo.RetryCount = dlq.RetryCount dlqInfo.NextRetry = timePtr(dlq.NextRetry) } // Collect any warnings for this instance var warnings []string if instType == "pve" { if warning, ok := m.backupPermissionWarnings[instName]; ok { warnings = append(warnings, warning) } } instances = append(instances, InstanceHealth{ Key: key, Type: instType, DisplayName: display, Instance: instName, Connection: connection, PollStatus: instanceStatus, Breaker: breakerInfo, DeadLetter: dlqInfo, Warnings: warnings, }.NormalizeCollections()) } response.Instances = instances } else { response.Instances = []InstanceHealth{} } return response } func isTransientError(err error) bool { if err == nil { return true } if errors.IsRetryableError(err) { return true } if stderrors.Is(err, context.Canceled) || stderrors.Is(err, context.DeadlineExceeded) { return true } return false } func (m *Monitor) GetState() models.StateSnapshot { if m == nil { return models.StateSnapshot{} } // Check if mock mode is enabled if mock.IsMockEnabled() { state := mock.CurrentFixtureGraph().State if state.ActiveAlerts == nil && m.alertManager != nil { // Populate snapshot lazily if the cache hasn't been filled yet. mock.UpdateAlertSnapshots(m.alertManager.GetActiveAlerts(), m.alertManager.GetRecentlyResolved()) state = mock.CurrentFixtureGraph().State } return state } if m.state == nil { return models.StateSnapshot{} } state := m.state.GetSnapshot() // Keep externally served alert arrays aligned with the live alert manager // even between explicit sync points, so APIs do not expose stale alert // counts or recently resolved incidents from cached state. state.ActiveAlerts = m.activeAlertsSnapshot() state.RecentlyResolved = m.recentlyResolvedAlertsSnapshot() // Surface filesystems reported by a unified pulse-agent inside a guest // (for example ZFS mounts that qemu-guest-agent's get-fsinfo cannot see // on PBS, #1438) in the guest overview disk listing. state.MergeLinkedHostDisksIntoGuests() return state } // ReadSnapshot returns a snapshot of the current infrastructure state, // respecting mock mode when enabled. // // This is the preferred accessor for consumer code that needs the full // StateSnapshot (e.g., chart rendering, reporting, AI state queries). // This method satisfies models.SnapshotProvider — the single canonical // interface that all consumer packages depend on. Fields available // via ReadState should be accessed there instead when practical. func (m *Monitor) ReadSnapshot() models.StateSnapshot { return m.GetState() } // BackupsSnapshot returns the current backup state. func (m *Monitor) BackupsSnapshot() models.Backups { return m.GetState().Backups } // PBSInstancesSnapshot returns the current PBS instances. func (m *Monitor) PBSInstancesSnapshot() []models.PBSInstance { if m == nil { return nil } readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil } pbsViews := readState.PBSInstances() if len(pbsViews) == 0 { return nil } instances := make([]models.PBSInstance, 0, len(pbsViews)) for _, instance := range pbsViews { if instance == nil { continue } instances = append(instances, pbsInstanceFromReadStateView(instance)) } return instances } // ReplicationJobsSnapshot returns the current replication jobs. func (m *Monitor) ReplicationJobsSnapshot() []models.ReplicationJob { return m.GetState().ReplicationJobs } // ConnectionHealthSnapshot returns the current connection health map. func (m *Monitor) ConnectionHealthSnapshot() map[string]bool { return m.GetState().ConnectionHealth } // HostsSnapshot returns the current hosts. func (m *Monitor) HostsSnapshot() []models.Host { if m == nil { return nil } readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil } readState = m.readStateWithStandaloneHostContinuity(readState) hostViews := readState.Hosts() if len(hostViews) == 0 { return nil } hosts := make([]models.Host, 0, len(hostViews)) for _, host := range hostViews { if host == nil { continue } hosts = append(hosts, hostFromReadStateView(host)) } return hosts } // VMsSnapshot returns the current VMs. func (m *Monitor) VMsSnapshot() []models.VM { if m == nil { return nil } readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil } vmViews := readState.VMs() if len(vmViews) == 0 { return nil } vms := make([]models.VM, 0, len(vmViews)) for _, vm := range vmViews { if vm == nil { continue } vms = append(vms, vmFromReadStateView(vm)) } return vms } // ContainersSnapshot returns the current system containers. func (m *Monitor) ContainersSnapshot() []models.Container { if m == nil { return nil } readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil } containerViews := readState.Containers() if len(containerViews) == 0 { return nil } containers := make([]models.Container, 0, len(containerViews)) for _, container := range containerViews { if container == nil { continue } containers = append(containers, containerFromReadStateView(container)) } return containers } // NodesSnapshot returns the current Proxmox nodes. func (m *Monitor) NodesSnapshot() []models.Node { if m == nil { return nil } readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil } nodeViews := readState.Nodes() if len(nodeViews) == 0 { return nil } nodes := make([]models.Node, 0, len(nodeViews)) for _, node := range nodeViews { if node == nil { continue } nodes = append(nodes, nodeFromReadStateView(node)) } return nodes } // DockerHostsSnapshot returns the current Docker hosts. func (m *Monitor) DockerHostsSnapshot() []models.DockerHost { if m == nil { return nil } readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil } dockerViews := readState.DockerHosts() if len(dockerViews) == 0 { return nil } hosts := make([]models.DockerHost, 0, len(dockerViews)) for _, host := range dockerViews { if host == nil { continue } hosts = append(hosts, dockerHostFromReadStateView(host)) } return hosts } // StorageSnapshot returns the current storage pools. func (m *Monitor) StorageSnapshot() []models.Storage { if m == nil { return nil } readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil } storagePools := readState.StoragePools() if len(storagePools) == 0 { return nil } storage := make([]models.Storage, 0, len(storagePools)) for _, pool := range storagePools { if pool == nil { continue } storage = append(storage, storageFromReadStateView(pool)) } return storage } func storageFromReadStateView(view *unifiedresources.StoragePoolView) models.Storage { if view == nil { return models.Storage{} } storageID := strings.TrimSpace(view.SourceID()) if storageID == "" { storageID = strings.TrimSpace(view.ID()) } nodes := view.AccessibleNodes() nodeIDs := storageNodeIDsFromReadState(view.Instance(), nodes) total := view.DiskTotal() used := view.DiskUsed() free := total - used if free < 0 { free = 0 } return models.Storage{ ID: storageID, Name: view.Name(), Node: view.Node(), Instance: view.Instance(), Nodes: nodes, NodeIDs: nodeIDs, NodeCount: len(nodes), Type: view.StorageType(), Status: string(view.Status()), Path: view.Path(), Total: total, Used: used, Free: free, Usage: view.DiskPercent(), Content: view.Content(), Shared: view.Shared(), Enabled: view.Enabled(), Active: view.Active(), LastSeen: view.LastSeen(), ZFSPool: storageZFSPoolFromReadStateView(view), } } func proxmoxRatioFromPercent(percent float64) float64 { if math.IsNaN(percent) || math.IsInf(percent, 0) || percent <= 0 { return 0 } return percent / 100 } func nodeFromReadStateView(view *unifiedresources.NodeView) models.Node { if view == nil { return models.Node{} } name := view.NodeName() displayName := "" if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != name { displayName = trimmed } return models.Node{ ID: firstNonEmptyString(view.SourceID(), view.ID()), Name: name, DisplayName: displayName, Instance: view.Instance(), Host: view.HostURL(), GuestURL: view.GuestURL(), Status: string(view.Status()), Type: "node", CPU: proxmoxRatioFromPercent(view.CPUPercent()), Memory: models.Memory{Used: view.MemoryUsed(), Total: view.MemoryTotal(), Free: maxInt64(0, view.MemoryTotal()-view.MemoryUsed()), Usage: view.MemoryPercent()}, Disk: models.Disk{Used: view.DiskUsed(), Total: view.DiskTotal(), Free: maxInt64(0, view.DiskTotal()-view.DiskUsed()), Usage: view.DiskPercent()}, Uptime: view.Uptime(), LoadAverage: view.LoadAverage(), KernelVersion: view.KernelVersion(), PVEVersion: view.PVEVersion(), CPUInfo: view.CPUInfo(), Temperature: view.TemperatureDetails(), TemperatureMonitoringEnabled: view.TemperatureMonitoringEnabled(), LastSeen: view.LastSeen(), ConnectionHealth: view.ConnectionHealth(), IsClusterMember: view.IsClusterMember(), ClusterName: view.ClusterName(), PendingUpdates: view.PendingUpdates(), PendingUpdatesCheckedAt: view.PendingUpdatesCheckedAt(), LinkedAgentID: view.LinkedAgentID(), } } func hostFromReadStateView(view *unifiedresources.HostView) models.Host { if view == nil { return models.Host{} } displayName := "" if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != view.Hostname() { displayName = trimmed } return models.Host{ ID: firstNonEmptyString(view.AgentID(), view.ID()), Hostname: view.Hostname(), DisplayName: displayName, Platform: view.Platform(), OSName: view.OSName(), OSVersion: view.OSVersion(), KernelVersion: view.KernelVersion(), Architecture: view.Architecture(), CPUCount: view.CPUCount(), CPUUsage: view.CPUPercent(), Memory: hostMemoryFromReadStateView(view), LoadAverage: view.LoadAverage(), Disks: hostDisksFromReadStateView(view.Disks()), DiskIO: hostDiskIOFromReadStateView(view.DiskIO()), NetworkInterfaces: hostNetworkInterfacesFromReadStateView(view.NetworkInterfaces()), Sensors: hostSensorsFromReadStateView(view.Sensors()), RAID: hostRAIDFromReadStateView(view.RAID()), Unraid: hostUnraidFromReadStateView(view.Unraid()), Ceph: hostCephFromReadStateView(view.Ceph()), Status: string(view.Status()), UptimeSeconds: view.UptimeSeconds(), IntervalSeconds: view.IntervalSeconds(), LastSeen: view.LastSeen(), AgentVersion: view.AgentVersion(), IntegrationSource: view.IntegrationSource(), MachineID: view.MachineID(), CommandsEnabled: view.CommandsEnabled(), ReportIP: view.ReportIP(), TokenID: view.TokenID(), TokenName: view.TokenName(), TokenHint: view.TokenHint(), TokenLastUsedAt: view.TokenLastUsedAt(), Tags: view.Tags(), DiskExclude: view.DiskExclude(), IsLegacy: view.IsLegacy(), NetInRate: view.NetInRate(), NetOutRate: view.NetOutRate(), DiskReadRate: view.DiskReadRate(), DiskWriteRate: view.DiskWriteRate(), LinkedNodeID: view.LinkedNodeID(), LinkedVMID: view.LinkedVMID(), LinkedContainerID: view.LinkedContainerID(), } } func vmFromReadStateView(view *unifiedresources.VMView) models.VM { if view == nil { return models.VM{} } totalMemory := view.MemoryTotal() usedMemory := view.MemoryUsed() totalDisk := view.DiskTotal() usedDisk := view.DiskUsed() return models.VM{ ID: firstNonEmptyString(view.SourceID(), view.ID()), VMID: view.VMID(), Name: view.Name(), Node: view.Node(), Instance: view.Instance(), Status: string(view.Status()), Type: "qemu", CPU: proxmoxRatioFromPercent(view.CPUPercent()), CPUs: view.CPUs(), Memory: models.Memory{Total: totalMemory, Used: usedMemory, Free: maxInt64(0, totalMemory-usedMemory), Usage: view.MemoryPercent()}, Disk: models.Disk{Used: usedDisk, Total: totalDisk, Free: maxInt64(0, totalDisk-usedDisk), Usage: view.DiskPercent()}, Disks: guestDisksFromReadStateView(view.Disks()), DiskStatusReason: view.DiskStatusReason(), IPAddresses: view.IPAddresses(), OSName: view.OSName(), OSVersion: view.OSVersion(), AgentVersion: view.AgentVersion(), NetworkInterfaces: guestNetworkInterfacesFromReadStateView(view.NetworkInterfaces()), NetworkIn: maxInt64(0, int64(view.NetIn())), NetworkOut: maxInt64(0, int64(view.NetOut())), DiskRead: maxInt64(0, int64(view.DiskRead())), DiskWrite: maxInt64(0, int64(view.DiskWrite())), Uptime: view.Uptime(), Template: view.Template(), LastBackup: view.LastBackup(), Tags: view.Tags(), Lock: view.Lock(), LastSeen: view.LastSeen(), } } func containerFromReadStateView(view *unifiedresources.ContainerView) models.Container { if view == nil { return models.Container{} } totalMemory := view.MemoryTotal() usedMemory := view.MemoryUsed() totalDisk := view.DiskTotal() usedDisk := view.DiskUsed() return models.Container{ ID: firstNonEmptyString(view.SourceID(), view.ID()), VMID: view.VMID(), Name: view.Name(), Node: view.Node(), Instance: view.Instance(), Status: string(view.Status()), Type: firstNonEmptyString(view.ContainerType(), "lxc"), CPU: proxmoxRatioFromPercent(view.CPUPercent()), CPUs: view.CPUs(), Memory: models.Memory{Total: totalMemory, Used: usedMemory, Free: maxInt64(0, totalMemory-usedMemory), Usage: view.MemoryPercent()}, Disk: models.Disk{Used: usedDisk, Total: totalDisk, Free: maxInt64(0, totalDisk-usedDisk), Usage: view.DiskPercent()}, Disks: guestDisksFromReadStateView(view.Disks()), NetworkIn: maxInt64(0, int64(view.NetIn())), NetworkOut: maxInt64(0, int64(view.NetOut())), DiskRead: maxInt64(0, int64(view.DiskRead())), DiskWrite: maxInt64(0, int64(view.DiskWrite())), Uptime: view.Uptime(), Template: view.Template(), LastBackup: view.LastBackup(), Tags: view.Tags(), Lock: view.Lock(), LastSeen: view.LastSeen(), IPAddresses: view.IPAddresses(), NetworkInterfaces: guestNetworkInterfacesFromReadStateView(view.NetworkInterfaces()), OSName: view.OSName(), IsOCI: view.IsOCI(), OSTemplate: view.OSTemplate(), HasDocker: view.HasDocker(), DockerCheckedAt: view.DockerCheckedAt(), } } func guestDisksFromReadStateView(disks []unifiedresources.DiskInfo) []models.Disk { if len(disks) == 0 { return nil } out := make([]models.Disk, 0, len(disks)) for _, disk := range disks { out = append(out, models.Disk{ Total: disk.Total, Used: disk.Used, Free: disk.Free, Usage: disk.Usage, Mountpoint: disk.Mountpoint, Type: disk.Filesystem, Device: disk.Device, }) } return out } func guestNetworkInterfacesFromReadStateView(interfaces []unifiedresources.NetworkInterface) []models.GuestNetworkInterface { if len(interfaces) == 0 { return nil } out := make([]models.GuestNetworkInterface, 0, len(interfaces)) for _, iface := range interfaces { out = append(out, models.GuestNetworkInterface{ Name: iface.Name, MAC: iface.MAC, Addresses: append([]string(nil), iface.Addresses...), RXBytes: maxInt64(0, int64(iface.RXBytes)), TXBytes: maxInt64(0, int64(iface.TXBytes)), }) } return out } func pbsInstanceFromReadStateView(view *unifiedresources.PBSInstanceView) models.PBSInstance { if view == nil { return models.PBSInstance{} } return models.PBSInstance{ ID: firstNonEmptyString(view.InstanceID(), view.ID()), Name: view.Name(), Host: view.HostURL(), GuestURL: view.GuestURL(), Status: string(view.Status()), Version: view.Version(), CPU: view.CPUPercent(), Memory: view.MemoryPercent(), MemoryUsed: view.MemoryUsed(), MemoryTotal: view.MemoryTotal(), Uptime: view.UptimeSeconds(), Datastores: view.DatastoreDetails(), BackupJobs: view.BackupJobs(), SyncJobs: view.SyncJobs(), VerifyJobs: view.VerifyJobs(), PruneJobs: view.PruneJobs(), GarbageJobs: view.GarbageJobs(), ConnectionHealth: view.ConnectionHealth(), LastSeen: view.LastSeen(), } } func hostMemoryFromReadStateView(view *unifiedresources.HostView) models.Memory { if view == nil { return models.Memory{} } total := view.MemoryTotal() used := view.MemoryUsed() return models.Memory{ Total: total, Used: used, Free: maxInt64(0, total-used), Usage: view.MemoryPercent(), SwapUsed: view.SwapUsed(), SwapTotal: view.SwapTotal(), } } func hostDisksFromReadStateView(disks []unifiedresources.DiskInfo) []models.Disk { if len(disks) == 0 { return nil } out := make([]models.Disk, 0, len(disks)) for _, disk := range disks { out = append(out, models.Disk{ Total: disk.Total, Used: disk.Used, Free: disk.Free, Usage: disk.Usage, Mountpoint: disk.Mountpoint, Type: disk.Filesystem, Device: disk.Device, }) } return out } func hostDiskIOFromReadStateView(diskIO []unifiedresources.HostDiskIOMeta) []models.DiskIO { if len(diskIO) == 0 { return nil } out := make([]models.DiskIO, 0, len(diskIO)) for _, entry := range diskIO { out = append(out, models.DiskIO{ Device: entry.Device, ReadBytes: entry.ReadBytes, WriteBytes: entry.WriteBytes, ReadOps: entry.ReadOps, WriteOps: entry.WriteOps, IOTime: entry.IOTimeMs, }) } return out } func hostNetworkInterfacesFromReadStateView(interfaces []unifiedresources.NetworkInterface) []models.HostNetworkInterface { if len(interfaces) == 0 { return nil } out := make([]models.HostNetworkInterface, 0, len(interfaces)) for _, iface := range interfaces { out = append(out, models.HostNetworkInterface{ Name: iface.Name, MAC: iface.MAC, Addresses: append([]string(nil), iface.Addresses...), RXBytes: iface.RXBytes, TXBytes: iface.TXBytes, SpeedMbps: int64PtrCopy(iface.SpeedMbps), }) } return out } func hostSensorsFromReadStateView(sensors *unifiedresources.HostSensorMeta) models.HostSensorSummary { if sensors == nil { return models.HostSensorSummary{} } out := models.HostSensorSummary{} if len(sensors.TemperatureCelsius) > 0 { out.TemperatureCelsius = make(map[string]float64, len(sensors.TemperatureCelsius)) for k, v := range sensors.TemperatureCelsius { out.TemperatureCelsius[k] = v } } if len(sensors.FanRPM) > 0 { out.FanRPM = make(map[string]float64, len(sensors.FanRPM)) for k, v := range sensors.FanRPM { out.FanRPM[k] = v } } if len(sensors.PowerWatts) > 0 { out.PowerWatts = make(map[string]float64, len(sensors.PowerWatts)) for k, v := range sensors.PowerWatts { out.PowerWatts[k] = v } } if len(sensors.Additional) > 0 { out.Additional = make(map[string]float64, len(sensors.Additional)) for k, v := range sensors.Additional { out.Additional[k] = v } } if len(sensors.Custom) > 0 { out.Custom = make([]models.HostCustomSensorMetric, len(sensors.Custom)) for i, metric := range sensors.Custom { out.Custom[i] = models.HostCustomSensorMetric{ ID: metric.ID, Name: metric.Name, Group: metric.Group, Subgroup: metric.Subgroup, Kind: metric.Kind, Unit: metric.Unit, Value: cloneFloat64Ptr(metric.Value), Status: metric.Status, ObservedAt: metric.ObservedAt, EventAt: cloneReportTimePtr(metric.EventAt), Error: metric.Error, AlertOnError: metric.AlertOnError, Stale: metric.Stale, } } } if len(sensors.GPU) > 0 { out.GPU = make([]models.HostGPUSensor, len(sensors.GPU)) for i, gpu := range sensors.GPU { out.GPU[i] = models.HostGPUSensor{ ID: gpu.ID, Name: gpu.Name, TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius), UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent), MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes), MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes), } } } if sensors.ThermalState != nil { out.ThermalState = hostThermalStateFromReadStateView(sensors.ThermalState) } if len(sensors.SMART) > 0 { out.SMART = make([]models.HostDiskSMART, 0, len(sensors.SMART)) for _, smart := range sensors.SMART { out.SMART = append(out.SMART, models.HostDiskSMART{ Device: smart.Device, Model: smart.Model, Serial: smart.Serial, WWN: smart.WWN, Type: smart.Type, Controller: smart.Controller, Target: smart.Target, SizeBytes: smart.SizeBytes, Temperature: smart.Temperature, Health: smart.Health, Standby: smart.Standby, Pool: smart.Pool, IO: physicalDiskIOFromUnifiedMeta(smart.IO), Collection: diskinventory.CloneStatus(smart.Collection), Attributes: smartAttributesCopy(smart.Attributes), }) } } return out } func hostThermalStateFromReadStateView(in *unifiedresources.HostThermalState) *models.HostThermalState { if in == nil { return nil } return &models.HostThermalState{ Source: in.Source, Pressure: in.Pressure, ThermalWarningLevel: cloneIntPtr(in.ThermalWarningLevel), PerformanceWarningLevel: cloneIntPtr(in.PerformanceWarningLevel), CPUPowerStatus: cloneIntPtr(in.CPUPowerStatus), LimitsPercent: cloneStringIntMap(in.LimitsPercent), } } func hostRAIDFromReadStateView(raid []unifiedresources.HostRAIDMeta) []models.HostRAIDArray { if len(raid) == 0 { return nil } out := make([]models.HostRAIDArray, 0, len(raid)) for _, entry := range raid { devices := make([]models.HostRAIDDevice, 0, len(entry.Devices)) for _, device := range entry.Devices { devices = append(devices, models.HostRAIDDevice{ Device: device.Device, State: device.State, Slot: device.Slot, }) } out = append(out, models.HostRAIDArray{ Device: entry.Device, Name: entry.Name, Level: entry.Level, State: entry.State, TotalDevices: entry.TotalDevices, ActiveDevices: entry.ActiveDevices, WorkingDevices: entry.WorkingDevices, FailedDevices: entry.FailedDevices, SpareDevices: entry.SpareDevices, UUID: entry.UUID, Devices: devices, RebuildPercent: entry.RebuildPercent, RebuildSpeed: entry.RebuildSpeed, Operation: entry.Operation, }) } return out } func hostUnraidFromReadStateView(unraid *unifiedresources.HostUnraidMeta) *models.HostUnraidStorage { if unraid == nil { return nil } out := &models.HostUnraidStorage{ ArrayStarted: unraid.ArrayStarted, ArrayState: unraid.ArrayState, SyncAction: unraid.SyncAction, SyncProgress: unraid.SyncProgress, SyncErrors: unraid.SyncErrors, NumProtected: unraid.NumProtected, NumDisabled: unraid.NumDisabled, NumInvalid: unraid.NumInvalid, NumMissing: unraid.NumMissing, } if len(unraid.Disks) > 0 { out.Disks = make([]models.HostUnraidDisk, 0, len(unraid.Disks)) for _, disk := range unraid.Disks { out.Disks = append(out.Disks, models.HostUnraidDisk{ Name: disk.Name, Device: disk.Device, Role: disk.Role, Status: disk.Status, RawStatus: disk.RawStatus, Model: disk.Model, Serial: disk.Serial, Filesystem: disk.Filesystem, Transport: disk.Transport, SizeBytes: disk.SizeBytes, UsedBytes: disk.UsedBytes, FreeBytes: disk.FreeBytes, Temperature: disk.Temperature, SpunDown: disk.SpunDown, ReadCount: disk.ReadCount, WriteCount: disk.WriteCount, ErrorCount: disk.ErrorCount, Slot: disk.Slot, }) } } return out } func hostCephFromReadStateView(ceph *unifiedresources.HostCephMeta) *models.HostCephCluster { if ceph == nil { return nil } out := &models.HostCephCluster{ FSID: ceph.FSID, Health: models.HostCephHealth{ Status: ceph.Health.Status, }, MonMap: models.HostCephMonitorMap{ Epoch: ceph.MonMap.Epoch, NumMons: ceph.MonMap.NumMons, }, MgrMap: models.HostCephManagerMap{ Available: ceph.MgrMap.Available, NumMgrs: ceph.MgrMap.NumMgrs, ActiveMgr: ceph.MgrMap.ActiveMgr, Standbys: ceph.MgrMap.Standbys, }, OSDMap: models.HostCephOSDMap{ Epoch: ceph.OSDMap.Epoch, NumOSDs: ceph.OSDMap.NumOSDs, NumUp: ceph.OSDMap.NumUp, NumIn: ceph.OSDMap.NumIn, NumDown: ceph.OSDMap.NumDown, NumOut: ceph.OSDMap.NumOut, }, PGMap: models.HostCephPGMap{ NumPGs: ceph.PGMap.NumPGs, BytesTotal: ceph.PGMap.BytesTotal, BytesUsed: ceph.PGMap.BytesUsed, BytesAvailable: ceph.PGMap.BytesAvailable, DataBytes: ceph.PGMap.DataBytes, UsagePercent: ceph.PGMap.UsagePercent, DegradedRatio: ceph.PGMap.DegradedRatio, MisplacedRatio: ceph.PGMap.MisplacedRatio, ReadBytesPerSec: ceph.PGMap.ReadBytesPerSec, WriteBytesPerSec: ceph.PGMap.WriteBytesPerSec, ReadOpsPerSec: ceph.PGMap.ReadOpsPerSec, WriteOpsPerSec: ceph.PGMap.WriteOpsPerSec, }, CollectedAt: ceph.CollectedAt, } if len(ceph.Health.Summary) > 0 { out.Health.Summary = make([]models.HostCephHealthSummary, 0, len(ceph.Health.Summary)) for _, summary := range ceph.Health.Summary { out.Health.Summary = append(out.Health.Summary, models.HostCephHealthSummary{ Severity: summary.Severity, Message: summary.Message, }) } } if len(ceph.Health.Checks) > 0 { out.Health.Checks = make(map[string]models.HostCephCheck, len(ceph.Health.Checks)) for name, check := range ceph.Health.Checks { out.Health.Checks[name] = models.HostCephCheck{ Severity: check.Severity, Message: check.Message, Detail: append([]string(nil), check.Detail...), } } } if len(ceph.MonMap.Monitors) > 0 { out.MonMap.Monitors = make([]models.HostCephMonitor, 0, len(ceph.MonMap.Monitors)) for _, monitor := range ceph.MonMap.Monitors { out.MonMap.Monitors = append(out.MonMap.Monitors, models.HostCephMonitor{ Name: monitor.Name, Rank: monitor.Rank, Addr: monitor.Addr, Status: monitor.Status, }) } } if len(ceph.Pools) > 0 { out.Pools = make([]models.HostCephPool, 0, len(ceph.Pools)) for _, pool := range ceph.Pools { out.Pools = append(out.Pools, models.HostCephPool{ ID: pool.ID, Name: pool.Name, BytesUsed: pool.BytesUsed, BytesAvailable: pool.BytesAvailable, Objects: pool.Objects, PercentUsed: pool.PercentUsed, }) } } if len(ceph.Services) > 0 { out.Services = make([]models.HostCephService, 0, len(ceph.Services)) for _, service := range ceph.Services { out.Services = append(out.Services, models.HostCephService{ Type: service.Type, Running: service.Running, Total: service.Total, Daemons: append([]string(nil), service.Daemons...), }) } } return out } func dockerHostFromReadStateView(view *unifiedresources.DockerHostView) models.DockerHost { if view == nil { return models.DockerHost{} } totalMemory := view.TotalMemoryBytes() if totalMemory == 0 { totalMemory = view.MemoryTotal() } usedMemory := view.MemoryUsed() freeMemory := maxInt64(0, totalMemory-usedMemory) return models.DockerHost{ ID: firstNonEmptyString(view.HostSourceID(), view.ID()), AgentID: view.AgentID(), Hostname: view.Hostname(), DisplayName: view.DisplayName(), CustomDisplayName: view.CustomDisplayName(), MachineID: view.MachineID(), OS: view.OS(), KernelVersion: view.KernelVersion(), Architecture: view.Architecture(), Runtime: view.Runtime(), RuntimeVersion: view.RuntimeVersion(), DockerVersion: view.DockerVersion(), CPUs: view.CPUs(), TotalMemoryBytes: totalMemory, UptimeSeconds: view.UptimeSeconds(), CPUUsage: view.CPUPercent(), LoadAverage: view.LoadAverage(), Memory: models.Memory{ Total: totalMemory, Used: usedMemory, Free: freeMemory, Usage: view.MemoryPercent(), }, Disks: hostDisksFromReadStateView(view.Disks()), NetworkInterfaces: hostNetworkInterfacesFromReadStateView(view.NetworkInterfaces()), Status: string(view.Status()), LastSeen: view.LastSeen(), IntervalSeconds: view.IntervalSeconds(), AgentVersion: view.AgentVersion(), Containers: view.Containers(), Services: view.Services(), Tasks: view.Tasks(), Swarm: dockerSwarmFromReadStateView(view.Swarm()), TokenID: view.TokenID(), TokenName: view.TokenName(), TokenHint: view.TokenHint(), TokenLastUsedAt: view.TokenLastUsedAt(), Hidden: view.Hidden(), PendingUninstall: view.PendingUninstall(), Command: view.Command(), IsLegacy: view.IsLegacy(), IdentityConflict: view.IdentityConflict(), NetInRate: view.NetInRate(), NetOutRate: view.NetOutRate(), DiskReadRate: view.DiskReadRate(), DiskWriteRate: view.DiskWriteRate(), } } func dockerSwarmFromReadStateView(in *unifiedresources.DockerSwarmInfo) *models.DockerSwarmInfo { if in == nil { return nil } return &models.DockerSwarmInfo{ NodeID: in.NodeID, NodeRole: in.NodeRole, LocalState: in.LocalState, ControlAvailable: in.ControlAvailable, ClusterID: in.ClusterID, ClusterName: in.ClusterName, Scope: in.Scope, Error: in.Error, } } func maxInt64(a, b int64) int64 { if a > b { return a } return b } func int64PtrCopy(in *int64) *int64 { if in == nil { return nil } out := *in return &out } func smartAttributesCopy(in *models.SMARTAttributes) *models.SMARTAttributes { if in == nil { return nil } out := *in out.PowerOnHours = int64PtrCopy(in.PowerOnHours) out.PowerCycles = int64PtrCopy(in.PowerCycles) out.ReallocatedSectors = int64PtrCopy(in.ReallocatedSectors) out.PendingSectors = int64PtrCopy(in.PendingSectors) out.OfflineUncorrectable = int64PtrCopy(in.OfflineUncorrectable) out.UDMACRCErrors = int64PtrCopy(in.UDMACRCErrors) if in.PercentageUsed != nil { value := *in.PercentageUsed out.PercentageUsed = &value } if in.AvailableSpare != nil { value := *in.AvailableSpare out.AvailableSpare = &value } out.MediaErrors = int64PtrCopy(in.MediaErrors) out.UnsafeShutdowns = int64PtrCopy(in.UnsafeShutdowns) return &out } func firstNonEmptyString(values ...string) string { for _, value := range values { if trimmed := strings.TrimSpace(value); trimmed != "" { return trimmed } } return "" } func storageZFSPoolFromReadStateView(view *unifiedresources.StoragePoolView) *models.ZFSPool { if view == nil { return nil } state := strings.TrimSpace(view.ZFSPoolState()) if !view.IsZFS() && state == "" && view.ZFSReadErrors() == 0 && view.ZFSWriteErrors() == 0 && view.ZFSChecksumErrors() == 0 { return nil } return &models.ZFSPool{ Name: view.Name(), State: state, ReadErrors: view.ZFSReadErrors(), WriteErrors: view.ZFSWriteErrors(), ChecksumErrors: view.ZFSChecksumErrors(), } } func storageNodeIDsFromReadState(instance string, nodes []string) []string { if len(nodes) == 0 { return nil } nodeIDs := make([]string, 0, len(nodes)) for _, node := range nodes { node = strings.TrimSpace(node) if node == "" { continue } if instance == "" { nodeIDs = append(nodeIDs, node) continue } nodeIDs = append(nodeIDs, instance+"-"+node) } if len(nodeIDs) == 0 { return nil } return nodeIDs } // ActiveAlertsSnapshot returns the current active alerts. func (m *Monitor) ActiveAlertsSnapshot() []models.Alert { return m.activeAlertsSnapshot() } // RecentlyResolvedSnapshot returns the recently resolved alerts. func (m *Monitor) RecentlyResolvedSnapshot() []models.ResolvedAlert { return m.recentlyResolvedAlertsSnapshot() } // PVEBackupsSnapshot returns the current PVE backups. func (m *Monitor) PVEBackupsSnapshot() models.PVEBackups { return m.GetState().PVEBackups } // PBSBackupsSnapshot returns the current PBS recovery artifacts. func (m *Monitor) PBSBackupsSnapshot() []models.PBSBackup { return m.GetState().PBSBackups } // BuildFrontendState returns the current state converted to frontend format. // This replaces the GetState().ToFrontend() pattern in consumer code. func (m *Monitor) BuildFrontendState() models.StateFrontend { return m.buildBroadcastFrontendStateFromSnapshot(m.GetState()) } // BuildBroadcastFrontendState returns frontend state ready for websocket // broadcasts, including the unified resource payload when a resource store is // configured. func (m *Monitor) BuildBroadcastFrontendState() models.StateFrontend { return m.buildBroadcastFrontendStateFromSnapshot(m.GetState()) } func buildFrontendStateFromSnapshot(snapshot models.StateSnapshot) models.StateFrontend { return snapshot.ToFrontend() } func (m *Monitor) buildBroadcastFrontendStateFromSnapshot(snapshot models.StateSnapshot) models.StateFrontend { frontendState := buildFrontendStateFromSnapshot(snapshot) m.updateResourceStoreForRead(snapshot) if m != nil && m.alertManager != nil { if liveAlerts := m.activeAlertsSnapshot(); len(liveAlerts) > 0 || len(frontendState.ActiveAlerts) > 0 { frontendState.ActiveAlerts = liveAlerts } } unifiedView := m.currentUnifiedStateView() metricsTargetResolver := broadcastMetricsTargetResolver(unifiedView.readState) broadcastResources := unifiedresources.CoalescePresentationHostResources(unifiedView.resources) broadcastResources = m.applyPersistedMetadataToUnifiedResources(broadcastResources) frontendState.Resources = convertResourcesForBroadcast(broadcastResources, metricsTargetResolver) frontendState.ConnectedInfrastructure = buildConnectedInfrastructure(broadcastResources, snapshot) if !unifiedView.freshness.IsZero() { frontendState.LastUpdate = unifiedView.freshness.UnixMilli() } return frontendState } // GetLiveStateSnapshot returns the underlying monitor state snapshot without // applying global mock mode overrides. // // This is useful for agent management endpoints that need to reflect actual // registrations even when mock mode is enabled for the UI/demo experience. func (m *Monitor) GetLiveStateSnapshot() models.StateSnapshot { if m == nil || m.state == nil { return models.EmptyStateSnapshot() } return m.state.GetSnapshot() } // GetLiveHostsSnapshot returns the underlying registered host agents without // applying global mock mode overrides. func (m *Monitor) GetLiveHostsSnapshot() []models.Host { if m == nil || m.state == nil { return nil } return m.state.GetSnapshot().Hosts } // SetOrgID sets the organization ID for this monitor instance. // This is used for tenant isolation in multi-tenant deployments. func (m *Monitor) SetOrgID(orgID string) { m.mu.Lock() defer m.mu.Unlock() m.orgID = strings.TrimSpace(orgID) } // GetOrgID returns the organization ID for this monitor instance. // Returns empty string for default/legacy monitors. func (m *Monitor) GetOrgID() string { m.mu.RLock() defer m.mu.RUnlock() return m.orgID } type stateBroadcaster interface { BroadcastState(state interface{}) BroadcastStateToTenant(orgID string, state interface{}) } type currentStateBroadcaster interface { BroadcastCurrentState() BroadcastCurrentStateToTenant(orgID string) } type stateSubscriberCounter interface { GetClientCount() int GetTenantClientCount(orgID string) int } func stateBroadcasterHasSubscribers(hub stateBroadcaster, orgID string) bool { if isNilStateBroadcaster(hub) { return false } counter, ok := hub.(stateSubscriberCounter) if !ok { return true } orgID = strings.TrimSpace(orgID) if orgID != "" { return counter.GetTenantClientCount(orgID) > 0 } return counter.GetClientCount() > 0 } func currentStateBroadcasterHasSubscribers(hub currentStateBroadcaster, orgID string) bool { if isNilCurrentStateBroadcaster(hub) { return false } counter, ok := hub.(stateSubscriberCounter) if !ok { return true } orgID = strings.TrimSpace(orgID) if orgID != "" { return counter.GetTenantClientCount(orgID) > 0 } return counter.GetClientCount() > 0 } // broadcastState broadcasts state to WebSocket clients. // Monitors with an explicit org ID (including "default") are tenant-scoped. // Legacy monitors without an org ID broadcast globally. func (m *Monitor) broadcastState(hub stateBroadcaster, frontendState interface{}) { if isNilStateBroadcaster(hub) { return } orgID := strings.TrimSpace(m.GetOrgID()) if !stateBroadcasterHasSubscribers(hub, orgID) { return } if orgID != "" { hub.BroadcastStateToTenant(orgID, frontendState) } else { hub.BroadcastState(frontendState) } } func (m *Monitor) broadcastCurrentState(hub currentStateBroadcaster) { if isNilCurrentStateBroadcaster(hub) { return } orgID := strings.TrimSpace(m.GetOrgID()) if !currentStateBroadcasterHasSubscribers(hub, orgID) { return } if orgID != "" { hub.BroadcastCurrentStateToTenant(orgID) } else { hub.BroadcastCurrentState() } } func isNilStateBroadcaster(hub stateBroadcaster) bool { if hub == nil { return true } value := reflect.ValueOf(hub) switch value.Kind() { case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: return value.IsNil() default: return false } } func isNilCurrentStateBroadcaster(hub currentStateBroadcaster) bool { if hub == nil { return true } value := reflect.ValueOf(hub) switch value.Kind() { case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: return value.IsNil() default: return false } } func (m *Monitor) broadcastEscalatedAlert(hub *websocket.Hub, alert *alerts.Alert) { if hub == nil || alert == nil { return } hub.BroadcastAlertToTenant(m.GetOrgID(), alert) } // SetMockMode switches between mock data and real infrastructure data at runtime. func (m *Monitor) SetMockMode(enable bool) error { current := mock.IsMockEnabled() if current == enable { log.Info().Bool("mockMode", enable).Msg("mock mode already in desired state") return nil } if enable { m.stopMockMetricsSampler() if err := mock.SetEnabled(true); err != nil { return err } m.alertManager.ClearActiveAlerts() m.mu.Lock() m.resetStateLocked() m.metricsHistory.Reset() m.mu.Unlock() m.StopDiscoveryService() m.mu.RLock() ctx := m.runtimeCtx m.mu.RUnlock() if ctx != nil { m.startMockMetricsSampler(ctx) } log.Info().Msg("switched monitor to mock mode") } else { m.stopMockMetricsSampler() if err := mock.SetEnabled(false); err != nil { return err } m.alertManager.ClearActiveAlerts() m.mu.Lock() m.resetStateLocked() m.metricsHistory.Reset() m.mu.Unlock() log.Info().Msg("switched monitor to real data mode") } m.mu.RLock() ctx := m.runtimeCtx hub := m.wsHub m.mu.RUnlock() if hub != nil { m.broadcastCurrentState(hub) } if enable && ctx != nil && keepRealPollingInMockMode() { // Keep real metrics flowing while mock mode is enabled. go m.poll(ctx, hub) } if !enable && ctx != nil { // Kick off an immediate poll to repopulate state with live data. go m.poll(ctx, hub) if hub != nil && m.config.DiscoveryEnabled { go m.StartDiscoveryService(ctx, hub, m.config.DiscoverySubnet) } } return nil } func (m *Monitor) resetStateLocked() { m.state = models.NewState() m.state.Stats = models.Stats{ StartTime: m.startTime, Version: "2.0.0-go", } } // GetStartTime returns the monitor start time func (m *Monitor) GetStartTime() time.Time { return m.startTime } // GetDiscoveryService returns the discovery service func (m *Monitor) GetDiscoveryService() *discovery.Service { return m.discoveryService } // StartDiscoveryService starts the discovery service if not already running func (m *Monitor) StartDiscoveryService(ctx context.Context, wsHub *websocket.Hub, subnet string) { m.mu.Lock() defer m.mu.Unlock() if m.discoveryService != nil { log.Debug().Msg("discovery service already running") return } if subnet == "" { subnet = "auto" } cfgProvider := func() config.DiscoveryConfig { return m.discoveryConfigSnapshot() } m.discoveryService = discovery.NewService(wsHub, 5*time.Minute, subnet, cfgProvider) if m.discoveryService != nil { m.discoveryService.Start(ctx) log.Info().Str("subnet", subnet).Msg("discovery service started") } else { log.Error().Msg("failed to create discovery service") } } // StopDiscoveryService stops the discovery service if running func (m *Monitor) StopDiscoveryService() { m.mu.Lock() defer m.mu.Unlock() if m.discoveryService != nil { m.discoveryService.Stop() m.discoveryService = nil log.Info().Msg("discovery service stopped") } } // EnableTemperatureMonitoring enables temperature data collection func (m *Monitor) EnableTemperatureMonitoring() { // Temperature collection is always enabled when tempCollector is initialized // This method exists for interface compatibility log.Info().Msg("temperature monitoring enabled") } // DisableTemperatureMonitoring disables temperature data collection func (m *Monitor) DisableTemperatureMonitoring() { // Temperature collection is always enabled when tempCollector is initialized // This method exists for interface compatibility log.Info().Msg("temperature monitoring disabled") } // SetResourceStore sets the resource store for polling optimization. // When set, the monitor will check if it should reduce polling frequency // for nodes that have host agents providing data. func (m *Monitor) SetResourceStore(store ResourceStoreInterface) { if thresholdStore, ok := store.(StaleThresholdResourceStore); ok { thresholdStore.SetStaleThresholds(m.resourceStaleThresholds()) } m.mu.Lock() m.resourceStore = store incidentStore := m.incidentStore m.mu.Unlock() m.installOperatorIntentResolver(store) log.Info().Msg("resource store set for polling optimization") if incidentStore != nil { if timelineStore, ok := store.(memory.IncidentTimelineStore); ok { incidentStore.SetResourceTimelineStore(timelineStore) } else { incidentStore.SetResourceTimelineStore(nil) } } // Immediately backfill the store from current state so ReadState // consumers have data as soon as the store is wired. // Guard against minimally initialized monitors (e.g., test fixtures // with bare &Monitor{}) where m.state may be nil. if store != nil && m.state != nil { m.updateResourceStore(m.GetState()) } } // SetSupplementalRecordsProvider configures source-native resource providers // that ingest alongside the legacy state snapshot path. func (m *Monitor) SetSupplementalRecordsProvider(source unifiedresources.DataSource, provider MonitorSupplementalRecordsProvider) { if m == nil { return } normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source)))) if normalized == "" { return } m.mu.Lock() if m.supplementalProviders == nil { m.supplementalProviders = make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider) } if provider == nil { delete(m.supplementalProviders, normalized) } else { m.supplementalProviders[normalized] = provider } m.mu.Unlock() m.updateResourceStore(m.GetState()) } // SetLicenseChecker wires the commercial feature gate used by monitoring-owned // Pro behaviour. It mirrors the alert manager's checker so entitlement lookups // stay request-free inside the polling loop. func (m *Monitor) SetLicenseChecker(checker func(feature string) bool) { if m == nil { return } m.licenseCheckerMu.Lock() m.licenseChecker = checker m.licenseCheckerMu.Unlock() } func (m *Monitor) hasLicensedFeature(feature string) bool { if m == nil { return false } m.licenseCheckerMu.RLock() checker := m.licenseChecker m.licenseCheckerMu.RUnlock() if checker == nil { return false } return checker(feature) } // SetRecoveryManager wires the recovery store manager for best-effort ingestion of // recovery points derived from polled backup/snapshot data. func (m *Monitor) SetRecoveryManager(manager *recoverymanager.Manager) { m.mu.Lock() m.recoveryManager = manager m.mu.Unlock() // Try cleanup during wiring so monitors that are already running still get // the migration once a recovery manager becomes available. go m.purgeStalePVEPBSBackupsBestEffort(context.Background()) } // GetNotificationManager returns the notification manager func (m *Monitor) GetNotificationManager() *notifications.NotificationManager { return m.notificationMgr } // GetConfigPersistence returns the config persistence manager func (m *Monitor) GetConfigPersistence() *config.ConfigPersistence { return m.configPersist } // GetMetricsStore returns the persistent metrics store func (m *Monitor) GetMetricsStore() *metrics.Store { return m.metricsStore } // GetMetricsHistory returns the in-memory metrics history for trend analysis // This is used by the AI context builder to compute trends and predictions func (m *Monitor) GetMetricsHistory() *MetricsHistory { return m.metricsHistory } // GetUnifiedResources returns the current unified resource view for this monitor. // Returns nil when no resource store is configured. func (m *Monitor) GetUnifiedResources() []unifiedresources.Resource { if m == nil { return nil } m.mu.RLock() store := m.resourceStore m.mu.RUnlock() if store == nil { return nil } return store.GetAll() } // MetricsTargetForResource resolves the metrics-store target for a canonical // unified resource ID. Resolution goes through the canonical unified view // (the same re-ingested registry the UI and /api/state read) because // callers hold IDs from that view: the raw resource store's canonical IDs // depend on per-boot ingest order for merged-source hosts, so the view and // the store can disagree about the same machine's ID. The raw store stays // as a fallback. Returns nil when nothing can resolve a target. func (m *Monitor) MetricsTargetForResource(resourceID string) *unifiedresources.MetricsTarget { if m == nil { return nil } if view := m.GetUnifiedReadStateOrSnapshot(); view != nil { if resolver, ok := view.(MetricsTargetResourceStore); ok { if target := resolver.MetricsTargetForResource(resourceID); target != nil { return target } } } m.mu.RLock() store := m.resourceStore m.mu.RUnlock() if store == nil { return nil } resolver, ok := store.(MetricsTargetResourceStore) if !ok { return nil } return resolver.MetricsTargetForResource(resourceID) } // resourceChangeTimeline is the slice of the resource store needed to read // the recorded state timeline for a canonical resource ID. type resourceChangeTimeline interface { GetRecentChanges(canonicalID string, since time.Time, limit int) ([]unifiedresources.ResourceChange, error) } // RecentResourceChanges returns the recorded change timeline for a canonical // unified resource ID since the given time, newest first (store order). // Returns nil when no resource store is wired or it does not record changes. func (m *Monitor) RecentResourceChanges(resourceID string, since time.Time, limit int) []unifiedresources.ResourceChange { if m == nil { return nil } m.mu.RLock() store := m.resourceStore m.mu.RUnlock() if store == nil { return nil } timeline, ok := store.(resourceChangeTimeline) if !ok { return nil } changes, err := timeline.GetRecentChanges(resourceID, since, limit) if err != nil { log.Warn().Err(err).Str("resourceID", resourceID).Msg("failed to read resource change timeline") return nil } return changes } type monitorUnifiedStateView struct { resources []unifiedresources.Resource readState unifiedresources.ReadState freshness time.Time } type unifiedResourceReadStateLister interface { GetAll() []unifiedresources.Resource } func monitorUnifiedStateViewFromSnapshot(snapshot models.StateSnapshot) monitorUnifiedStateView { registry := unifiedresources.NewRegistry(nil) registry.IngestSnapshot(snapshot) adapter := unifiedresources.NewMonitorAdapter(registry) return monitorUnifiedStateView{ resources: registry.List(), readState: adapter, freshness: snapshot.LastUpdate, } } func monitorUnifiedStateViewFromResources(resources []unifiedresources.Resource, freshness time.Time) monitorUnifiedStateView { registry := unifiedresources.NewRegistry(nil) registry.IngestResources(resources) adapter := unifiedresources.NewMonitorAdapter(registry) return monitorUnifiedStateView{ resources: registry.List(), readState: adapter, freshness: freshness, } } func (m *Monitor) unifiedStateViewWithStandaloneHostContinuity(view monitorUnifiedStateView) monitorUnifiedStateView { if m == nil || view.readState == nil { return view } readState := m.readStateWithStandaloneHostContinuity(view.readState) view.readState = readState lister, ok := readState.(unifiedResourceReadStateLister) if !ok { return view } resources := lister.GetAll() view.resources = resources if view.freshness.IsZero() { view.freshness = latestUnifiedResourceLastSeen(resources) } return view } func latestUnifiedResourceLastSeen(resources []unifiedresources.Resource) time.Time { var latest time.Time for _, resource := range resources { if resource.LastSeen.After(latest) { latest = resource.LastSeen } } return latest } func (m *Monitor) currentUnifiedStateView() monitorUnifiedStateView { if m == nil { return monitorUnifiedStateView{} } if mock.IsMockEnabled() { // Read the version before the snapshot so a tick landing in between // caches newer data under an older token (harmless rebuild next // call) rather than ever serving stale data under a newer one. version := mock.FixtureDataVersion() m.mockUnifiedViewMu.Lock() if m.mockUnifiedViewValid && m.mockUnifiedViewVersion == version { view := m.mockUnifiedView m.mockUnifiedViewMu.Unlock() return view } m.mockUnifiedViewMu.Unlock() resources, freshness := mock.UnifiedResourceSnapshot() if len(resources) > 0 || !freshness.IsZero() { // Consumers share this view between ticks, mirroring the // sharing semantics the persistent-store ReadState path has // always had in real mode: views are read-only. view := monitorUnifiedStateViewFromResources(resources, freshness) m.mockUnifiedViewMu.Lock() m.mockUnifiedView = view m.mockUnifiedViewVersion = version m.mockUnifiedViewValid = true m.mockUnifiedViewMu.Unlock() return view } return monitorUnifiedStateViewFromSnapshot(m.GetState()) } m.mu.RLock() store := m.resourceStore state := m.state m.mu.RUnlock() if store == nil { return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromSnapshot(m.GetState())) } resources := store.GetAll() freshness := unifiedResourceFreshness(store, state) if readState, ok := store.(unifiedresources.ReadState); ok { return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateView{ resources: resources, readState: readState, freshness: freshness, }) } if len(resources) > 0 || state == nil { return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromResources(resources, freshness)) } return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromSnapshot(m.GetState())) } func (m *Monitor) currentUnifiedResourceFreshness() time.Time { if m == nil { return time.Time{} } m.mu.RLock() store := m.resourceStore state := m.state m.mu.RUnlock() return unifiedResourceFreshness(store, state) } func unifiedResourceFreshness(store ResourceStoreInterface, state *models.State) time.Time { freshness := time.Time{} if freshnessStore, ok := store.(UnifiedResourceFreshnessStore); ok { freshness = freshnessStore.UnifiedResourceFreshness() } if freshness.IsZero() && state != nil { freshness = state.GetLastUpdate() } return freshness } // UnifiedResourceSnapshot returns a canonical unified-resource seed plus the // associated freshness marker. In mock mode it returns the shared mock // unified-resource fixture graph rather than the live resource store. func (m *Monitor) UnifiedResourceSnapshot() ([]unifiedresources.Resource, time.Time) { view := m.currentUnifiedStateView() // REST /api/resources seeds its registry from this snapshot. Apply the // same user-metadata hydration (container customUrl) as the websocket // broadcast path, so the two payload shapes cannot drift. return m.applyPersistedMetadataToUnifiedResources(view.resources), view.freshness } // GetUnifiedReadState returns a typed unified read-state provider when the // configured resource store supports it. func (m *Monitor) GetUnifiedReadState() unifiedresources.ReadState { if m == nil { return nil } m.mu.RLock() store := m.resourceStore m.mu.RUnlock() if store == nil { return nil } readState, ok := store.(unifiedresources.ReadState) if !ok { return nil } return readState } // GetUnifiedReadStateOrSnapshot returns unified read-state when available. // If the monitor has not been wired with a resource store yet, it creates an // ephemeral snapshot-backed adapter to preserve read access without exposing // direct state reads to consumer packages. func (m *Monitor) GetUnifiedReadStateOrSnapshot() unifiedresources.ReadState { return m.currentUnifiedStateView().readState } // shouldSkipNodeMetrics returns true if we should skip detailed metric polling // for the given node because a host agent is providing richer data. // This helps reduce API load when agents are active. func (m *Monitor) shouldSkipNodeMetrics(nodeName string) bool { m.mu.RLock() store := m.resourceStore m.mu.RUnlock() if store == nil { return false } should := store.ShouldSkipAPIPolling(nodeName) if should { log.Debug(). Str("node", nodeName). Msg("Skipping detailed node metrics - host agent provides data") } return should } // readPathRegistryFreshness bounds how stale the unified registry may be // when a read path (state API, websocket hydrate) serves it without // rebuilding. It only needs to be wide enough to collapse same-cycle rebuild // storms (agent report + broadcast + state request); it must stay well under // the poll cadence so every poll-cycle broadcast still publishes a fresh // registry. const readPathRegistryFreshness = 2 * time.Second // readRefreshResourceStore is implemented by stores whose read paths can // refresh without queueing behind ingest rebuilds (see MonitorAdapter). type readRefreshResourceStore interface { TryReplaceRegistryForRead(snapshot models.StateSnapshot, maxAge time.Duration, supply func() map[unifiedresources.DataSource][]unifiedresources.IngestRecord) bool } // updateResourceStoreForRead keeps the resource store fresh on behalf of a // read path. Ingest boundaries (agent reports) call updateResourceStore and // always rebuild; serving state must not redo work an ingest just did, and // above all must not queue behind an in-flight rebuild whose change-record // and identity-pin persistence can take seconds per transaction on slow // volumes (#1665: /api/state stuck for minutes with SQLite on NFS). func (m *Monitor) updateResourceStoreForRead(state models.StateSnapshot) { m.mu.RLock() store := m.resourceStore m.mu.RUnlock() if store == nil { return } readStore, ok := store.(readRefreshResourceStore) if !ok { m.updateResourceStore(state) return } if thresholdStore, ok := store.(StaleThresholdResourceStore); ok { thresholdStore.SetStaleThresholds(m.resourceStaleThresholds()) } snapshotForStore := state if ownedSources := m.providerOwnedSnapshotSources(); len(ownedSources) > 0 { snapshotForStore = unifiedresources.SnapshotWithoutSources(state, ownedSources) } rebuilt := readStore.TryReplaceRegistryForRead(snapshotForStore, readPathRegistryFreshness, m.collectSupplementalRecordsBySource) if !rebuilt { return } recordSupplementalResourceChanges(store, m.collectSupplementalChanges()) m.syncUnifiedAgentMetrics(store) m.syncUnifiedVMMetrics(store) m.syncUnifiedStorageMetrics(store) m.syncUnifiedPhysicalDiskMetrics(store) m.syncUnifiedAppContainerMetrics(store) m.syncUnifiedResourceAlertsToState(store.GetAll()) } // updateResourceStore populates the canonical resource store from current // monitoring state. Callers use it at accepted-ingest boundaries and before // broadcast hydration so every ReadState consumer observes the same snapshot. func (m *Monitor) updateResourceStore(state models.StateSnapshot) { m.mu.RLock() store := m.resourceStore m.mu.RUnlock() if store == nil { log.Debug().Msg("[Resources] No resource store configured, skipping population") return } if thresholdStore, ok := store.(StaleThresholdResourceStore); ok { thresholdStore.SetStaleThresholds(m.resourceStaleThresholds()) } log.Debug(). Int("nodes", len(state.Nodes)). Int("vms", len(state.VMs)). Int("containers", len(state.Containers)). Int("hosts", len(state.Hosts)). Int("dockerHosts", len(state.DockerHosts)). Msg("[Resources] Populating resource store from state snapshot") snapshotForStore := state ownedSources := m.providerOwnedSnapshotSources() if len(ownedSources) > 0 { snapshotForStore = unifiedresources.SnapshotWithoutSources(state, ownedSources) sourceNames := make([]string, 0, len(ownedSources)) for _, source := range ownedSources { sourceNames = append(sourceNames, string(source)) } log.Debug(). Strs("sources", sourceNames). Msg("[Resources] Suppressing legacy snapshot slices for provider-owned sources") } recordsBySource := m.collectSupplementalRecordsBySource() supplementalChanges := m.collectSupplementalChanges() if atomicStore, ok := store.(AtomicSnapshotResourceStore); ok { atomicStore.PopulateSnapshotAndSupplemental(snapshotForStore, recordsBySource) recordSupplementalResourceChanges(store, supplementalChanges) m.syncUnifiedAgentMetrics(store) m.syncUnifiedVMMetrics(store) m.syncUnifiedStorageMetrics(store) m.syncUnifiedPhysicalDiskMetrics(store) m.syncUnifiedAppContainerMetrics(store) for source, records := range recordsBySource { if len(records) == 0 { continue } log.Debug(). Str("source", string(source)). Int("records", len(records)). Msg("[Resources] Atomically ingested supplemental records") } m.syncUnifiedResourceAlertsToState(store.GetAll()) return } store.PopulateFromSnapshot(snapshotForStore) supplementalStore, ok := store.(SupplementalRecordStore) if ok { for source, records := range recordsBySource { if len(records) == 0 { continue } supplementalStore.PopulateSupplementalRecords(source, records) log.Debug(). Str("source", string(source)). Int("records", len(records)). Msg("[Resources] Ingested supplemental records") } } recordSupplementalResourceChanges(store, supplementalChanges) m.syncUnifiedAgentMetrics(store) m.syncUnifiedVMMetrics(store) m.syncUnifiedStorageMetrics(store) m.syncUnifiedPhysicalDiskMetrics(store) m.syncUnifiedAppContainerMetrics(store) m.syncUnifiedResourceAlertsToState(store.GetAll()) } // refreshUnifiedResourceStoreAfterAgentStateChange makes accepted agent // ingest and removal immediately visible to canonical ReadState consumers. // WebSocket broadcasts may also rebuild the store for their own hydrate path, // but client presence must never be the trigger that publishes agent-backed // runtime truth or retires removed inventory. func (m *Monitor) refreshUnifiedResourceStoreAfterAgentStateChange() { if m == nil || m.state == nil { return } m.updateResourceStore(m.GetState()) } func recordSupplementalResourceChanges(store ResourceStoreInterface, changes []unifiedresources.ResourceChange) { if store == nil || len(changes) == 0 { return } recorder, ok := store.(canonicalResourceChangeRecorder) if !ok || recorder == nil { return } for _, change := range changes { if err := recorder.RecordChange(change); err != nil { log.Warn(). Err(err). Str("resource_id", change.ResourceID). Str("change_id", change.ID). Str("kind", string(change.Kind)). Msg("failed to record supplemental canonical resource change") } } } func (m *Monitor) syncUnifiedAgentMetrics(store ResourceStoreInterface) { if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) { return } resolver, ok := store.(MetricsTargetResourceStore) if !ok { return } now := time.Now() storeWrites := make([]metrics.WriteMetric, 0) appendStoreWrite := func(resourceType, resourceID, metricType string, value float64) { if m.metricsStore == nil { return } storeWrites = append(storeWrites, metrics.WriteMetric{ ResourceType: resourceType, ResourceID: resourceID, MetricType: metricType, Value: value, Timestamp: now, Tier: metrics.TierRaw, }) } seenTargets := make(map[string]struct{}) for _, resource := range store.GetAll() { if resource.Type != unifiedresources.ResourceTypeAgent || resource.Metrics == nil { continue } if shouldSkipMockOwnedUnifiedMetricSync(resource) { continue } if monitorHasSource(resource.Sources, unifiedresources.SourceAgent) || monitorHasSource(resource.Sources, unifiedresources.SourceProxmox) || monitorHasSource(resource.Sources, unifiedresources.SourceDocker) { continue } target := resolver.MetricsTargetForResource(resource.ID) if target == nil || target.ResourceType != "agent" || strings.TrimSpace(target.ResourceID) == "" { continue } targetID := strings.TrimSpace(target.ResourceID) if _, ok := seenTargets[targetID]; ok { continue } seenTargets[targetID] = struct{}{} metricKey := fmt.Sprintf("agent:%s", targetID) if metric := resource.Metrics.CPU; metric != nil { value := metric.Percent if value == 0 { value = metric.Value } if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "cpu", value, now) } appendStoreWrite("agent", targetID, "cpu", value) } if metric := resource.Metrics.Memory; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) { value := metric.Percent if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "memory", value, now) } appendStoreWrite("agent", targetID, "memory", value) } if metric := resource.Metrics.Disk; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) { value := metric.Percent if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "disk", value, now) } appendStoreWrite("agent", targetID, "disk", value) } if metric := resource.Metrics.NetIn; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "netin", metric.Value, now) } appendStoreWrite("agent", targetID, "netin", metric.Value) } if metric := resource.Metrics.NetOut; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "netout", metric.Value, now) } appendStoreWrite("agent", targetID, "netout", metric.Value) } if metric := resource.Metrics.DiskRead; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "diskread", metric.Value, now) } appendStoreWrite("agent", targetID, "diskread", metric.Value) } if metric := resource.Metrics.DiskWrite; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "diskwrite", metric.Value, now) } appendStoreWrite("agent", targetID, "diskwrite", metric.Value) } } if len(storeWrites) > 0 { m.metricsStore.WriteBatchSync(storeWrites) } } func (m *Monitor) syncUnifiedVMMetrics(store ResourceStoreInterface) { if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) { return } resolver, ok := store.(MetricsTargetResourceStore) if !ok { return } now := time.Now() storeWrites := make([]metrics.WriteMetric, 0) appendStoreWrite := func(resourceType, resourceID, metricType string, value float64) { if m.metricsStore == nil { return } storeWrites = append(storeWrites, metrics.WriteMetric{ ResourceType: resourceType, ResourceID: resourceID, MetricType: metricType, Value: value, Timestamp: now, Tier: metrics.TierRaw, }) } seenTargets := make(map[string]struct{}) for _, resource := range store.GetAll() { if resource.Type != unifiedresources.ResourceTypeVM || resource.Metrics == nil { continue } if shouldSkipMockOwnedUnifiedMetricSync(resource) { continue } hasNativeVMWriter := false for _, source := range resource.Sources { if source == unifiedresources.SourceProxmox { hasNativeVMWriter = true break } } if hasNativeVMWriter { continue } target := resolver.MetricsTargetForResource(resource.ID) if target == nil || target.ResourceType != "vm" || strings.TrimSpace(target.ResourceID) == "" { continue } targetID := strings.TrimSpace(target.ResourceID) if _, ok := seenTargets[targetID]; ok { continue } seenTargets[targetID] = struct{}{} if metric := resource.Metrics.CPU; metric != nil { value := metric.Percent if value == 0 { value = metric.Value } if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(targetID, "cpu", value, now) } appendStoreWrite("vm", targetID, "cpu", value) } if metric := resource.Metrics.Memory; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) { value := metric.Percent if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(targetID, "memory", value, now) } appendStoreWrite("vm", targetID, "memory", value) } if metric := resource.Metrics.Disk; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) { value := metric.Percent if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(targetID, "disk", value, now) } appendStoreWrite("vm", targetID, "disk", value) } if metric := resource.Metrics.NetIn; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(targetID, "netin", metric.Value, now) } appendStoreWrite("vm", targetID, "netin", metric.Value) } if metric := resource.Metrics.NetOut; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(targetID, "netout", metric.Value, now) } appendStoreWrite("vm", targetID, "netout", metric.Value) } if metric := resource.Metrics.DiskRead; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(targetID, "diskread", metric.Value, now) } appendStoreWrite("vm", targetID, "diskread", metric.Value) } if metric := resource.Metrics.DiskWrite; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(targetID, "diskwrite", metric.Value, now) } appendStoreWrite("vm", targetID, "diskwrite", metric.Value) } } if len(storeWrites) > 0 { m.metricsStore.WriteBatchSync(storeWrites) } } func (m *Monitor) syncUnifiedStorageMetrics(store ResourceStoreInterface) { if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) { return } resolver, ok := store.(MetricsTargetResourceStore) if !ok { return } now := time.Now() storeWrites := make([]metrics.WriteMetric, 0) appendStoreWrite := func(resourceType, resourceID, metricType string, value float64) { if m.metricsStore == nil { return } storeWrites = append(storeWrites, metrics.WriteMetric{ ResourceType: resourceType, ResourceID: resourceID, MetricType: metricType, Value: value, Timestamp: now, Tier: metrics.TierRaw, }) } seenTargets := make(map[string]struct{}) for _, resource := range store.GetAll() { if resource.Type != unifiedresources.ResourceTypeStorage || resource.Metrics == nil || resource.Metrics.Disk == nil { continue } if shouldSkipMockOwnedUnifiedMetricSync(resource) { continue } // Native Proxmox storage already writes to history during the storage poller. if resource.Storage != nil && resource.Storage.Platform == "" { hasProxmoxSource := false for _, source := range resource.Sources { if source == unifiedresources.SourceProxmox { hasProxmoxSource = true break } } if hasProxmoxSource { continue } } target := resolver.MetricsTargetForResource(resource.ID) var targetID string if target != nil && target.ResourceType == "storage" && strings.TrimSpace(target.ResourceID) != "" { targetID = strings.TrimSpace(target.ResourceID) } else { targetID = resource.ID } if _, ok := seenTargets[targetID]; ok { continue } seenTargets[targetID] = struct{}{} disk := resource.Metrics.Disk usage := disk.Percent used := int64(0) total := int64(0) free := int64(0) if disk.Used != nil { used = *disk.Used } if disk.Total != nil { total = *disk.Total } if total > 0 { free = total - used if usage == 0 && used > 0 { usage = (float64(used) / float64(total)) * 100 } } if m.metricsHistory != nil { m.metricsHistory.AddStorageMetric(targetID, "usage", usage, now) if total > 0 { m.metricsHistory.AddStorageMetric(targetID, "used", float64(used), now) m.metricsHistory.AddStorageMetric(targetID, "total", float64(total), now) m.metricsHistory.AddStorageMetric(targetID, "avail", float64(free), now) } } appendStoreWrite("storage", targetID, "usage", usage) if total > 0 { appendStoreWrite("storage", targetID, "used", float64(used)) appendStoreWrite("storage", targetID, "total", float64(total)) appendStoreWrite("storage", targetID, "avail", float64(free)) } } if len(storeWrites) > 0 { m.metricsStore.WriteBatchSync(storeWrites) } } func (m *Monitor) syncUnifiedPhysicalDiskMetrics(store ResourceStoreInterface) { if store == nil || m.metricsStore == nil { return } resolver, ok := store.(MetricsTargetResourceStore) if !ok { return } now := time.Now() seenTargets := make(map[string]struct{}) for _, resource := range store.GetAll() { if resource.Type != unifiedresources.ResourceTypePhysicalDisk || resource.PhysicalDisk == nil { continue } if shouldSkipMockOwnedUnifiedMetricSync(resource) { continue } hasNativeWriter := false for _, source := range resource.Sources { if source == unifiedresources.SourceProxmox || source == unifiedresources.SourceAgent { hasNativeWriter = true break } } if hasNativeWriter { continue } target := resolver.MetricsTargetForResource(resource.ID) if target == nil || target.ResourceType != "disk" || strings.TrimSpace(target.ResourceID) == "" { continue } targetID := strings.TrimSpace(target.ResourceID) if _, ok := seenTargets[targetID]; ok { continue } seenTargets[targetID] = struct{}{} nodeName := "" for _, hostname := range resource.Identity.Hostnames { if hostname = strings.TrimSpace(hostname); hostname != "" { nodeName = hostname break } } if nodeName == "" { nodeName = firstNonEmptyString(strings.TrimSpace(resource.ParentName), strings.TrimSpace(resource.Name)) } disk := models.PhysicalDisk{ ID: resource.ID, Node: nodeName, DevPath: resource.PhysicalDisk.DevPath, Model: resource.PhysicalDisk.Model, Serial: resource.PhysicalDisk.Serial, WWN: resource.PhysicalDisk.WWN, Type: resource.PhysicalDisk.DiskType, Size: resource.PhysicalDisk.SizeBytes, Health: resource.PhysicalDisk.Health, Wearout: resource.PhysicalDisk.Wearout, Temperature: resource.PhysicalDisk.Temperature, RPM: resource.PhysicalDisk.RPM, Used: resource.PhysicalDisk.Used, SmartAttributes: smartAttributesFromUnifiedMeta(resource.PhysicalDisk.SMART), LastChecked: resource.LastSeen, } if disk.Serial == "" { disk.ID = targetID } m.writeSMARTMetrics(disk, now) } } func (m *Monitor) syncUnifiedAppContainerMetrics(store ResourceStoreInterface) { if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) { return } resolver, ok := store.(MetricsTargetResourceStore) if !ok { return } now := time.Now() storeWrites := make([]metrics.WriteMetric, 0) appendStoreWrite := func(resourceType, resourceID, metricType string, value float64) { if m.metricsStore == nil { return } storeWrites = append(storeWrites, metrics.WriteMetric{ ResourceType: resourceType, ResourceID: resourceID, MetricType: metricType, Value: value, Timestamp: now, Tier: metrics.TierRaw, }) } seenTargets := make(map[string]struct{}) for _, resource := range store.GetAll() { if resource.Type != unifiedresources.ResourceTypeAppContainer || resource.Metrics == nil { continue } if shouldSkipMockOwnedUnifiedMetricSync(resource) { continue } hasDockerSource := false for _, source := range resource.Sources { if source == unifiedresources.SourceDocker { hasDockerSource = true break } } if hasDockerSource { continue } target := resolver.MetricsTargetForResource(resource.ID) if target == nil || target.ResourceType != "app-container" || strings.TrimSpace(target.ResourceID) == "" { continue } targetID := strings.TrimSpace(target.ResourceID) if _, ok := seenTargets[targetID]; ok { continue } seenTargets[targetID] = struct{}{} metricKey := fmt.Sprintf("docker:%s", targetID) if metric := resource.Metrics.CPU; metric != nil { value := metric.Percent if value == 0 { value = metric.Value } if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "cpu", value, now) } appendStoreWrite("dockerContainer", targetID, "cpu", value) } if metric := resource.Metrics.Memory; metric != nil && (metric.Total != nil || metric.Percent > 0) { value := metric.Percent if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "memory", value, now) } appendStoreWrite("dockerContainer", targetID, "memory", value) } if metric := resource.Metrics.Disk; metric != nil && (metric.Total != nil || metric.Percent > 0) { value := metric.Percent if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "disk", value, now) } appendStoreWrite("dockerContainer", targetID, "disk", value) } if metric := resource.Metrics.NetIn; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "netin", metric.Value, now) } appendStoreWrite("dockerContainer", targetID, "netin", metric.Value) } if metric := resource.Metrics.NetOut; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "netout", metric.Value, now) } appendStoreWrite("dockerContainer", targetID, "netout", metric.Value) } if metric := resource.Metrics.DiskRead; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "diskread", metric.Value, now) } appendStoreWrite("dockerContainer", targetID, "diskread", metric.Value) } if metric := resource.Metrics.DiskWrite; metric != nil { if m.metricsHistory != nil { m.metricsHistory.AddGuestMetric(metricKey, "diskwrite", metric.Value, now) } appendStoreWrite("dockerContainer", targetID, "diskwrite", metric.Value) } } if len(storeWrites) > 0 { m.metricsStore.WriteBatchSync(storeWrites) } } func shouldSkipMockOwnedUnifiedMetricSync(resource unifiedresources.Resource) bool { if !mock.IsMockEnabled() { return false } // In mock mode the canonical mock sampler owns chart/history continuity for // the entire demo estate. Unified-resource sync must not append a second // live timeline on top of seeded mock history for any resource class. _ = resource return true } func shouldSkipNativeMockStateMetricWrites() bool { return mock.IsMockEnabled() } // getUnifiedResourcesForBroadcast retrieves all resources from the store. // Returns nil if no resource store is configured. func (m *Monitor) getUnifiedResourcesForBroadcast() []unifiedresources.Resource { m.mu.RLock() store := m.resourceStore m.mu.RUnlock() if store == nil { log.Debug().Msg("[Resources] No store for broadcast") return nil } allResources := store.GetAll() log.Debug().Int("count", len(allResources)).Msg("[Resources] Got resources for broadcast") if len(allResources) == 0 { return nil } return allResources } // getResourcesForBroadcast retrieves all resources from the store and converts // them to frontend format. func (m *Monitor) getResourcesForBroadcast() []models.ResourceFrontend { m.mu.RLock() store := m.resourceStore m.mu.RUnlock() return convertResourcesForBroadcast( m.applyPersistedMetadataToUnifiedResources(m.getUnifiedResourcesForBroadcast()), broadcastMetricsTargetResolver(store), ) } func (m *Monitor) applyPersistedMetadataToUnifiedResources(resources []unifiedresources.Resource) []unifiedresources.Resource { if len(resources) == 0 || m == nil { return resources } out := make([]unifiedresources.Resource, len(resources)) copy(out, resources) for i := range out { resource := &out[i] switch unifiedresources.ContractResourceType(*resource) { case unifiedresources.ResourceTypeAppContainer: if resource.Docker == nil { continue } hostID := strings.TrimSpace(resource.Docker.HostSourceID) containerID := strings.TrimSpace(resource.Docker.ContainerID) if hostID == "" { continue } if customURL, ok := m.dockerAppContainerCustomURL(*resource, hostID, containerID); ok { // The metadata record is authoritative even when empty: an // explicit clear must remove a stale URL still carried by an // older unified-resource snapshot. resource.CustomURL = strings.TrimSpace(customURL) } case unifiedresources.ResourceTypePod, unifiedresources.ResourceTypeK8sDeployment, unifiedresources.ResourceTypeK8sService: if customURL, ok := m.kubernetesWorkloadCustomURL(*resource); ok { resource.CustomURL = strings.TrimSpace(customURL) } case unifiedresources.ResourceTypeAgent, unifiedresources.ResourceType("docker-host"), unifiedresources.ResourceTypePBS, unifiedresources.ResourceTypePMG, unifiedresources.ResourceTypeK8sCluster, unifiedresources.ResourceTypeK8sNode: if customURL, ok := m.hostResourceCustomURL(*resource); ok { resource.CustomURL = customURL } } } return out } func appendUniqueMetadataCandidate(candidates []string, seen map[string]struct{}, value string) []string { value = strings.TrimSpace(value) if value == "" { return candidates } if _, exists := seen[value]; exists { return candidates } seen[value] = struct{}{} return append(candidates, value) } func hostResourceMetadataCandidates(resource unifiedresources.Resource) []string { candidates := make([]string, 0, 20) seen := make(map[string]struct{}, 20) add := func(value string) { candidates = appendUniqueMetadataCandidate(candidates, seen, value) } if resource.Docker != nil { add(resource.Docker.HostSourceID) add(resource.Docker.AgentID) } if resource.Agent != nil { add(resource.Agent.AgentID) } if resource.PBS != nil { add(resource.PBS.InstanceID) } if resource.PMG != nil { add(resource.PMG.InstanceID) } if resource.Kubernetes != nil { if unifiedresources.ContractResourceType(resource) == unifiedresources.ResourceTypeK8sNode { clusterID := strings.TrimSpace(resource.Kubernetes.ClusterID) nodeName := strings.TrimSpace(resource.Kubernetes.NodeName) if nodeName == "" { nodeName = strings.TrimSpace(resource.Name) } if clusterID != "" && nodeName != "" { add(clusterID + ":node:" + nodeName) } add(resource.Kubernetes.NodeUID) } add(resource.Kubernetes.ClusterID) add(resource.Kubernetes.AgentID) } if resource.Proxmox != nil { add(resource.Proxmox.SourceID) add(resource.Proxmox.NodeName) } if resource.DiscoveryTarget != nil { add(resource.DiscoveryTarget.AgentID) add(resource.DiscoveryTarget.ResourceID) } if resource.MetricsTarget != nil { add(resource.MetricsTarget.ResourceID) } if resource.Canonical != nil { add(resource.Canonical.PrimaryID) if prefixAt := strings.IndexByte(resource.Canonical.PrimaryID, ':'); prefixAt >= 0 { add(resource.Canonical.PrimaryID[prefixAt+1:]) } for _, alias := range resource.Canonical.Aliases { add(alias) } for _, supersededID := range resource.Canonical.SupersededIDs { add(supersededID) } } add(resource.Identity.MachineID) add(resource.ID) add(resource.Name) return candidates } func (m *Monitor) hostResourceCustomURL(resource unifiedresources.Resource) (string, bool) { if m == nil { return "", false } candidates := hostResourceMetadataCandidates(resource) if unifiedresources.ContractResourceType(resource) == unifiedresources.ResourceType("docker-host") && m.dockerMetadataStore != nil { for _, candidate := range candidates { if meta := m.dockerMetadataStore.GetHostMetadata(candidate); meta != nil { if customURL := strings.TrimSpace(meta.CustomURL); customURL != "" { return customURL, true } } } return "", false } if m.hostMetadataStore != nil { for _, candidate := range candidates { if meta := m.hostMetadataStore.Get(candidate); meta != nil { if customURL := strings.TrimSpace(meta.CustomURL); customURL != "" { return customURL, true } } } } return "", false } func (m *Monitor) dockerAppContainerCustomURL( resource unifiedresources.Resource, hostID, containerID string, ) (string, bool) { if m == nil { return "", false } if m.guestMetadataStore != nil { if stableKey := dockerAppContainerMetadataKey(hostID, resource.Name); stableKey != "" { if meta := m.guestMetadataStore.Get(stableKey); meta != nil { return strings.TrimSpace(meta.CustomURL), true } } for _, key := range append( []string{strings.TrimSpace(resource.ID)}, dockerAppContainerGuestMetadataLegacyKeys(hostID, containerID)..., ) { if key == "" { continue } if meta := m.guestMetadataStore.Get(key); meta != nil { return strings.TrimSpace(meta.CustomURL), true } } } if m.dockerMetadataStore != nil { // The runtime key is where drawer saves used to land, so it is always at // least as fresh as the stable name key, which only receives // copy-if-missing snapshots of it. Prefer the runtime record while it // exists; the stable name key covers recreated containers whose old // runtime key no longer matches. if meta := m.dockerMetadataStore.Get(dockerContainerRuntimeMetadataKey(hostID, containerID)); meta != nil { return strings.TrimSpace(meta.CustomURL), true } if stableKey := dockerContainerNameMetadataKey(hostID, resource.Name); stableKey != "" { if meta := m.dockerMetadataStore.Get(stableKey); meta != nil { return strings.TrimSpace(meta.CustomURL), true } } } return "", false } // convertResourcesForBroadcast converts unified resources into the frontend payload shape. func convertResourcesForBroadcast( allResources []unifiedresources.Resource, metricsTargetResolvers ...MetricsTargetResourceStore, ) []models.ResourceFrontend { if len(allResources) == 0 { return []models.ResourceFrontend{} } allResources = attachBroadcastMetricsTargets( allResources, firstBroadcastMetricsTargetResolver(metricsTargetResolvers), ) allResources = unifiedresources.CoalescePresentationHostResources(allResources) type broadcastResource struct { input models.ResourceConvertInput sortKey string resourceID string } converted := make([]broadcastResource, 0, len(allResources)) for _, r := range allResources { input := monitorResourceToConvertInput(r) sortKey := strings.ToLower(input.DisplayName) if sortKey == "" { sortKey = strings.ToLower(input.Name) } converted = append(converted, broadcastResource{ input: input, sortKey: sortKey, resourceID: input.ID, }) } sort.Slice(converted, func(i, j int) bool { if converted[i].sortKey == converted[j].sortKey { return converted[i].resourceID < converted[j].resourceID } return converted[i].sortKey < converted[j].sortKey }) result := make([]models.ResourceFrontend, len(converted)) for i, resource := range converted { result[i] = models.ConvertResourceToFrontend(resource.input) } return result } func broadcastMetricsTargetResolver(source interface{}) MetricsTargetResourceStore { resolver, ok := source.(MetricsTargetResourceStore) if !ok { return nil } return resolver } func firstBroadcastMetricsTargetResolver( resolvers []MetricsTargetResourceStore, ) MetricsTargetResourceStore { for _, resolver := range resolvers { if resolver != nil { return resolver } } return nil } func attachBroadcastMetricsTargets( resources []unifiedresources.Resource, resolver MetricsTargetResourceStore, ) []unifiedresources.Resource { if len(resources) == 0 || resolver == nil { return resources } out := make([]unifiedresources.Resource, len(resources)) for i, resource := range resources { if resource.MetricsTarget == nil { resource.MetricsTarget = validBroadcastMetricsTarget( resolver.MetricsTargetForResource(resource.ID), ) } out[i] = resource } return out } func validBroadcastMetricsTarget( target *unifiedresources.MetricsTarget, ) *unifiedresources.MetricsTarget { if target == nil { return nil } resourceType := strings.TrimSpace(target.ResourceType) resourceID := strings.TrimSpace(target.ResourceID) if resourceType == "" || resourceID == "" { return nil } return &unifiedresources.MetricsTarget{ ResourceType: resourceType, ResourceID: resourceID, } } func monitorResourceToConvertInput(resource unifiedresources.Resource) models.ResourceConvertInput { resourceType := monitorFrontendResourceType(resource) if resource.DiscoveryTarget == nil { resource.DiscoveryTarget = monitorDiscoveryTarget(resource, resourceType) } if resource.MetricsTarget == nil { resource.MetricsTarget = monitorMetricsTarget(resource, resourceType) } unifiedresources.RefreshCanonicalMetadata(&resource) name, displayName := monitorFrontendNames(resource, resourceType) platformID := monitorPlatformID(resource, resourceType) input := models.ResourceConvertInput{ ID: resource.ID, Type: resourceType, Technology: resource.Technology, Name: name, DisplayName: displayName, PlatformID: platformID, PlatformType: monitorPlatformType(resource, resourceType), SourceType: monitorSourceType(resource.Sources), Sources: monitorSourceKeys(resource.Sources), ParentID: monitorStringValue(resource.ParentID), ParentName: resource.ParentName, ChildCount: resource.ChildCount, ClusterID: monitorClusterID(resource), Status: monitorFrontendStatus(resource, resourceType), CPU: monitorMetricInput(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.CPU })), Memory: monitorMetricInput(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Memory })), Disk: monitorMetricInput(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Disk })), Temperature: monitorTemperature(resource), Uptime: monitorUptime(resource), Tags: append([]string(nil), resource.Tags...), Labels: monitorLabels(resource), CustomURL: strings.TrimSpace(resource.CustomURL), LastSeenUnix: monitorLastSeenUnix(resource.LastSeen), IncidentCount: resource.IncidentCount, IncidentCode: resource.IncidentCode, IncidentSeverity: string(resource.IncidentSeverity), IncidentSummary: resource.IncidentSummary, IncidentCategory: resource.IncidentCategory, IncidentLabel: resource.IncidentLabel, IncidentPriority: resource.IncidentPriority, IncidentImpactSummary: resource.IncidentImpactSummary, IncidentUrgency: resource.IncidentUrgency, IncidentAction: resource.IncidentAction, Identity: monitorIdentity(resource, name), DiscoveryTarget: monitorRawJSON(resource.DiscoveryTarget), MetricsTarget: monitorRawJSON(resource.MetricsTarget), Canonical: monitorRawJSON(resource.Canonical), Policy: monitorRawJSON(resource.Policy), AISafeSummary: resource.AISafeSummary, Capabilities: monitorRawJSON(resource.Capabilities), Relationships: monitorRawJSON(resource.Relationships), RecentChanges: monitorRawJSON(resource.RecentChanges), FacetCounts: monitorRawJSON(resource.FacetCounts), Incidents: monitorRawJSON(resource.Incidents), Proxmox: monitorRawJSON(resource.Proxmox), Storage: monitorRawJSON(resource.Storage), Agent: monitorRawJSON(resource.Agent), Docker: monitorRawJSON(resource.Docker), PBS: monitorRawJSON(resource.PBS), PMG: monitorRawJSON(resource.PMG), Kubernetes: monitorRawJSON(resource.Kubernetes), PhysicalDisk: monitorRawJSON(resource.PhysicalDisk), Ceph: monitorRawJSON(resource.Ceph), TrueNAS: monitorRawJSON(resource.TrueNAS), VMware: monitorRawJSON(resource.VMware), Availability: monitorRawJSON(resource.Availability), AvailabilityChecks: monitorRawJSON(resource.AvailabilityChecks), PlatformData: monitorPlatformData(resource, resourceType, platformID), } hasNetwork, rx, tx := monitorNetworkMetricInput(resource.Metrics) input.HasNetwork = hasNetwork input.NetworkRX = rx input.NetworkTX = tx hasDiskIO, diskRead, diskWrite := monitorDiskIOMetricInput(resource.Metrics) input.HasDiskIO = hasDiskIO input.DiskReadRate = diskRead input.DiskWriteRate = diskWrite return input } func monitorRawJSON(value interface{}) json.RawMessage { if value == nil { return nil } encoded, err := json.Marshal(value) if err != nil || string(encoded) == "null" { return nil } return encoded } func monitorDiscoveryTarget(resource unifiedresources.Resource, resourceType string) *unifiedresources.DiscoveryTarget { switch resourceType { case "agent", "docker-host": agentID := monitorAgentTargetID(resource) if agentID == "" { return nil } return &unifiedresources.DiscoveryTarget{ ResourceType: string(unifiedresources.ResourceTypeAgent), AgentID: agentID, ResourceID: agentID, Hostname: monitorTargetHostname(resource), } default: return nil } } func monitorMetricsTarget(resource unifiedresources.Resource, resourceType string) *unifiedresources.MetricsTarget { switch resourceType { case "agent", "docker-host": agentID := monitorAgentTargetID(resource) if agentID == "" { return nil } return &unifiedresources.MetricsTarget{ ResourceType: string(unifiedresources.ResourceTypeAgent), ResourceID: agentID, } default: return nil } } func monitorAgentTargetID(resource unifiedresources.Resource) string { if resource.Agent != nil { if id := strings.TrimSpace(resource.Agent.AgentID); id != "" { return id } } if resource.Docker != nil { if id := strings.TrimSpace(resource.Docker.AgentID); id != "" { return id } if id := strings.TrimSpace(resource.Docker.HostSourceID); id != "" { return id } } if resource.Kubernetes != nil { if id := strings.TrimSpace(resource.Kubernetes.AgentID); id != "" { return id } } return "" } func monitorTargetHostname(resource unifiedresources.Resource) string { if resource.Agent != nil { if hostname := strings.TrimSpace(resource.Agent.Hostname); hostname != "" { return hostname } } if resource.Docker != nil { if hostname := strings.TrimSpace(resource.Docker.Hostname); hostname != "" { return hostname } } if resource.PBS != nil { if hostname := strings.TrimSpace(resource.PBS.Hostname); hostname != "" { return hostname } } if resource.PMG != nil { if hostname := strings.TrimSpace(resource.PMG.Hostname); hostname != "" { return hostname } } return strings.TrimSpace(resource.Name) } func monitorFrontendResourceType(resource unifiedresources.Resource) string { return string(unifiedresources.ContractResourceType(resource)) } func monitorFrontendNames(resource unifiedresources.Resource, resourceType string) (string, string) { name := strings.TrimSpace(unifiedresources.ResourceDisplayName(resource)) if name == "" { name = resource.ID } return name, name } func monitorPlatformType(resource unifiedresources.Resource, resourceType string) string { if resource.Proxmox != nil { return "proxmox-pve" } if resource.VMware != nil { return "vmware-vsphere" } if resource.TrueNAS != nil { return "truenas" } if resource.Availability != nil { return "generic" } if storagePlatform := monitorStoragePlatformType(resource.Storage, resource.Sources); storagePlatform != "" { return storagePlatform } switch resourceType { case "vm", "system-container": return "proxmox-pve" case "storage", "pool": if monitorHasSource(resource.Sources, unifiedresources.SourceProxmox) { return "proxmox-pve" } if monitorHasSource(resource.Sources, unifiedresources.SourcePBS) { return "proxmox-pbs" } if monitorHasSource(resource.Sources, unifiedresources.SourceTrueNAS) { return "truenas" } if monitorHasSource(resource.Sources, unifiedresources.SourceVMware) { return "vmware-vsphere" } if monitorHasSource(resource.Sources, unifiedresources.SourceAgent) { return "agent" } return "generic" case "docker-host", "app-container": return "docker" case "k8s-cluster", "k8s-node", "pod", "k8s-deployment": return "kubernetes" case "pbs": return "proxmox-pbs" case "pmg": return "proxmox-pmg" case "agent": return "agent" default: if monitorHasSource(resource.Sources, unifiedresources.SourceK8s) { return "kubernetes" } if monitorHasSource(resource.Sources, unifiedresources.SourceDocker) { return "docker" } if monitorHasSource(resource.Sources, unifiedresources.SourcePBS) { return "proxmox-pbs" } if monitorHasSource(resource.Sources, unifiedresources.SourcePMG) { return "proxmox-pmg" } if monitorHasSource(resource.Sources, unifiedresources.SourceAgent) { return "agent" } if monitorHasSource(resource.Sources, unifiedresources.SourceProxmox) { return "proxmox-pve" } for _, source := range resource.Sources { if candidate := strings.TrimSpace(string(source)); candidate != "" { return candidate } } return "unknown" } } func monitorStoragePlatformType(storage *unifiedresources.StorageMeta, sources []unifiedresources.DataSource) string { if storage == nil { return "" } switch strings.ToLower(strings.TrimSpace(storage.Platform)) { case "proxmox", "proxmox-pve", "pve": return "proxmox-pve" case "pbs", "proxmox-pbs": return "proxmox-pbs" case "truenas": return "truenas" case "vmware", "vmware-vsphere", "vsphere": return "vmware-vsphere" case "docker", "podman": return "docker" case "kubernetes", "k8s": return "kubernetes" case "unraid": if monitorHasSource(sources, unifiedresources.SourceAgent) { return "agent" } return "generic" default: return "" } } func monitorPlatformID(resource unifiedresources.Resource, resourceType string) string { switch resourceType { case "node", "vm", "system-container": if resource.Proxmox != nil && strings.TrimSpace(resource.Proxmox.Instance) != "" { return strings.TrimSpace(resource.Proxmox.Instance) } case "agent": if resource.Agent != nil && strings.TrimSpace(resource.Agent.AgentID) != "" { return strings.TrimSpace(resource.Agent.AgentID) } case "docker-host": if resource.Docker != nil && strings.TrimSpace(resource.Docker.Hostname) != "" { return strings.TrimSpace(resource.Docker.Hostname) } case "app-container": if resource.Docker != nil && strings.TrimSpace(resource.Docker.Hostname) != "" { return strings.TrimSpace(resource.Docker.Hostname) } if resource.ParentID != nil { return strings.TrimSpace(*resource.ParentID) } case "k8s-cluster", "k8s-node", "pod", "k8s-deployment": if resource.Kubernetes != nil && strings.TrimSpace(resource.Kubernetes.AgentID) != "" { return strings.TrimSpace(resource.Kubernetes.AgentID) } case "pbs": if resource.PBS != nil && strings.TrimSpace(resource.PBS.Hostname) != "" { return strings.TrimSpace(resource.PBS.Hostname) } case "pmg": if resource.PMG != nil && strings.TrimSpace(resource.PMG.Hostname) != "" { return strings.TrimSpace(resource.PMG.Hostname) } case "network-endpoint": if resource.Availability != nil { if targetID := strings.TrimSpace(resource.Availability.TargetID); targetID != "" { return targetID } if address := strings.TrimSpace(resource.Availability.Address); address != "" { return address } } } return resource.ID } func monitorFrontendStatus(resource unifiedresources.Resource, resourceType string) string { switch resourceType { case "app-container": switch resource.Status { case unifiedresources.StatusOnline: return "running" case unifiedresources.StatusOffline: return "stopped" case unifiedresources.StatusWarning: return "degraded" } case "pod": if resource.Kubernetes != nil { phase := strings.ToLower(strings.TrimSpace(resource.Kubernetes.PodPhase)) switch phase { case "pending", "unknown": return "degraded" case "succeeded", "failed": return "stopped" } // Phase=Running is not enough — a pod can have Phase=Running with // CrashLoopBackOff containers, which statusFromKubernetesPod // correctly classifies as StatusWarning. Fall through to the // unified-status mapping below so that surfaces as "degraded". } } switch resource.Status { case unifiedresources.StatusOnline: if monitorIsWorkloadType(resourceType) || resourceType == "pod" { return "running" } return "online" case unifiedresources.StatusOffline: if monitorIsWorkloadType(resourceType) || resourceType == "pod" { return "stopped" } return "offline" case unifiedresources.StatusWarning: return "degraded" default: return "unknown" } } func monitorIsWorkloadType(resourceType string) bool { switch resourceType { case "app-container", "system-container", "vm", "oci-container": return true default: return false } } func monitorClusterID(resource unifiedresources.Resource) string { if clusterID := strings.TrimSpace(unifiedresources.ResourceClusterName(resource)); clusterID != "" { return clusterID } if resource.Docker != nil && resource.Docker.Swarm != nil { if name := strings.TrimSpace(resource.Docker.Swarm.ClusterName); name != "" { return name } if id := strings.TrimSpace(resource.Docker.Swarm.ClusterID); id != "" { return id } } return "" } func monitorMetricInput(metric *unifiedresources.MetricValue) *models.ResourceMetricInput { if metric == nil { return nil } current := metric.Percent if current == 0 { current = metric.Value } if metric.Percent != 0 && metric.Value != 0 { current = math.Max(metric.Percent, metric.Value) } result := &models.ResourceMetricInput{Current: current} if metric.Total != nil { total := *metric.Total result.Total = &total } if metric.Used != nil { used := *metric.Used result.Used = &used } if result.Total != nil && result.Used != nil { free := *result.Total - *result.Used result.Free = &free } return result } func monitorNetworkMetricInput(metrics *unifiedresources.ResourceMetrics) (bool, int64, int64) { if metrics == nil || (metrics.NetIn == nil && metrics.NetOut == nil) { return false, 0, 0 } var rx int64 var tx int64 if metrics.NetIn != nil { rx = int64(math.Round(metrics.NetIn.Value)) } if metrics.NetOut != nil { tx = int64(math.Round(metrics.NetOut.Value)) } return true, rx, tx } func monitorDiskIOMetricInput(metrics *unifiedresources.ResourceMetrics) (bool, int64, int64) { if metrics == nil || (metrics.DiskRead == nil && metrics.DiskWrite == nil) { return false, 0, 0 } var read int64 var write int64 if metrics.DiskRead != nil { read = int64(math.Round(metrics.DiskRead.Value)) } if metrics.DiskWrite != nil { write = int64(math.Round(metrics.DiskWrite.Value)) } return true, read, write } func monitorTemperature(resource unifiedresources.Resource) *float64 { if resource.Agent != nil && resource.Agent.Temperature != nil { value := *resource.Agent.Temperature return &value } if resource.Proxmox != nil && resource.Proxmox.Temperature != nil { value := *resource.Proxmox.Temperature return &value } if resource.Docker != nil && resource.Docker.Temperature != nil { value := *resource.Docker.Temperature return &value } if resource.Kubernetes != nil && resource.Kubernetes.Temperature != nil { value := *resource.Kubernetes.Temperature return &value } return nil } func monitorUptime(resource unifiedresources.Resource) *int64 { if resource.Agent != nil && resource.Agent.UptimeSeconds > 0 { value := resource.Agent.UptimeSeconds return &value } if resource.Proxmox != nil && resource.Proxmox.Uptime > 0 { value := resource.Proxmox.Uptime return &value } if resource.Docker != nil && resource.Docker.UptimeSeconds > 0 { value := resource.Docker.UptimeSeconds return &value } if resource.Kubernetes != nil && resource.Kubernetes.UptimeSeconds > 0 { value := resource.Kubernetes.UptimeSeconds return &value } if resource.PBS != nil && resource.PBS.UptimeSeconds > 0 { value := resource.PBS.UptimeSeconds return &value } if resource.PMG != nil && resource.PMG.UptimeSeconds > 0 { value := resource.PMG.UptimeSeconds return &value } if resource.TrueNAS != nil && resource.TrueNAS.UptimeSeconds > 0 { value := resource.TrueNAS.UptimeSeconds return &value } // Canonical Resource.Uptime is the universal fallback. The vSphere // adapter populates only this field for ESXi hosts and VMs (no // vmware-specific UptimeSeconds carve-out), so without this fall the // websocket broadcast layer would drop uptime for VMware-backed rows // even though the REST `/api/resources` payload exposes it on // `resource.uptime` directly. if resource.Uptime > 0 { value := resource.Uptime return &value } return nil } func monitorLabels(resource unifiedresources.Resource) map[string]string { if resource.Kubernetes == nil || len(resource.Kubernetes.Labels) == 0 { return nil } labels := make(map[string]string, len(resource.Kubernetes.Labels)) for key, value := range resource.Kubernetes.Labels { labels[key] = value } return labels } func monitorIdentity(resource unifiedresources.Resource, fallbackName string) *models.ResourceIdentityInput { hostname := "" if resource.Agent != nil { hostname = strings.TrimSpace(resource.Agent.Hostname) } if hostname == "" && resource.Docker != nil { hostname = strings.TrimSpace(resource.Docker.Hostname) } if hostname == "" && resource.Proxmox != nil { hostname = strings.TrimSpace(resource.Proxmox.NodeName) } if hostname == "" && resource.Availability != nil { hostname = strings.TrimSpace(resource.Availability.Address) } if hostname == "" { for _, candidate := range resource.Identity.Hostnames { if trimmed := strings.TrimSpace(candidate); trimmed != "" { hostname = trimmed break } } } if hostname == "" { hostname = fallbackName } ips := make([]string, 0, len(resource.Identity.IPAddresses)) for _, ip := range resource.Identity.IPAddresses { trimmed := strings.TrimSpace(ip) if trimmed == "" { continue } ips = append(ips, trimmed) } machineID := strings.TrimSpace(resource.Identity.MachineID) if hostname == "" && machineID == "" && len(ips) == 0 { return nil } return &models.ResourceIdentityInput{ Hostname: hostname, MachineID: machineID, IPs: ips, } } func monitorPlatformData(resource unifiedresources.Resource, resourceType string, platformID string) json.RawMessage { var payload interface{} switch resourceType { case "node": if resource.Proxmox != nil { payload = map[string]interface{}{ "instance": resource.Proxmox.Instance, "host": "", "guestURL": "", "pveVersion": resource.Proxmox.PVEVersion, "kernelVersion": resource.Proxmox.KernelVersion, "cpuInfo": resource.Proxmox.CPUInfo, "loadAverage": []float64{}, "isClusterMember": resource.Proxmox.ClusterName != "", "clusterName": resource.Proxmox.ClusterName, "connectionHealth": monitorSourceStatus(resource.SourceStatus, unifiedresources.SourceProxmox), } } case "vm": payload = buildProxmoxVMPayload(resource) case "system-container", "oci-container": payload = buildProxmoxVMPayload(resource) case "agent": if resource.Proxmox != nil { payload = map[string]interface{}{ "instance": resource.Proxmox.Instance, "host": "", "guestURL": "", "pveVersion": resource.Proxmox.PVEVersion, "kernelVersion": resource.Proxmox.KernelVersion, "cpuInfo": resource.Proxmox.CPUInfo, "loadAverage": []float64{}, "isClusterMember": resource.Proxmox.ClusterName != "", "clusterName": resource.Proxmox.ClusterName, "connectionHealth": monitorSourceStatus(resource.SourceStatus, unifiedresources.SourceProxmox), } } else if resource.Agent != nil { payload = map[string]interface{}{ "platform": resource.Agent.Platform, "osName": resource.Agent.OSName, "osVersion": resource.Agent.OSVersion, "kernelVersion": resource.Agent.KernelVersion, "architecture": resource.Agent.Architecture, "agentVersion": resource.Agent.AgentVersion, "interfaces": resource.Agent.NetworkInterfaces, "disks": resource.Agent.Disks, "memory": resource.Agent.Memory, } } case "docker-host": if resource.Docker != nil { payload = map[string]interface{}{ "agentId": platformID, "runtime": resource.Docker.Runtime, "runtimeVersion": resource.Docker.RuntimeVersion, "dockerVersion": resource.Docker.DockerVersion, "os": resource.Docker.OS, "kernelVersion": resource.Docker.KernelVersion, "architecture": resource.Docker.Architecture, "agentVersion": resource.Docker.AgentVersion, "swarm": resource.Docker.Swarm, "interfaces": resource.Docker.NetworkInterfaces, "disks": resource.Docker.Disks, } } case "app-container": if resource.Docker != nil { payload = map[string]interface{}{ "hostId": monitorStringValue(resource.ParentID), "hostName": resource.Docker.Hostname, "image": resource.Docker.Image, "state": strings.ToLower(string(resource.Status)), "status": strings.ToLower(string(resource.Status)), "health": "", "createdAt": time.Time{}, } } case "k8s-cluster": if resource.Kubernetes != nil { payload = map[string]interface{}{ "agentId": resource.Kubernetes.AgentID, "server": resource.Kubernetes.Server, "context": resource.Kubernetes.Context, "version": resource.Kubernetes.Version, "customDisplayName": "", "hidden": false, "pendingUninstall": resource.Kubernetes.PendingUninstall, "nodeCount": resource.ChildCount, } } case "k8s-node": if resource.Kubernetes != nil { payload = map[string]interface{}{ "clusterId": resource.Kubernetes.ClusterID, "ready": resource.Kubernetes.Ready, "unschedulable": resource.Kubernetes.Unschedulable, "kubeletVersion": resource.Kubernetes.KubeletVersion, "containerRuntimeVersion": resource.Kubernetes.ContainerRuntimeVersion, "osImage": resource.Kubernetes.OSImage, "kernelVersion": resource.Kubernetes.KernelVersion, "architecture": resource.Kubernetes.Architecture, "capacityCpuCores": resource.Kubernetes.CapacityCPU, "capacityMemoryBytes": resource.Kubernetes.CapacityMemoryBytes, "capacityPods": resource.Kubernetes.CapacityPods, "allocatableCpuCores": resource.Kubernetes.AllocCPU, "allocatableMemoryBytes": resource.Kubernetes.AllocMemoryBytes, "allocatablePods": resource.Kubernetes.AllocPods, "roles": append([]string(nil), resource.Kubernetes.Roles...), } } case "pod": if resource.Kubernetes != nil { payload = map[string]interface{}{ "clusterId": resource.Kubernetes.ClusterID, "namespace": resource.Kubernetes.Namespace, "nodeName": resource.Kubernetes.NodeName, "phase": resource.Kubernetes.PodPhase, "restarts": resource.Kubernetes.Restarts, "ownerKind": resource.Kubernetes.OwnerKind, "ownerName": resource.Kubernetes.OwnerName, } } case "k8s-deployment": if resource.Kubernetes != nil { payload = map[string]interface{}{ "clusterId": resource.Kubernetes.ClusterID, "namespace": resource.Kubernetes.Namespace, "desiredReplicas": resource.Kubernetes.DesiredReplicas, "updatedReplicas": resource.Kubernetes.UpdatedReplicas, "readyReplicas": resource.Kubernetes.ReadyReplicas, "availableReplicas": resource.Kubernetes.AvailableReplicas, } } case "pbs": if resource.PBS != nil { payload = map[string]interface{}{ "host": resource.PBS.Hostname, "version": resource.PBS.Version, "connectionHealth": resource.PBS.ConnectionHealth, "memoryUsed": monitorMetricUsed(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Memory })), "memoryTotal": monitorMetricTotal(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Memory })), "numDatastores": resource.PBS.DatastoreCount, } } case "pmg": if resource.PMG != nil { payload = map[string]interface{}{ "host": resource.PMG.Hostname, "version": resource.PMG.Version, "connectionHealth": resource.PMG.ConnectionHealth, "nodeCount": resource.PMG.NodeCount, "queueActive": resource.PMG.QueueActive, "queueDeferred": resource.PMG.QueueDeferred, "queueHold": resource.PMG.QueueHold, "queueIncoming": resource.PMG.QueueIncoming, "queueTotal": resource.PMG.QueueTotal, "mailCountTotal": resource.PMG.MailCountTotal, "spamIn": resource.PMG.SpamIn, "virusIn": resource.PMG.VirusIn, "lastUpdated": resource.PMG.LastUpdated, } } case "storage", "pool": payload = monitorStoragePlatformData(resource, platformID) case "network-endpoint": if resource.Availability != nil { payload = map[string]interface{}{ "targetId": resource.Availability.TargetID, "address": resource.Availability.Address, "protocol": resource.Availability.Protocol, "port": resource.Availability.Port, "path": resource.Availability.Path, "enabled": resource.Availability.Enabled, "available": resource.Availability.Available, "lastChecked": resource.Availability.LastChecked, "lastSuccess": resource.Availability.LastSuccess, "latencyMillis": resource.Availability.LatencyMillis, "consecutiveFailures": resource.Availability.ConsecutiveFailures, "lastError": resource.Availability.LastError, "failureThreshold": resource.Availability.FailureThreshold, "pollIntervalSeconds": resource.Availability.PollIntervalSeconds, "timeoutMillis": resource.Availability.TimeoutMillis, } } } if payload == nil { return nil } payload = monitorAttachSourceKeys(payload, resource.Sources) if payload == nil { return nil } encoded, err := json.Marshal(payload) if err != nil { return nil } return encoded } func monitorAttachSourceKeys(payload interface{}, sources []unifiedresources.DataSource) interface{} { sourceKeys := monitorSourceKeys(sources) if len(sourceKeys) == 0 { return payload } if payloadMap, ok := payload.(map[string]interface{}); ok { if payloadMap == nil { return nil } payloadMap["sources"] = sourceKeys return payloadMap } return payload } func monitorStoragePlatformData(resource unifiedresources.Resource, platformID string) map[string]interface{} { nodeLabel := resource.ParentName if nodeLabel == "" { nodeLabel = monitorStringValue(resource.ParentID) } payload := map[string]interface{}{ "instance": platformID, "node": nodeLabel, "active": resource.Status == unifiedresources.StatusOnline, } if resource.Proxmox != nil { if strings.TrimSpace(resource.Proxmox.Instance) != "" { payload["instance"] = strings.TrimSpace(resource.Proxmox.Instance) } if strings.TrimSpace(resource.Proxmox.NodeName) != "" { payload["node"] = strings.TrimSpace(resource.Proxmox.NodeName) } } if resource.Storage == nil { payload["type"] = "" payload["content"] = "" payload["shared"] = false payload["enabled"] = true return payload } storage := resource.Storage payload["type"] = storage.Type payload["content"] = storage.Content payload["contentTypes"] = append([]string(nil), storage.ContentTypes...) payload["shared"] = storage.Shared payload["enabled"] = storage.Enabled payload["active"] = storage.Active payload["isCeph"] = storage.IsCeph payload["isZfs"] = storage.IsZFS payload["platform"] = storage.Platform payload["topology"] = storage.Topology payload["protection"] = storage.Protection payload["pool"] = storage.Pool payload["path"] = storage.Path payload["nodes"] = append([]string(nil), storage.Nodes...) payload["risk"] = storage.Risk payload["riskSummary"] = storage.RiskSummary payload["postureSummary"] = storage.PostureSummary payload["protectionReduced"] = storage.ProtectionReduced payload["protectionSummary"] = storage.ProtectionSummary payload["rebuildInProgress"] = storage.RebuildInProgress payload["rebuildSummary"] = storage.RebuildSummary payload["arrayState"] = storage.ArrayState payload["syncAction"] = storage.SyncAction payload["syncProgress"] = storage.SyncProgress payload["numProtected"] = storage.NumProtected payload["numDisabled"] = storage.NumDisabled payload["numInvalid"] = storage.NumInvalid payload["numMissing"] = storage.NumMissing payload["zfsPoolState"] = storage.ZFSPoolState payload["zfsReadErrors"] = storage.ZFSReadErrors payload["zfsWriteErrors"] = storage.ZFSWriteErrors payload["zfsChecksumErrors"] = storage.ZFSChecksumErrors if storage.ZFSPool != nil { payload["zfsPool"] = storage.ZFSPool } return payload } func convertProxmoxDisks(disks []unifiedresources.DiskInfo) []map[string]interface{} { if len(disks) == 0 { return nil } out := make([]map[string]interface{}, 0, len(disks)) for _, d := range disks { usage := float64(0) if d.Total > 0 { usage = float64(d.Used) / float64(d.Total) * 100 } out = append(out, map[string]interface{}{ "total": d.Total, "used": d.Used, "free": d.Free, "usage": usage, "mountpoint": d.Mountpoint, "type": d.Filesystem, "device": d.Device, }) } return out } func buildProxmoxVMPayload(resource unifiedresources.Resource) map[string]interface{} { if resource.Proxmox == nil { return nil } return map[string]interface{}{ "vmid": resource.Proxmox.VMID, "node": resource.Proxmox.NodeName, "instance": resource.Proxmox.Instance, "cpus": resource.Proxmox.CPUs, "template": resource.Proxmox.Template, "networkIn": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.NetIn }), "networkOut": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.NetOut }), "diskRead": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.DiskRead }), "diskWrite": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.DiskWrite }), "disks": convertProxmoxDisks(resource.Proxmox.Disks), "swapUsed": resource.Proxmox.SwapUsed, "swapTotal": resource.Proxmox.SwapTotal, "balloon": resource.Proxmox.Balloon, "lastBackup": resource.Proxmox.LastBackup, "ipAddresses": append([]string(nil), resource.Identity.IPAddresses...), } } func monitorMetricValue(metrics *unifiedresources.ResourceMetrics, pick func(*unifiedresources.ResourceMetrics) *unifiedresources.MetricValue) *unifiedresources.MetricValue { if metrics == nil { return nil } return pick(metrics) } func monitorMetricInt64(metrics *unifiedresources.ResourceMetrics, pick func(*unifiedresources.ResourceMetrics) *unifiedresources.MetricValue) int64 { metric := monitorMetricValue(metrics, pick) if metric == nil { return 0 } return int64(math.Round(metric.Value)) } func monitorMetricUsed(metric *unifiedresources.MetricValue) int64 { if metric == nil || metric.Used == nil { return 0 } return *metric.Used } func monitorMetricTotal(metric *unifiedresources.MetricValue) int64 { if metric == nil || metric.Total == nil { return 0 } return *metric.Total } func monitorSourceStatus(statuses map[unifiedresources.DataSource]unifiedresources.SourceStatus, source unifiedresources.DataSource) string { if statuses == nil { return "" } status, ok := statuses[source] if !ok { return "" } return status.Status } func monitorHasSource(sources []unifiedresources.DataSource, source unifiedresources.DataSource) bool { for _, candidate := range sources { if candidate == source { return true } } return false } func monitorSourceKeys(sources []unifiedresources.DataSource) []string { if len(sources) == 0 { return nil } keys := make([]string, 0, len(sources)) seen := make(map[string]struct{}, len(sources)) for _, source := range sources { key := strings.TrimSpace(string(source)) if key == "" { continue } if _, ok := seen[key]; ok { continue } seen[key] = struct{}{} keys = append(keys, key) } return keys } func monitorSourceType(sources []unifiedresources.DataSource) string { if len(sources) > 1 { return "hybrid" } if len(sources) == 1 { switch sources[0] { case unifiedresources.SourceAgent, unifiedresources.SourceDocker, unifiedresources.SourceK8s: return "agent" default: return "api" } } return "api" } func monitorStringValue(value *string) string { if value == nil { return "" } return strings.TrimSpace(*value) } func monitorLastSeenUnix(value time.Time) int64 { // Zero means the resource has never actually been sighted (synthesized // offline placeholders). Report 0 so the frontend renders "never" instead // of fabricating a fresh sighting. if value.IsZero() { return 0 } return value.UnixMilli() } // pollStorageBackupsWithNodes polls backups using a provided nodes list to avoid duplicate GetNodes calls // Stop gracefully stops the monitor // guestMetadataDrainTimeout bounds how long Stop waits for queued guest // metadata writes. It matches tenantMonitorShutdownTimeout so a wedged store // cannot hold up tenant teardown indefinitely. const guestMetadataDrainTimeout = 2 * time.Second func (m *Monitor) Stop() { log.Info().Msg("stopping monitor") // Stop the alert manager to save history if m.alertManager != nil { m.alertManager.Stop() } // Stop notification manager if m.notificationMgr != nil { m.notificationMgr.Stop() } // Drain background guest-metadata writes before the data directory can be // torn down. Without this a queued write lands after shutdown and leaves a // stray guest_metadata.json.tmp behind. if m.guestMetadataStore != nil { m.guestMetadataStore.WaitForPendingWrites(guestMetadataDrainTimeout) } // Close persistent metrics store (flushes buffered data) if m.metricsStore != nil { if err := m.metricsStore.Close(); err != nil { log.Error().Err(err).Msg("failed to close metrics store") } else { log.Info().Msg("metrics store closed successfully") } } log.Info().Msg("monitor stopped") } // recordAuthFailure records an authentication failure for a node