mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-22 03:35:52 +00:00
rcourtman
e09fe06201
The critical (severity-escalation) threshold was hardcoded as Trigger + 10 for all metric types. For percentage metrics (cpu, memory, disk, usage) with high triggers, this made critical escalation unreachable: a CPU trigger of 95% produced a critical threshold of 105%, which is impossible for a 0-100% metric. Fix: add computeCriticalThreshold() helper that caps the critical threshold at 99 for percentage metrics. Non-percentage metrics (temperature, diskRead, diskWrite, networkIn, networkOut) keep the Trigger + 10 offset unchanged. Applied to all three code paths that derive critical from trigger: - buildCanonicalMetricSpec (canonical evaluation) - checkMetric new alert creation (legacy path) - checkMetric existing alert update (legacy path)
703 lines
22 KiB
Go
703 lines
22 KiB
Go
package alerts
|
||
|
||
import (
|
||
"strings"
|
||
|
||
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||
"github.com/rs/zerolog/log"
|
||
)
|
||
|
||
// Override Key Format by Resource Type
|
||
//
|
||
// The config.Overrides map is keyed by resource ID. Each resource type
|
||
// uses the following canonical key format:
|
||
//
|
||
// VM / Container: monitoring resource ID, e.g. "pve1:node1:100"
|
||
// Node: node.ID e.g. "node/pve-1"
|
||
// Agent: host.ID e.g. "host1" (without "agent:" prefix)
|
||
// Agent Disk: "agent:{hostID}/disk:{mountpoint}" e.g. "agent:host1/disk:root"
|
||
// Storage: storage.ID e.g. "local-lvm"
|
||
// PBS: pbs.ID e.g. "pbs-1"
|
||
// PMG: pmg.ID e.g. "pmg-1"
|
||
// Docker Container: "docker:{hostID}/{containerID}"
|
||
// Kubernetes: canonical unified resource ID for cluster/node/namespace/deployment/pod
|
||
// TrueNAS System: canonical unified agent resource ID
|
||
// TrueNAS Storage: canonical unified storage resource ID for pool/dataset
|
||
// TrueNAS Disk: canonical unified physical_disk resource ID
|
||
// VMware vSphere: canonical unified resource ID for host/vm/datastore/network
|
||
//
|
||
// For guests, CheckUnifiedResource expects the same canonical resource ID the
|
||
// monitoring pipeline already uses. It does not translate IDs.
|
||
// UnifiedResourceMetric holds a single metric value for unified evaluation.
|
||
type UnifiedResourceMetric struct {
|
||
Value float64
|
||
Percent float64
|
||
}
|
||
|
||
// UnifiedResourceInput is the data needed to evaluate alerts for a unified resource.
|
||
// This avoids importing unifiedresources (which would cause an import cycle).
|
||
type UnifiedResourceInput struct {
|
||
ID string
|
||
Type string // lowercase: "vm", "system-container", "app-container", "agent", "pbs", "storage", "pmg"
|
||
Name string
|
||
Node string
|
||
Instance string
|
||
CPU *UnifiedResourceMetric
|
||
Memory *UnifiedResourceMetric
|
||
Disk *UnifiedResourceMetric
|
||
DiskRead *UnifiedResourceMetric
|
||
DiskWrite *UnifiedResourceMetric
|
||
NetworkIn *UnifiedResourceMetric
|
||
NetworkOut *UnifiedResourceMetric
|
||
Temperature *UnifiedResourceMetric
|
||
}
|
||
|
||
type unifiedMetricCandidate struct {
|
||
Spec alertspecs.ResourceAlertSpec
|
||
Value float64
|
||
Threshold *HysteresisThreshold
|
||
}
|
||
|
||
// unifiedAlertType maps a resource type key to the alert system's display type.
|
||
func unifiedAlertType(typeKey string) string {
|
||
switch typeKey {
|
||
case "vm":
|
||
return "VM"
|
||
case "system-container":
|
||
return "Container"
|
||
case "app-container":
|
||
return "Docker"
|
||
case "agent":
|
||
return "Agent"
|
||
case "node":
|
||
return "Node"
|
||
case "pbs":
|
||
return "PBS"
|
||
case "storage":
|
||
return "Storage"
|
||
case "pmg":
|
||
return "PMG"
|
||
case "k8s-cluster":
|
||
return "Kubernetes Cluster"
|
||
case "k8s-node":
|
||
return "Kubernetes Node"
|
||
case "k8s-deployment":
|
||
return "Kubernetes Deployment"
|
||
case "k8s-namespace":
|
||
return "Kubernetes Namespace"
|
||
case "pod":
|
||
return "Kubernetes Pod"
|
||
case "truenas-system":
|
||
return "TrueNAS System"
|
||
case "truenas-pool":
|
||
return "TrueNAS Pool"
|
||
case "truenas-dataset":
|
||
return "TrueNAS Dataset"
|
||
case "truenas-disk":
|
||
return "TrueNAS Disk"
|
||
case "vmware-host":
|
||
return "vSphere Host"
|
||
case "vmware-vm":
|
||
return "vSphere VM"
|
||
case "vmware-datastore":
|
||
return "vSphere Datastore"
|
||
case "vmware-network":
|
||
return "vSphere Network"
|
||
default:
|
||
return typeKey
|
||
}
|
||
}
|
||
|
||
func supportsUnifiedIOMetrics(typeKey string) bool {
|
||
switch typeKey {
|
||
case "vm", "system-container", "app-container", "k8s-cluster", "k8s-deployment", "pod", "truenas-system", "vmware-host", "vmware-vm":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func isUnifiedKubernetesAlertType(typeKey string) bool {
|
||
switch typeKey {
|
||
case "k8s-cluster", "k8s-node", "k8s-deployment", "k8s-namespace", "pod":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func isUnifiedTrueNASAlertType(typeKey string) bool {
|
||
switch typeKey {
|
||
case "truenas-system", "truenas-pool", "truenas-dataset", "truenas-disk":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func isUnifiedVMwareAlertType(typeKey string) bool {
|
||
switch typeKey {
|
||
case "vmware-host", "vmware-vm", "vmware-datastore", "vmware-network":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func isUnifiedModernPlatformAlertType(typeKey string) bool {
|
||
return isUnifiedKubernetesAlertType(typeKey) || isUnifiedTrueNASAlertType(typeKey) || isUnifiedVMwareAlertType(typeKey)
|
||
}
|
||
|
||
func (m *Manager) unifiedPlatformAlertsDisabledNoLock(typeKey string) bool {
|
||
switch {
|
||
case isUnifiedKubernetesAlertType(typeKey):
|
||
return m.config.DisableAllKubernetes
|
||
case isUnifiedTrueNASAlertType(typeKey):
|
||
return m.config.DisableAllTrueNAS
|
||
case isUnifiedVMwareAlertType(typeKey):
|
||
return m.config.DisableAllVMware
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// defaultThresholdsForResourceType returns the default ThresholdConfig for a resource type key.
|
||
func (m *Manager) defaultThresholdsForResourceType(typeKey string) ThresholdConfig {
|
||
switch typeKey {
|
||
case "vm", "system-container", "app-container":
|
||
return cloneThresholdConfig(m.config.GuestDefaults)
|
||
case "agent":
|
||
return cloneThresholdConfig(m.config.AgentDefaults)
|
||
case "node":
|
||
return cloneThresholdConfig(m.config.NodeDefaults)
|
||
case "pbs":
|
||
return cloneThresholdConfig(m.config.PBSDefaults)
|
||
case "storage":
|
||
return ThresholdConfig{Usage: cloneThreshold(&m.config.StorageDefault)}
|
||
case "k8s-cluster", "k8s-node", "k8s-deployment", "pod":
|
||
return cloneThresholdConfig(m.config.KubernetesDefaults)
|
||
case "k8s-namespace":
|
||
return ThresholdConfig{}
|
||
case "truenas-system":
|
||
return cloneThresholdConfig(m.config.TrueNASDefaults)
|
||
case "truenas-pool", "truenas-dataset":
|
||
defaults := cloneThresholdConfig(m.config.TrueNASDefaults)
|
||
return ThresholdConfig{Usage: cloneThreshold(defaults.Usage)}
|
||
case "truenas-disk":
|
||
return cloneThresholdConfig(m.config.TrueNASDiskDefaults)
|
||
case "vmware-host", "vmware-vm":
|
||
return cloneThresholdConfig(m.config.VMwareDefaults)
|
||
case "vmware-datastore":
|
||
defaults := cloneThresholdConfig(m.config.VMwareDefaults)
|
||
return ThresholdConfig{Usage: cloneThreshold(defaults.Usage)}
|
||
case "vmware-network":
|
||
return ThresholdConfig{}
|
||
default:
|
||
return ThresholdConfig{}
|
||
}
|
||
}
|
||
|
||
// resolveThresholdOverride applies an exact resource-ID override onto an existing base config.
|
||
// Callers must hold m.mu when reading config through this helper.
|
||
func (m *Manager) resolveThresholdOverride(base ThresholdConfig, resourceID string) ThresholdConfig {
|
||
if override, exists := m.config.Overrides[resourceID]; exists {
|
||
return m.applyThresholdOverride(base, override)
|
||
}
|
||
|
||
return base
|
||
}
|
||
|
||
// resolveGuestThresholdOverride applies the shared guest override lookup onto an existing base config.
|
||
// Callers must hold m.mu when reading config through this helper.
|
||
func (m *Manager) resolveGuestThresholdOverride(base ThresholdConfig, guest any, guestID string) ThresholdConfig {
|
||
if override, exists := lookupGuestOverride(m.config.Overrides, guest, guestID); exists {
|
||
return m.applyThresholdOverride(base, override)
|
||
}
|
||
return base
|
||
}
|
||
|
||
// resolveResourceThresholds builds the effective thresholds for a resource type and ID.
|
||
// Callers must hold m.mu when reading config through this helper.
|
||
func (m *Manager) resolveResourceThresholds(typeKey, resourceID string) ThresholdConfig {
|
||
if isGuestThresholdResourceType(typeKey) {
|
||
return m.resolveGuestThresholdOverride(m.defaultThresholdsForResourceType(typeKey), nil, resourceID)
|
||
}
|
||
if typeKey == "storage" {
|
||
return m.resolveStorageThresholdOverride(m.defaultThresholdsForResourceType(typeKey), resourceID, nil)
|
||
}
|
||
return m.resolveThresholdOverride(m.defaultThresholdsForResourceType(typeKey), resourceID)
|
||
}
|
||
|
||
// evaluateUnifiedMetrics runs the common metric dispatch path for unified resources.
|
||
func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds ThresholdConfig, opts *metricOptions) {
|
||
if input == nil {
|
||
return
|
||
}
|
||
|
||
for _, candidate := range buildUnifiedMetricCandidates(input, thresholds) {
|
||
m.checkMetricWithCanonicalSpec(candidate.Spec, input.Name, input.Node, input.Instance, unifiedAlertType(input.Type), candidate.Value, candidate.Threshold, opts)
|
||
}
|
||
}
|
||
|
||
func buildUnifiedMetricCandidates(input *UnifiedResourceInput, thresholds ThresholdConfig) []unifiedMetricCandidate {
|
||
if input == nil {
|
||
return nil
|
||
}
|
||
|
||
resourceType, ok := unifiedMetricResourceType(input.Type)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
|
||
candidates := make([]unifiedMetricCandidate, 0, 8)
|
||
appendCandidate := func(metricType string, metric *UnifiedResourceMetric, value float64, threshold *HysteresisThreshold) {
|
||
if metric == nil {
|
||
return
|
||
}
|
||
spec, err := buildCanonicalMetricSpec(input.ID, input.Name, resourceType, metricType, threshold)
|
||
if err != nil {
|
||
log.Warn().
|
||
Err(err).
|
||
Str("resourceID", input.ID).
|
||
Str("resourceType", input.Type).
|
||
Str("metricType", metricType).
|
||
Msg("Skipping invalid canonical unified metric spec")
|
||
return
|
||
}
|
||
candidates = append(candidates, unifiedMetricCandidate{
|
||
Spec: spec,
|
||
Value: value,
|
||
Threshold: threshold,
|
||
})
|
||
}
|
||
|
||
appendCandidate("cpu", input.CPU, input.CPUValue(), thresholds.CPU)
|
||
appendCandidate("memory", input.Memory, input.MemoryValue(), thresholds.Memory)
|
||
appendCandidate("disk", input.Disk, input.DiskValue(), thresholds.Disk)
|
||
|
||
if supportsUnifiedIOMetrics(input.Type) {
|
||
appendCandidate("diskRead", input.DiskRead, input.DiskReadValue(), thresholds.DiskRead)
|
||
appendCandidate("diskWrite", input.DiskWrite, input.DiskWriteValue(), thresholds.DiskWrite)
|
||
appendCandidate("networkIn", input.NetworkIn, input.NetworkInValue(), thresholds.NetworkIn)
|
||
appendCandidate("networkOut", input.NetworkOut, input.NetworkOutValue(), thresholds.NetworkOut)
|
||
}
|
||
|
||
if unifiedStorageUsageResourceType(input.Type) && input.Disk != nil {
|
||
appendCandidate("usage", input.Disk, input.DiskValue(), thresholds.Usage)
|
||
}
|
||
|
||
appendCandidate("temperature", input.Temperature, input.TemperatureValue(), thresholds.Temperature)
|
||
|
||
return candidates
|
||
}
|
||
|
||
func unifiedStorageUsageResourceType(typeKey string) bool {
|
||
switch typeKey {
|
||
case "storage", "truenas-pool", "truenas-dataset", "vmware-datastore":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// isPercentageMetric returns true for metrics whose values are bounded 0–100.
|
||
// The critical threshold for these must stay below 100 or escalation will never fire.
|
||
func isPercentageMetric(metricType string) bool {
|
||
switch metricType {
|
||
case "cpu", "memory", "disk", "usage":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// computeCriticalThreshold derives the critical (severity-escalation) threshold
|
||
// from the trigger value. For percentage metrics (0–100 range) the result is
|
||
// capped at 99 so that critical escalation remains reachable when the trigger
|
||
// is set high (e.g. trigger 95 → critical 99, not 105).
|
||
func computeCriticalThreshold(trigger float64, metricType string) float64 {
|
||
critical := trigger + 10
|
||
if isPercentageMetric(metricType) && critical > 99 {
|
||
return 99
|
||
}
|
||
return critical
|
||
}
|
||
|
||
func buildCanonicalMetricSpec(resourceID, title string, resourceType unifiedresources.ResourceType, metricType string, threshold *HysteresisThreshold) (alertspecs.ResourceAlertSpec, error) {
|
||
spec := alertspecs.ResourceAlertSpec{
|
||
ID: canonicalMetricSpecID(resourceID, metricType),
|
||
ResourceID: resourceID,
|
||
ResourceType: resourceType,
|
||
Kind: alertspecs.AlertSpecKindMetricThreshold,
|
||
Severity: alertspecs.AlertSeverityWarning,
|
||
Title: title,
|
||
Disabled: threshold == nil || threshold.Trigger <= 0,
|
||
MetricThreshold: &alertspecs.MetricThresholdSpec{
|
||
Metric: metricType,
|
||
Direction: alertspecs.ThresholdDirectionAbove,
|
||
Trigger: 0,
|
||
},
|
||
}
|
||
|
||
if threshold != nil {
|
||
spec.MetricThreshold.Trigger = threshold.Trigger
|
||
if threshold.Clear > 0 && threshold.Clear < threshold.Trigger {
|
||
recovery := threshold.Clear
|
||
spec.MetricThreshold.Recovery = &recovery
|
||
}
|
||
critical := computeCriticalThreshold(threshold.Trigger, metricType)
|
||
spec.MetricThreshold.Critical = &critical
|
||
}
|
||
|
||
return spec, spec.Validate()
|
||
}
|
||
|
||
func (m *Manager) checkMetricWithCanonicalSpec(spec alertspecs.ResourceAlertSpec, resourceName, node, instance, resourceType string, value float64, threshold *HysteresisThreshold, opts *metricOptions) {
|
||
if spec.MetricThreshold == nil {
|
||
return
|
||
}
|
||
m.evaluateCanonicalMetricAlert(spec, resourceName, node, instance, resourceType, value, threshold, opts)
|
||
}
|
||
|
||
func unifiedMetricResourceType(typeKey string) (unifiedresources.ResourceType, bool) {
|
||
switch typeKey {
|
||
case "vm":
|
||
return unifiedresources.ResourceTypeVM, true
|
||
case "system-container":
|
||
return unifiedresources.ResourceTypeSystemContainer, true
|
||
case "app-container":
|
||
return unifiedresources.ResourceTypeAppContainer, true
|
||
case "agent":
|
||
return unifiedresources.ResourceTypeAgent, true
|
||
case "node":
|
||
return unifiedresources.ResourceType("node"), true
|
||
case "pbs":
|
||
return unifiedresources.ResourceTypePBS, true
|
||
case "storage":
|
||
return unifiedresources.ResourceTypeStorage, true
|
||
case "pmg":
|
||
return unifiedresources.ResourceTypePMG, true
|
||
case "k8s-cluster":
|
||
return unifiedresources.ResourceTypeK8sCluster, true
|
||
case "k8s-node":
|
||
return unifiedresources.ResourceTypeK8sNode, true
|
||
case "k8s-deployment":
|
||
return unifiedresources.ResourceTypeK8sDeployment, true
|
||
case "k8s-namespace":
|
||
return unifiedresources.ResourceTypeK8sNamespace, true
|
||
case "pod":
|
||
return unifiedresources.ResourceTypePod, true
|
||
case "truenas-system":
|
||
return unifiedresources.ResourceTypeAgent, true
|
||
case "truenas-pool", "truenas-dataset":
|
||
return unifiedresources.ResourceTypeStorage, true
|
||
case "truenas-disk":
|
||
return unifiedresources.ResourceTypePhysicalDisk, true
|
||
case "vmware-host":
|
||
return unifiedresources.ResourceTypeAgent, true
|
||
case "vmware-vm":
|
||
return unifiedresources.ResourceTypeVM, true
|
||
case "vmware-datastore":
|
||
return unifiedresources.ResourceTypeStorage, true
|
||
case "vmware-network":
|
||
return unifiedresources.ResourceTypeNetwork, true
|
||
default:
|
||
return "", false
|
||
}
|
||
}
|
||
|
||
func (m *Manager) CheckUnifiedResourceMetrics(resources []unifiedresources.Resource) {
|
||
if m == nil {
|
||
return
|
||
}
|
||
for _, resource := range resources {
|
||
input, ok := UnifiedResourceInputFromResource(resource)
|
||
if !ok {
|
||
continue
|
||
}
|
||
m.CheckUnifiedResource(input)
|
||
}
|
||
}
|
||
|
||
func UnifiedResourceInputFromResource(resource unifiedresources.Resource) (*UnifiedResourceInput, bool) {
|
||
typeKey, ok := unifiedAlertResourceType(resource)
|
||
if !ok {
|
||
return nil, false
|
||
}
|
||
resourceID := unifiedresources.CanonicalResourceID(resource.ID)
|
||
if strings.TrimSpace(resourceID) == "" {
|
||
return nil, false
|
||
}
|
||
|
||
input := &UnifiedResourceInput{
|
||
ID: resourceID,
|
||
Type: typeKey,
|
||
Name: unifiedResourceAlertName(resource),
|
||
Node: unifiedResourceAlertNode(resource),
|
||
Instance: unifiedResourceAlertInstance(resource, typeKey),
|
||
}
|
||
|
||
if metrics := resource.Metrics; metrics != nil {
|
||
input.CPU = unifiedResourceMetric(metrics.CPU)
|
||
input.Memory = unifiedResourceMetric(metrics.Memory)
|
||
input.Disk = unifiedResourceMetric(metrics.Disk)
|
||
input.DiskRead = unifiedResourceMetric(metrics.DiskRead)
|
||
input.DiskWrite = unifiedResourceMetric(metrics.DiskWrite)
|
||
input.NetworkIn = unifiedResourceMetric(metrics.NetIn)
|
||
input.NetworkOut = unifiedResourceMetric(metrics.NetOut)
|
||
}
|
||
if resource.Temperature != nil {
|
||
input.Temperature = &UnifiedResourceMetric{Value: *resource.Temperature, Percent: *resource.Temperature}
|
||
} else if resource.PhysicalDisk != nil && resource.PhysicalDisk.Temperature > 0 {
|
||
value := float64(resource.PhysicalDisk.Temperature)
|
||
input.Temperature = &UnifiedResourceMetric{Value: value, Percent: value}
|
||
}
|
||
|
||
return input, true
|
||
}
|
||
|
||
func unifiedAlertResourceType(resource unifiedresources.Resource) (string, bool) {
|
||
resourceType := unifiedresources.CanonicalResourceType(resource.Type)
|
||
switch resourceType {
|
||
case unifiedresources.ResourceTypeK8sCluster,
|
||
unifiedresources.ResourceTypeK8sNode,
|
||
unifiedresources.ResourceTypeK8sDeployment,
|
||
unifiedresources.ResourceTypeK8sNamespace,
|
||
unifiedresources.ResourceTypePod:
|
||
return string(resourceType), true
|
||
case unifiedresources.ResourceTypeAgent:
|
||
if resource.TrueNAS != nil || resourceHasSource(resource, unifiedresources.SourceTrueNAS) {
|
||
return "truenas-system", true
|
||
}
|
||
if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
|
||
return "vmware-host", true
|
||
}
|
||
case unifiedresources.ResourceTypeVM:
|
||
if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
|
||
return "vmware-vm", true
|
||
}
|
||
case unifiedresources.ResourceTypeStorage:
|
||
if isTrueNASStorageResource(resource) {
|
||
switch strings.ToLower(strings.TrimSpace(resource.Storage.Topology)) {
|
||
case "dataset":
|
||
return "truenas-dataset", true
|
||
default:
|
||
return "truenas-pool", true
|
||
}
|
||
}
|
||
if isVMwareDatastoreResource(resource) {
|
||
return "vmware-datastore", true
|
||
}
|
||
case unifiedresources.ResourceTypePhysicalDisk:
|
||
if resourceHasSource(resource, unifiedresources.SourceTrueNAS) || resource.TrueNAS != nil {
|
||
return "truenas-disk", true
|
||
}
|
||
case unifiedresources.ResourceTypeNetwork:
|
||
if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
|
||
return "vmware-network", true
|
||
}
|
||
}
|
||
return "", false
|
||
}
|
||
|
||
func isTrueNASStorageResource(resource unifiedresources.Resource) bool {
|
||
if resource.TrueNAS != nil || resourceHasSource(resource, unifiedresources.SourceTrueNAS) {
|
||
return true
|
||
}
|
||
if resource.Storage == nil {
|
||
return false
|
||
}
|
||
return strings.EqualFold(strings.TrimSpace(resource.Storage.Platform), string(unifiedresources.SourceTrueNAS))
|
||
}
|
||
|
||
func isVMwareDatastoreResource(resource unifiedresources.Resource) bool {
|
||
if resource.VMware != nil || resourceHasSource(resource, unifiedresources.SourceVMware) {
|
||
return true
|
||
}
|
||
if resource.Storage == nil {
|
||
return false
|
||
}
|
||
platform := strings.TrimSpace(resource.Storage.Platform)
|
||
return strings.EqualFold(platform, "vmware-vsphere") || strings.EqualFold(platform, string(unifiedresources.SourceVMware))
|
||
}
|
||
|
||
func resourceHasSource(resource unifiedresources.Resource, source unifiedresources.DataSource) bool {
|
||
for _, candidate := range resource.Sources {
|
||
if candidate == source {
|
||
return true
|
||
}
|
||
}
|
||
if _, ok := resource.SourceStatus[source]; ok {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func unifiedResourceMetric(metric *unifiedresources.MetricValue) *UnifiedResourceMetric {
|
||
if metric == nil {
|
||
return nil
|
||
}
|
||
return &UnifiedResourceMetric{Value: metric.Value, Percent: metric.Percent}
|
||
}
|
||
|
||
func unifiedResourceAlertName(resource unifiedresources.Resource) string {
|
||
if resource.Canonical != nil && strings.TrimSpace(resource.Canonical.DisplayName) != "" {
|
||
return strings.TrimSpace(resource.Canonical.DisplayName)
|
||
}
|
||
if strings.TrimSpace(resource.Name) != "" {
|
||
return strings.TrimSpace(resource.Name)
|
||
}
|
||
return strings.TrimSpace(resource.ID)
|
||
}
|
||
|
||
func unifiedResourceAlertNode(resource unifiedresources.Resource) string {
|
||
if strings.TrimSpace(resource.ParentName) != "" {
|
||
return strings.TrimSpace(resource.ParentName)
|
||
}
|
||
if resource.Canonical != nil && strings.TrimSpace(resource.Canonical.Hostname) != "" {
|
||
return strings.TrimSpace(resource.Canonical.Hostname)
|
||
}
|
||
if resource.Kubernetes != nil {
|
||
if strings.TrimSpace(resource.Kubernetes.NodeName) != "" {
|
||
return strings.TrimSpace(resource.Kubernetes.NodeName)
|
||
}
|
||
if strings.TrimSpace(resource.Kubernetes.ClusterName) != "" {
|
||
return strings.TrimSpace(resource.Kubernetes.ClusterName)
|
||
}
|
||
}
|
||
if resource.TrueNAS != nil && strings.TrimSpace(resource.TrueNAS.Hostname) != "" {
|
||
return strings.TrimSpace(resource.TrueNAS.Hostname)
|
||
}
|
||
if resource.VMware != nil {
|
||
if strings.TrimSpace(resource.VMware.RuntimeHostName) != "" {
|
||
return strings.TrimSpace(resource.VMware.RuntimeHostName)
|
||
}
|
||
if strings.TrimSpace(resource.VMware.ClusterName) != "" {
|
||
return strings.TrimSpace(resource.VMware.ClusterName)
|
||
}
|
||
if strings.TrimSpace(resource.VMware.DatacenterName) != "" {
|
||
return strings.TrimSpace(resource.VMware.DatacenterName)
|
||
}
|
||
if strings.TrimSpace(resource.VMware.VCenterHost) != "" {
|
||
return strings.TrimSpace(resource.VMware.VCenterHost)
|
||
}
|
||
}
|
||
return unifiedResourceAlertName(resource)
|
||
}
|
||
|
||
func unifiedResourceAlertInstance(resource unifiedresources.Resource, typeKey string) string {
|
||
if resource.Kubernetes != nil {
|
||
if strings.TrimSpace(resource.Kubernetes.ClusterName) != "" {
|
||
return strings.TrimSpace(resource.Kubernetes.ClusterName)
|
||
}
|
||
return "Kubernetes"
|
||
}
|
||
if strings.HasPrefix(typeKey, "truenas-") {
|
||
return "TrueNAS"
|
||
}
|
||
if strings.HasPrefix(typeKey, "vmware-") {
|
||
if resource.VMware != nil {
|
||
if strings.TrimSpace(resource.VMware.ConnectionName) != "" {
|
||
return strings.TrimSpace(resource.VMware.ConnectionName)
|
||
}
|
||
if strings.TrimSpace(resource.VMware.VCenterHost) != "" {
|
||
return strings.TrimSpace(resource.VMware.VCenterHost)
|
||
}
|
||
}
|
||
return "vSphere"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func (i *UnifiedResourceInput) CPUValue() float64 {
|
||
if i == nil || i.CPU == nil {
|
||
return 0
|
||
}
|
||
return i.CPU.Percent
|
||
}
|
||
|
||
func (i *UnifiedResourceInput) MemoryValue() float64 {
|
||
if i == nil || i.Memory == nil {
|
||
return 0
|
||
}
|
||
return i.Memory.Percent
|
||
}
|
||
|
||
func (i *UnifiedResourceInput) DiskValue() float64 {
|
||
if i == nil || i.Disk == nil {
|
||
return 0
|
||
}
|
||
return i.Disk.Percent
|
||
}
|
||
|
||
func (i *UnifiedResourceInput) DiskReadValue() float64 {
|
||
if i == nil || i.DiskRead == nil {
|
||
return 0
|
||
}
|
||
return i.DiskRead.Value
|
||
}
|
||
|
||
func (i *UnifiedResourceInput) DiskWriteValue() float64 {
|
||
if i == nil || i.DiskWrite == nil {
|
||
return 0
|
||
}
|
||
return i.DiskWrite.Value
|
||
}
|
||
|
||
func (i *UnifiedResourceInput) NetworkInValue() float64 {
|
||
if i == nil || i.NetworkIn == nil {
|
||
return 0
|
||
}
|
||
return i.NetworkIn.Value
|
||
}
|
||
|
||
func (i *UnifiedResourceInput) NetworkOutValue() float64 {
|
||
if i == nil || i.NetworkOut == nil {
|
||
return 0
|
||
}
|
||
return i.NetworkOut.Value
|
||
}
|
||
|
||
func (i *UnifiedResourceInput) TemperatureValue() float64 {
|
||
if i == nil || i.Temperature == nil {
|
||
return 0
|
||
}
|
||
return i.Temperature.Value
|
||
}
|
||
|
||
// CheckUnifiedResource evaluates threshold-based metric alerts for a unified resource.
|
||
// It resolves thresholds (defaults + overrides) and calls checkMetric for each
|
||
// available metric. Discrete event alerts (offline, RAID, backup age, etc.)
|
||
// are NOT evaluated here — they remain in the typed Check* methods.
|
||
func (m *Manager) CheckUnifiedResource(input *UnifiedResourceInput) {
|
||
if input == nil {
|
||
return
|
||
}
|
||
|
||
m.mu.RLock()
|
||
if !m.config.Enabled {
|
||
m.mu.RUnlock()
|
||
return
|
||
}
|
||
if m.unifiedPlatformAlertsDisabledNoLock(input.Type) {
|
||
m.mu.RUnlock()
|
||
return
|
||
}
|
||
|
||
thresholds := m.resolveResourceThresholds(input.Type, input.ID)
|
||
m.mu.RUnlock()
|
||
|
||
if thresholds.Disabled {
|
||
return
|
||
}
|
||
|
||
log.Debug().
|
||
Str("resourceID", input.ID).
|
||
Str("resourceName", input.Name).
|
||
Str("resourceType", unifiedAlertType(input.Type)).
|
||
Msg("Evaluating unified resource metrics")
|
||
|
||
m.evaluateUnifiedMetrics(input, thresholds, nil)
|
||
}
|