mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-20 02:42:11 +00:00
rcourtman
7ce2cfaa66
Move guest metric projection, per-disk guest metric evaluation, powered-off lifecycle alerts, Pulse tag controls, relaxed thresholds, and guest suppression cleanup into internal/alerts/guest.go. Record guest.go as the Proxmox guest alert owner in the alerts subsystem contract and add a Pulse-tag characterization test. Proof: go test ./internal/alerts/...
605 lines
17 KiB
Go
605 lines
17 KiB
Go
package alerts
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// ReevaluateGuestAlert reevaluates a specific guest's alerts with full threshold resolution including custom rules.
|
|
// This should be called by the monitor with the current guest state.
|
|
func (m *Manager) ReevaluateGuestAlert(guest any, guestID string) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
// Get the correct thresholds for this guest (includes custom rules evaluation)
|
|
thresholds := m.getGuestThresholds(guest, guestID)
|
|
|
|
// Check all metric types for this guest
|
|
metricTypes := []string{"cpu", "memory", "disk", "diskRead", "diskWrite", "networkIn", "networkOut"}
|
|
|
|
for _, metricType := range metricTypes {
|
|
alertID := canonicalMetricStateID(guestID, metricType)
|
|
alert, exists := m.getActiveAlertNoLock(alertID)
|
|
if !exists {
|
|
alert, exists = m.getActiveAlertNoLock(fmt.Sprintf("%s-%s", guestID, metricType))
|
|
}
|
|
if !exists {
|
|
continue
|
|
}
|
|
trackingKey := canonicalTrackingKeyForAlert(alert)
|
|
if trackingKey == "" {
|
|
trackingKey = alertID
|
|
}
|
|
|
|
// Get the threshold for this metric
|
|
var threshold *HysteresisThreshold
|
|
switch metricType {
|
|
case "cpu":
|
|
threshold = thresholds.CPU
|
|
case "memory":
|
|
threshold = thresholds.Memory
|
|
case "disk":
|
|
threshold = thresholds.Disk
|
|
case "diskRead":
|
|
threshold = thresholds.DiskRead
|
|
case "diskWrite":
|
|
threshold = thresholds.DiskWrite
|
|
case "networkIn":
|
|
threshold = thresholds.NetworkIn
|
|
case "networkOut":
|
|
threshold = thresholds.NetworkOut
|
|
}
|
|
|
|
// If threshold is disabled or doesn't exist, clear the alert
|
|
if threshold == nil || threshold.Trigger <= 0 {
|
|
m.clearAlertNoLock(trackingKey)
|
|
// Also clear any pending alert for this metric
|
|
if _, isPending := m.pendingAlerts[trackingKey]; isPending {
|
|
delete(m.pendingAlerts, trackingKey)
|
|
log.Debug().
|
|
Str("alertID", alertID).
|
|
Msg("Cleared pending alert - threshold disabled")
|
|
}
|
|
log.Info().
|
|
Str("alertID", alertID).
|
|
Str("metric", metricType).
|
|
Msg("Cleared alert - threshold disabled")
|
|
continue
|
|
}
|
|
|
|
// Check if alert should be cleared based on new threshold
|
|
clearThreshold := threshold.Clear
|
|
if clearThreshold <= 0 {
|
|
clearThreshold = threshold.Trigger
|
|
}
|
|
|
|
if alert.Value <= clearThreshold || alert.Value < threshold.Trigger {
|
|
m.clearAlertNoLock(trackingKey)
|
|
log.Info().
|
|
Str("alertID", alertID).
|
|
Str("metric", metricType).
|
|
Float64("value", alert.Value).
|
|
Float64("trigger", threshold.Trigger).
|
|
Float64("clear", clearThreshold).
|
|
Msg("Cleared alert - value now below threshold after config change")
|
|
}
|
|
}
|
|
}
|
|
|
|
// CheckGuest checks a guest (VM or container) against thresholds.
|
|
func (m *Manager) CheckGuest(guest any, instanceName string) {
|
|
m.mu.RLock()
|
|
enabled := m.config.Enabled
|
|
disableAllGuests := m.config.DisableAllGuests
|
|
disableAllGuestsOffline := m.config.DisableAllGuestsOffline
|
|
ignoredGuestPrefixes := m.config.IgnoredGuestPrefixes
|
|
guestTagWhitelist := m.config.GuestTagWhitelist
|
|
guestTagBlacklist := m.config.GuestTagBlacklist
|
|
m.mu.RUnlock()
|
|
|
|
if !enabled {
|
|
log.Debug().Msg("checkGuest: alerts disabled globally")
|
|
return
|
|
}
|
|
if disableAllGuests {
|
|
log.Debug().Msg("checkGuest: all guest alerts disabled")
|
|
return
|
|
}
|
|
|
|
snapshot, ok := extractGuestSnapshot(guest)
|
|
if !ok {
|
|
log.Debug().
|
|
Str("type", fmt.Sprintf("%T", guest)).
|
|
Msg("CheckGuest: unsupported guest type")
|
|
return
|
|
}
|
|
|
|
guestID := snapshot.ID
|
|
name := snapshot.Name
|
|
node := snapshot.Node
|
|
guestType := snapshot.displayType()
|
|
status := snapshot.Status
|
|
cpu := snapshot.CPUPercent
|
|
memUsage := snapshot.MemUsage
|
|
diskUsage := snapshot.DiskUsage
|
|
diskRead := snapshot.DiskRead
|
|
diskWrite := snapshot.DiskWrite
|
|
netIn := snapshot.NetworkIn
|
|
netOut := snapshot.NetworkOut
|
|
disks := snapshot.Disks
|
|
tags := snapshot.Tags
|
|
|
|
// Debug logging for high memory VMs
|
|
if snapshot.Kind == guestKindVM && memUsage > 85 {
|
|
log.Debug().
|
|
Str("vm", name).
|
|
Float64("memUsage", memUsage).
|
|
Str("status", status).
|
|
Msg("VM with high memory detected in CheckGuest")
|
|
}
|
|
|
|
// Check ignored prefixes
|
|
for _, prefix := range ignoredGuestPrefixes {
|
|
if prefix != "" && strings.HasPrefix(name, prefix) {
|
|
if cleared := m.suppressGuestAlerts(guestID); cleared {
|
|
m.saveActiveAlertsAsync("ignored-prefix")
|
|
}
|
|
return
|
|
}
|
|
}
|
|
|
|
settings := parsePulseTags(tags)
|
|
if settings.Suppress {
|
|
if cleared := m.suppressGuestAlerts(guestID); cleared {
|
|
m.saveActiveAlertsAsync("pulse-no-alerts")
|
|
}
|
|
log.Debug().
|
|
Str("guestID", guestID).
|
|
Msg("Pulse no-alerts tag active; suppressing guest alerts")
|
|
return
|
|
}
|
|
|
|
// Custom Tag Filtering
|
|
if len(guestTagBlacklist) > 0 || len(guestTagWhitelist) > 0 {
|
|
// Normalize tags once for checking
|
|
normalizedTags := make(map[string]bool)
|
|
for _, tag := range tags {
|
|
normalizedTags[strings.ToLower(strings.TrimSpace(tag))] = true
|
|
}
|
|
|
|
// Check Blacklist
|
|
for _, block := range guestTagBlacklist {
|
|
if normalizedTags[strings.ToLower(strings.TrimSpace(block))] {
|
|
if cleared := m.suppressGuestAlerts(guestID); cleared {
|
|
m.saveActiveAlertsAsync("tag-blacklist")
|
|
}
|
|
log.Debug().Str("guestID", guestID).Msg("guest suppressed by tag blacklist")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Check Whitelist
|
|
if len(guestTagWhitelist) > 0 {
|
|
found := false
|
|
for _, allow := range guestTagWhitelist {
|
|
if normalizedTags[strings.ToLower(strings.TrimSpace(allow))] {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
if cleared := m.suppressGuestAlerts(guestID); cleared {
|
|
m.saveActiveAlertsAsync("tag-whitelist")
|
|
}
|
|
log.Debug().Str("guestID", guestID).Msg("guest suppressed by tag whitelist (required tag not found)")
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
monitorOnly := settings.MonitorOnly
|
|
if monitorOnly || m.guestHasMonitorOnlyAlerts(guestID) {
|
|
log.Debug().
|
|
Str("guest", name).
|
|
Bool("monitorOnly", monitorOnly).
|
|
Msg("Pulse monitor-only status applied")
|
|
}
|
|
|
|
// Handle non-running guests
|
|
// Proxmox VM states: running, stopped, paused, suspended
|
|
if status != "running" {
|
|
// Check for powered-off state and generate alert if configured
|
|
if status == "stopped" {
|
|
if disableAllGuestsOffline {
|
|
// Clear any pending powered-off tracking and alerts when globally disabled
|
|
m.mu.Lock()
|
|
delete(m.offlineConfirmations, guestID)
|
|
m.mu.Unlock()
|
|
m.clearAlert(canonicalPoweredStateStateID(guestID))
|
|
} else {
|
|
m.mu.RLock()
|
|
thresholds := m.getGuestThresholds(guest, guestID)
|
|
m.mu.RUnlock()
|
|
m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly)
|
|
}
|
|
} else {
|
|
// For paused/suspended, clear powered-off alert
|
|
m.clearGuestPoweredOffAlert(guestID, name)
|
|
}
|
|
|
|
alertsCleared := m.clearGuestMetricAlerts(guestID)
|
|
|
|
if alertsCleared > 0 {
|
|
log.Debug().
|
|
Str("guest", name).
|
|
Str("status", status).
|
|
Int("alertsCleared", alertsCleared).
|
|
Msg("Cleared metric alerts for non-running guest")
|
|
m.saveActiveAlertsAsync("guest-not-running")
|
|
}
|
|
return
|
|
}
|
|
|
|
// If guest is running, clear any powered-off alert
|
|
m.clearGuestPoweredOffAlert(guestID, name)
|
|
|
|
// Get thresholds (check custom rules, then overrides, then defaults)
|
|
m.mu.RLock()
|
|
thresholds := m.getGuestThresholds(guest, guestID)
|
|
m.mu.RUnlock()
|
|
|
|
if settings.Relaxed {
|
|
thresholds = applyRelaxedGuestThresholds(thresholds)
|
|
log.Info().
|
|
Str("guest", name).
|
|
Float64("trigger", thresholds.CPU.Trigger).
|
|
Msg("Applied relaxed thresholds for pulse-relaxed tag")
|
|
}
|
|
|
|
// If alerts are disabled for this guest, clear any existing alerts and return
|
|
if thresholds.Disabled {
|
|
if alertsCleared := m.clearGuestMetricAlerts(guestID); alertsCleared > 0 {
|
|
log.Info().
|
|
Str("guest", name).
|
|
Int("alertsCleared", alertsCleared).
|
|
Msg("Cleared guest metric alerts because alerts are disabled")
|
|
m.saveActiveAlertsAsync("guest-alerts-disabled")
|
|
}
|
|
return
|
|
}
|
|
|
|
// Check each metric
|
|
log.Debug().
|
|
Str("guest", name).
|
|
Float64("cpu", cpu).
|
|
Float64("memory", memUsage).
|
|
Float64("disk", diskUsage).
|
|
Interface("thresholds", thresholds).
|
|
Msg("Checking guest thresholds")
|
|
|
|
// Evaluate standard metrics through unified path
|
|
var evalOpts *metricOptions
|
|
if monitorOnly {
|
|
evalOpts = &metricOptions{MonitorOnly: true}
|
|
}
|
|
m.evaluateUnifiedMetrics(&UnifiedResourceInput{
|
|
ID: guestID,
|
|
Type: snapshot.resourceType(),
|
|
Name: name,
|
|
Node: node,
|
|
Instance: instanceName,
|
|
CPU: &UnifiedResourceMetric{Percent: cpu},
|
|
Memory: &UnifiedResourceMetric{Percent: memUsage},
|
|
Disk: &UnifiedResourceMetric{Percent: diskUsage},
|
|
DiskRead: &UnifiedResourceMetric{Value: float64(diskRead) / 1024 / 1024},
|
|
DiskWrite: &UnifiedResourceMetric{Value: float64(diskWrite) / 1024 / 1024},
|
|
NetworkIn: &UnifiedResourceMetric{Value: float64(netIn) / 1024 / 1024},
|
|
NetworkOut: &UnifiedResourceMetric{Value: float64(netOut) / 1024 / 1024},
|
|
}, thresholds, evalOpts)
|
|
|
|
if thresholds.Disk != nil && thresholds.Disk.Trigger > 0 && len(disks) > 0 {
|
|
seenDiskKeys := make(map[string]struct{})
|
|
seenDiskResources := make(map[string]struct{})
|
|
for idx, disk := range disks {
|
|
if disk.Total <= 0 {
|
|
continue
|
|
}
|
|
if disk.Usage < 0 {
|
|
continue
|
|
}
|
|
|
|
label := strings.TrimSpace(disk.Mountpoint)
|
|
if label == "" {
|
|
label = strings.TrimSpace(disk.Device)
|
|
}
|
|
if label == "" {
|
|
label = fmt.Sprintf("Disk %d", idx+1)
|
|
}
|
|
|
|
keySource := label
|
|
if disk.Device != "" && !strings.EqualFold(disk.Device, label) {
|
|
keySource = fmt.Sprintf("%s-%s", label, disk.Device)
|
|
}
|
|
sanitizedKey := sanitizeAlertKey(keySource)
|
|
if sanitizedKey == "" {
|
|
sanitizedKey = fmt.Sprintf("disk-%d", idx+1)
|
|
}
|
|
|
|
// Avoid duplicate checks if two disks resolve to the same key
|
|
if _, exists := seenDiskKeys[sanitizedKey]; exists {
|
|
continue
|
|
}
|
|
seenDiskKeys[sanitizedKey] = struct{}{}
|
|
|
|
perDiskResourceID := fmt.Sprintf("%s-disk-%s", guestID, sanitizedKey)
|
|
seenDiskResources[perDiskResourceID] = struct{}{}
|
|
message := fmt.Sprintf("%s disk (%s) at %.1f%%", guestType, label, disk.Usage)
|
|
|
|
log.Debug().
|
|
Str("guest", name).
|
|
Str("node", node).
|
|
Str("instance", instanceName).
|
|
Str("diskLabel", label).
|
|
Float64("usage", disk.Usage).
|
|
Msg("Evaluating individual disk for alert thresholds")
|
|
|
|
metadata := map[string]interface{}{
|
|
"mountpoint": disk.Mountpoint,
|
|
"device": disk.Device,
|
|
"diskType": disk.Type,
|
|
"totalBytes": disk.Total,
|
|
"usedBytes": disk.Used,
|
|
"freeBytes": disk.Free,
|
|
"diskIndex": idx,
|
|
"label": label,
|
|
}
|
|
resourceType, ok := unifiedMetricResourceType(snapshot.resourceType())
|
|
if !ok {
|
|
m.checkMetric(perDiskResourceID, name, node, instanceName, snapshot.resourceType(), "disk", disk.Usage, thresholds.Disk, &metricOptions{
|
|
Metadata: metadata,
|
|
Message: message,
|
|
MonitorOnly: monitorOnly,
|
|
})
|
|
continue
|
|
}
|
|
spec, err := buildCanonicalMetricSpec(perDiskResourceID, name, resourceType, "disk", thresholds.Disk)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resourceID", perDiskResourceID).
|
|
Str("guest", name).
|
|
Msg("Skipping invalid canonical guest disk metric spec")
|
|
continue
|
|
}
|
|
|
|
m.checkMetricWithCanonicalSpec(spec, name, node, instanceName, snapshot.resourceType(), disk.Usage, thresholds.Disk, &metricOptions{
|
|
Metadata: metadata,
|
|
Message: message,
|
|
MonitorOnly: monitorOnly,
|
|
})
|
|
}
|
|
if cleared := m.cleanupGuestDiskAlerts(guestID, seenDiskResources); cleared > 0 {
|
|
m.saveActiveAlertsAsync("guest-disk-set-changed")
|
|
}
|
|
} else if cleared := m.cleanupGuestDiskAlerts(guestID, nil); cleared > 0 {
|
|
m.saveActiveAlertsAsync("guest-disk-alerts-cleared")
|
|
}
|
|
}
|
|
|
|
// checkGuestPoweredOff creates an alert for powered-off guests.
|
|
func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestType string, monitorOnly bool) {
|
|
m.mu.RLock()
|
|
thresholds := m.getGuestThresholds(guestSnapshotFromIdentity(guestID, name, node, instanceName, guestType, "stopped"), guestID)
|
|
m.mu.RUnlock()
|
|
m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly)
|
|
}
|
|
|
|
func (m *Manager) checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool) {
|
|
alertID := fmt.Sprintf("guest-powered-off-%s", guestID)
|
|
severity := NormalizePoweredOffSeverity(thresholds.PoweredOffSeverity)
|
|
resourceType := unifiedresources.ResourceTypeVM
|
|
if strings.EqualFold(guestType, "container") {
|
|
resourceType = unifiedresources.ResourceTypeSystemContainer
|
|
}
|
|
spec, err := buildCanonicalPoweredStateSpec(guestID, name, resourceType, severity, 2, thresholds.Disabled || thresholds.DisableConnectivity)
|
|
if err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("guest", name).
|
|
Str("guestID", guestID).
|
|
Msg("Skipping invalid canonical guest powered-state spec")
|
|
return
|
|
}
|
|
|
|
m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
|
|
Spec: spec,
|
|
Evidence: alertspecs.AlertEvidence{
|
|
ObservedAt: time.Now(),
|
|
PoweredState: &alertspecs.PoweredStateEvidence{
|
|
Expected: alertspecs.PowerStateOn,
|
|
Observed: alertspecs.PowerStateOff,
|
|
},
|
|
},
|
|
Tracking: m.offlineConfirmations,
|
|
TrackingKey: guestID,
|
|
AlertID: alertID,
|
|
AlertType: "powered-off",
|
|
ResourceID: guestID,
|
|
ResourceName: name,
|
|
Node: node,
|
|
Instance: instanceName,
|
|
Message: fmt.Sprintf("%s '%s' is powered off", guestType, name),
|
|
Metadata: map[string]interface{}{
|
|
"monitorOnly": monitorOnly,
|
|
"resourceType": strings.ToLower(guestType),
|
|
},
|
|
AddToRecent: true,
|
|
AddToHistory: true,
|
|
DispatchAsync: false,
|
|
})
|
|
}
|
|
|
|
// clearGuestPoweredOffAlert removes powered-off alert when guest starts running.
|
|
func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) {
|
|
alertID := canonicalPoweredStateStateID(guestID)
|
|
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
// Reset confirmation count when guest comes back online
|
|
if count, exists := m.offlineConfirmations[guestID]; exists && count > 0 {
|
|
log.Debug().
|
|
Str("guest", name).
|
|
Int("previousCount", count).
|
|
Msg("Guest is running, resetting powered-off confirmation count")
|
|
delete(m.offlineConfirmations, guestID)
|
|
}
|
|
|
|
// Check if powered-off alert exists
|
|
alert, exists := m.getActiveAlertNoLock(alertID)
|
|
if !exists {
|
|
return
|
|
}
|
|
|
|
// Remove from active alerts
|
|
m.removeActiveAlertNoLock(alertID)
|
|
|
|
downtime := time.Since(alert.StartTime)
|
|
resolvedAlert := &ResolvedAlert{
|
|
Alert: alert,
|
|
ResolvedTime: time.Now(),
|
|
}
|
|
m.addRecentlyResolvedWithPrimaryLock(resolvedAlert)
|
|
|
|
// Send recovery notification (async to avoid deadlock because callback acquires m.mu.RLock
|
|
// via ShouldSuppressResolvedNotification, and we currently hold m.mu.Lock)
|
|
m.safeCallResolvedAlertCallback(alert, effectiveAlertID(alert, alertID), true)
|
|
|
|
// Log recovery
|
|
log.Info().
|
|
Str("guest", name).
|
|
Dur("downtime", downtime).
|
|
Msg("Guest is now running")
|
|
}
|
|
|
|
type pulseTagSettings struct {
|
|
Suppress bool
|
|
MonitorOnly bool
|
|
Relaxed bool
|
|
}
|
|
|
|
func parsePulseTags(tags []string) pulseTagSettings {
|
|
settings := pulseTagSettings{}
|
|
for _, raw := range tags {
|
|
tag := strings.TrimSpace(strings.ToLower(raw))
|
|
switch tag {
|
|
case "pulse-no-alerts":
|
|
settings.Suppress = true
|
|
case "pulse-monitor-only":
|
|
settings.MonitorOnly = true
|
|
case "pulse-relaxed":
|
|
settings.Relaxed = true
|
|
}
|
|
}
|
|
return settings
|
|
}
|
|
|
|
func applyRelaxedGuestThresholds(cfg ThresholdConfig) ThresholdConfig {
|
|
relaxed := cloneThresholdConfig(cfg)
|
|
|
|
adjust := func(th **HysteresisThreshold, minTrigger float64) {
|
|
if *th == nil {
|
|
*th = &HysteresisThreshold{Trigger: minTrigger, Clear: minTrigger - 5}
|
|
return
|
|
}
|
|
ensureHysteresisThreshold(*th)
|
|
if (*th).Trigger < minTrigger {
|
|
(*th).Trigger = minTrigger
|
|
}
|
|
if (*th).Clear >= (*th).Trigger {
|
|
(*th).Clear = (*th).Trigger - 5
|
|
}
|
|
if (*th).Clear < 0 {
|
|
(*th).Clear = 0
|
|
}
|
|
}
|
|
|
|
adjust(&relaxed.CPU, 95)
|
|
adjust(&relaxed.Memory, 92)
|
|
adjust(&relaxed.Disk, 95)
|
|
|
|
return relaxed
|
|
}
|
|
|
|
func (m *Manager) suppressGuestAlerts(guestID string) bool {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
cleared := false
|
|
|
|
for storageKey, alert := range m.activeAlerts {
|
|
alertID := effectiveAlertID(alert, storageKey)
|
|
trackingKey := canonicalTrackingKeyForAlert(alert)
|
|
if alert == nil {
|
|
continue
|
|
}
|
|
if alert.ResourceID == guestID || strings.HasPrefix(alert.ResourceID, guestID+"/") || strings.HasPrefix(alertID, guestID) {
|
|
m.clearAlertNoLock(alertID)
|
|
delete(m.recentAlerts, trackingKey)
|
|
delete(m.pendingAlerts, trackingKey)
|
|
delete(m.suppressedUntil, trackingKey)
|
|
delete(m.alertRateLimit, trackingKey)
|
|
cleared = true
|
|
}
|
|
}
|
|
|
|
for key := range m.pendingAlerts {
|
|
if strings.HasPrefix(key, guestID) {
|
|
delete(m.pendingAlerts, key)
|
|
}
|
|
}
|
|
for key := range m.recentAlerts {
|
|
if strings.HasPrefix(key, guestID) {
|
|
delete(m.recentAlerts, key)
|
|
}
|
|
}
|
|
for key := range m.suppressedUntil {
|
|
if strings.HasPrefix(key, guestID) {
|
|
delete(m.suppressedUntil, key)
|
|
}
|
|
}
|
|
for key := range m.alertRateLimit {
|
|
if strings.HasPrefix(key, guestID) {
|
|
delete(m.alertRateLimit, key)
|
|
}
|
|
}
|
|
|
|
delete(m.offlineConfirmations, guestID)
|
|
|
|
return cleared
|
|
}
|
|
|
|
func (m *Manager) guestHasMonitorOnlyAlerts(guestID string) bool {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
|
|
for _, alert := range m.activeAlerts {
|
|
if alert == nil {
|
|
continue
|
|
}
|
|
if alert.ResourceID != guestID {
|
|
continue
|
|
}
|
|
if isMonitorOnlyAlert(alert) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|