mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 14:00:29 +00:00
129 lines
4.6 KiB
Go
129 lines
4.6 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
const backupIntentEvidenceMaxAge = 5 * time.Minute
|
|
|
|
type resourceOperatorIntentReader interface {
|
|
GetResourceOperatorState(canonicalID string) (unifiedresources.ResourceOperatorState, bool, error)
|
|
}
|
|
|
|
type resourceIntentIdentityReader interface {
|
|
ResolveCanonicalResourceID(ref string) (string, bool)
|
|
}
|
|
|
|
type resourceIntentAncestorReader interface {
|
|
ResolveCanonicalResourceAncestors(ref string) []string
|
|
}
|
|
|
|
func (m *Monitor) installOperatorIntentResolver(store ResourceStoreInterface) {
|
|
if m == nil || m.alertManager == nil {
|
|
return
|
|
}
|
|
identityReader, hasIdentityReader := store.(resourceIntentIdentityReader)
|
|
if hasIdentityReader && identityReader != nil {
|
|
m.alertManager.SetResourceIntentIdentityResolver(identityReader.ResolveCanonicalResourceID)
|
|
} else {
|
|
m.alertManager.SetResourceIntentIdentityResolver(nil)
|
|
}
|
|
reader, ok := store.(resourceOperatorIntentReader)
|
|
if !ok || reader == nil {
|
|
m.alertManager.SetOperatorIntentContextResolver(nil)
|
|
return
|
|
}
|
|
ancestorReader, hasAncestorReader := store.(resourceIntentAncestorReader)
|
|
m.alertManager.SetOperatorIntentContextResolver(func(resourceID string, now time.Time) (alerts.OperatorIntentContext, bool) {
|
|
if hasIdentityReader {
|
|
if canonicalID, found := identityReader.ResolveCanonicalResourceID(resourceID); found {
|
|
resourceID = canonicalID
|
|
}
|
|
}
|
|
state, found, err := reader.GetResourceOperatorState(resourceID)
|
|
if err != nil {
|
|
log.Warn().Err(err).Str("resourceID", resourceID).Msg("Failed to read operator state for alert intent")
|
|
return alerts.OperatorIntentContext{}, false
|
|
}
|
|
context := alerts.OperatorIntentContext{}
|
|
if found {
|
|
context = alerts.OperatorIntentContext{
|
|
IntentionallyOffline: state.IntentionallyOffline,
|
|
MonitoringMode: string(state.MonitoringMode),
|
|
LifecycleState: string(state.LifecycleState),
|
|
}
|
|
}
|
|
|
|
applyActiveMaintenance := func(candidate unifiedresources.ResourceOperatorState, sourceID string, inherited bool) {
|
|
occurrence, active := candidate.ActiveMaintenanceOccurrenceAt(now)
|
|
if !active {
|
|
return
|
|
}
|
|
// Overlapping exact/inherited schedules remain suppressed until the
|
|
// latest active end. This avoids promising an early delivery resume.
|
|
if context.MaintenanceEndAt != nil && !occurrence.EndAt.After(*context.MaintenanceEndAt) {
|
|
return
|
|
}
|
|
startAt, endAt := occurrence.StartAt, occurrence.EndAt
|
|
context.MaintenanceStartAt = &startAt
|
|
context.MaintenanceEndAt = &endAt
|
|
context.MaintenanceReason = candidate.MaintenanceReason
|
|
context.MaintenanceSourceID = sourceID
|
|
context.MaintenanceInherited = inherited
|
|
context.MaintenanceScope = string(candidate.MaintenanceScope)
|
|
}
|
|
if found {
|
|
applyActiveMaintenance(state, resourceID, false)
|
|
}
|
|
if hasAncestorReader {
|
|
for _, ancestorID := range ancestorReader.ResolveCanonicalResourceAncestors(resourceID) {
|
|
ancestorState, ancestorFound, ancestorErr := reader.GetResourceOperatorState(ancestorID)
|
|
if ancestorErr != nil {
|
|
log.Warn().Err(ancestorErr).Str("resourceID", resourceID).Str("ancestorID", ancestorID).Msg("Failed to read inherited operator maintenance state")
|
|
continue
|
|
}
|
|
if !ancestorFound || ancestorState.MaintenanceScope != unifiedresources.MaintenanceScopeResourceAndDescendants {
|
|
continue
|
|
}
|
|
applyActiveMaintenance(ancestorState, ancestorID, true)
|
|
}
|
|
}
|
|
return context, found || context.MaintenanceEndAt != nil
|
|
})
|
|
}
|
|
|
|
func (m *Monitor) resolveBackupIntentContext(_ string, instance, node string, vmid int, now time.Time) (alerts.BackupIntentContext, bool) {
|
|
if m == nil || m.state == nil || vmid <= 0 {
|
|
return alerts.BackupIntentContext{}, false
|
|
}
|
|
instance = strings.TrimSpace(instance)
|
|
node = strings.TrimSpace(node)
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
|
|
for _, task := range m.state.GetSnapshot().PVEBackups.BackupTasks {
|
|
if task.VMID != vmid || (instance != "" && task.Instance != instance) || (node != "" && task.Node != "" && task.Node != node) {
|
|
continue
|
|
}
|
|
if task.ObservedAt.IsZero() || now.Sub(task.ObservedAt) > backupIntentEvidenceMaxAge || task.ObservedAt.After(now.Add(time.Minute)) {
|
|
continue
|
|
}
|
|
status := strings.ToLower(strings.TrimSpace(task.Status))
|
|
if !task.EndTime.IsZero() || status == "ok" || status == "stopped" || status == "error" || status == "warning" {
|
|
continue
|
|
}
|
|
return alerts.BackupIntentContext{
|
|
Active: true,
|
|
ObservedAt: task.ObservedAt,
|
|
Evidence: "pve_vzdump_task:" + task.ID,
|
|
}, true
|
|
}
|
|
return alerts.BackupIntentContext{}, false
|
|
}
|