pulse/internal/alerts/escalation.go

package alerts

import (
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rs/zerolog/log"
)

// escalationChecker runs periodically to check for alerts that need escalation and cleanup
func (m *Manager) escalationChecker() {
	ticker := time.NewTicker(1 * time.Minute)
	cleanupTicker := time.NewTicker(10 * time.Minute) // Run cleanup every 10 minutes
	defer ticker.Stop()
	defer cleanupTicker.Stop()

	for {
		select {
		case <-ticker.C:
			m.checkEscalations()
		case <-cleanupTicker.C:
			m.Cleanup(24 * time.Hour) // Clean up acknowledged alerts older than 24 hours
		case <-m.escalationStop:
			return
		}
	}
}

// checkEscalations checks all active alerts for escalation
func (m *Manager) checkEscalations() {
	m.expireOperationalSuppressions()
	m.mu.Lock()
	defer m.mu.Unlock()

	// Respect global alert and activation controls before escalating.
	// Escalations should never bypass a user disabling alerts.
	if !m.config.Enabled || m.config.ActivationState != ActivationActive {
		return
	}

	if !m.config.Schedule.Escalation.Enabled {
		return
	}

	now := m.policyNow()
	for _, alert := range m.activeAlerts {
		// A supported symptom has a primary delivery owner. Like monitor-only
		// alerts, it stays visible without its own escalation or critical repeat.
		if alert == nil || alert.Acknowledged || isMonitorOnlyAlert(alert) || isSupportedInfrastructureSymptom(alert) {
			continue
		}
		if m.awaitsBackupRevalidationNoLock(alert) {
			continue
		}
		if _, snoozed := alertSnoozeUntil(alert, now); snoozed {
			continue
		}

		// Check each escalation level
		escalationBaseline := alert.StartTime
		if transition := alert.LatestTransition; transition != nil &&
			transition.Cause == operationaltrust.TransitionSuppressionExpired &&
			transition.At.After(escalationBaseline) {
			// Snoozed escalation levels are deliberately skipped, not replayed in
			// a burst. After resume, the remaining escalation schedule starts
			// from the explicit resume transition while incident age is preserved.
			escalationBaseline = transition.At
		}
		for i, level := range m.config.Schedule.Escalation.Levels {
			// Skip if we've already escalated to this level
			if alert.LastEscalation >= i+1 {
				continue
			}

			// Check if it's time to escalate
			escalateTime := escalationBaseline.Add(time.Duration(level.After) * time.Minute)
			if now.After(escalateTime) {
				// Update alert escalation state
				alert.LastEscalation = i + 1
				alert.EscalationTimes = append(alert.EscalationTimes, now)

				log.Info().
					Str("alertID", alert.ID).
					Str("trackingKey", canonicalTrackingKeyForAlert(alert)).
					Int("level", i+1).
					Str("notify", level.Notify).
					Msg("Alert escalated")

				// Trigger escalation callback
				m.safeCallEscalateCallback(alert, i+1)
			}
		}

		escalation := m.config.Schedule.Escalation
		if !escalation.RepeatCritical || alert.Level != AlertLevelCritical || len(escalation.Levels) == 0 || alert.LastEscalation < len(escalation.Levels) {
			continue
		}
		repeatEvery := time.Duration(escalation.RepeatEvery) * time.Minute
		if repeatEvery <= 0 {
			continue
		}
		lastLevel := escalation.Levels[len(escalation.Levels)-1]
		repeatBaseline := escalationBaseline.Add(time.Duration(lastLevel.After) * time.Minute)
		if count := len(alert.EscalationTimes); count > 0 && alert.EscalationTimes[count-1].After(repeatBaseline) {
			repeatBaseline = alert.EscalationTimes[count-1]
		}
		if !now.After(repeatBaseline.Add(repeatEvery)) {
			continue
		}

		alert.EscalationTimes = append(alert.EscalationTimes, now)
		log.Info().
			Str("alertID", alert.ID).
			Str("trackingKey", canonicalTrackingKeyForAlert(alert)).
			Int("level", len(escalation.Levels)).
			Int("repeatEveryMinutes", escalation.RepeatEvery).
			Msg("Critical alert escalation repeated")
		m.safeCallEscalationRepeatCallback(alert, len(escalation.Levels))
	}
}

// PrepareEscalationNotification revalidates an asynchronously dispatched
// escalation against the current policy and occurrence. The callback snapshot
// is not authority to send after recovery, acknowledgement or a policy edit.
// This does not recall a notification already handed to a destination.
func (m *Manager) PrepareEscalationNotification(snapshot *Alert, level int) (*Alert, EscalationLevel, bool) {
	if m == nil || snapshot == nil {
		return nil, EscalationLevel{}, false
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	if !m.config.Enabled || m.config.ActivationState != ActivationActive || !m.config.Schedule.Escalation.Enabled || level <= 0 || level > len(m.config.Schedule.Escalation.Levels) {
		return nil, EscalationLevel{}, false
	}
	active, ok := m.getActiveAlertNoLock(snapshot.ID)
	if !ok || active == nil || !active.StartTime.Equal(snapshot.StartTime) || active.Acknowledged || isMonitorOnlyAlert(active) || isSupportedInfrastructureSymptom(active) {
		return nil, EscalationLevel{}, false
	}
	if m.awaitsBackupRevalidationNoLock(active) {
		return nil, EscalationLevel{}, false
	}
	if _, snoozed := alertSnoozeUntil(active, m.policyNow()); snoozed {
		return nil, EscalationLevel{}, false
	}
	target := m.config.Schedule.Escalation.Levels[level-1]
	target.DestinationIDs = append([]string(nil), target.DestinationIDs...)
	return cloneAlertForOutput(active), target, true
}