pulse/internal/notifications/webhook_enhanced.go

package notifications

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rs/zerolog/log"
)

var webhookHTTPStatusPattern = regexp.MustCompile(`\b(?:status|http)\s+(\d{3})\b`)

// EnhancedWebhookConfig extends WebhookConfig with template support
type EnhancedWebhookConfig struct {
	WebhookConfig
	Service         string                 `json:"service"`         // discord, slack, teams, pagerduty, generic
	PayloadTemplate string                 `json:"payloadTemplate"` // Go template for payload
	RetryEnabled    bool                   `json:"retryEnabled"`
	RetryCount      int                    `json:"retryCount"`
	FilterRules     WebhookFilterRules     `json:"filterRules"`
	CustomFields    map[string]interface{} `json:"customFields"`    // For template variables
	ResponseLogging bool                   `json:"responseLogging"` // Log structured response metadata, never provider text
}

// WebhookFilterRules defines filtering for this webhook
type WebhookFilterRules struct {
	Levels        []string `json:"levels"`        // Only send these levels
	Types         []string `json:"types"`         // Only send these alert types
	Nodes         []string `json:"nodes"`         // Only send from these nodes
	ResourceTypes []string `json:"resourceTypes"` // vm, container, storage, etc
}

// WebhookPayloadData contains all data available for templates
type WebhookPayloadData struct {
	// Alert fields
	ID                 string
	MessageKey         string
	Level              string
	Type               string
	ResourceName       string
	ResourceID         string
	Node               string
	NodeDisplayName    string
	Instance           string
	Message            string
	Value              float64
	Threshold          float64
	ValueFormatted     string
	ThresholdFormatted string
	StartTime          string
	Duration           string
	Timestamp          string
	ResourceType       string
	Acknowledged       bool
	AckTime            string
	AckUser            string

	// Event context (alert vs resolved)
	Event         string // "alert" or "resolved"
	ResolvedAt    string // RFC3339 timestamp when alert was resolved
	ResolvedAtISO string // Same as ResolvedAt (alias for template consistency)
	// NotRecovered marks a resolved notification in which at least one close
	// was not a recovery (an alert that moved to a linked Pulse agent), so
	// templates can drop recovery colours and check marks.
	NotRecovered bool

	// Tenant identity (multi-tenant orgs and MSP client runtimes).
	// Empty on plain single-tenant installs.
	TenantID   string
	TenantName string

	// Additional context
	Metadata     map[string]interface{}
	CustomFields map[string]interface{}
	AlertCount   int
	Alerts       []*alerts.Alert // For grouped alerts
	ChatID       string          // For Telegram webhooks
	Mention      string          // Platform-specific mention text
}

func canonicalWebhookConfigForEnhanced(webhook EnhancedWebhookConfig) WebhookConfig {
	canonical := webhook.WebhookConfig
	if strings.TrimSpace(webhook.Service) != "" {
		canonical.Service = webhook.Service
	}
	if strings.TrimSpace(webhook.PayloadTemplate) != "" {
		canonical.Template = webhook.PayloadTemplate
	}
	return canonical
}

func BuildEnhancedWebhookTestConfig(basicWebhook WebhookConfig, requestedService string) EnhancedWebhookConfig {
	webhook := EnhancedWebhookConfig{
		WebhookConfig: basicWebhook,
		Service:       "generic",
		RetryEnabled:  false,
	}

	if len(basicWebhook.CustomFields) > 0 {
		customFields := make(map[string]interface{}, len(basicWebhook.CustomFields))
		for key, value := range basicWebhook.CustomFields {
			customFields[key] = value
		}
		webhook.CustomFields = customFields
	}

	if basicWebhook.Template != "" {
		webhook.PayloadTemplate = basicWebhook.Template
	}

	if strings.TrimSpace(requestedService) != "" {
		webhook.Service = requestedService
		webhook.WebhookConfig.Service = requestedService
	} else if strings.TrimSpace(basicWebhook.Service) != "" {
		webhook.Service = basicWebhook.Service
	}

	if webhook.PayloadTemplate == "" {
		templates := GetWebhookTemplates()
		for _, tmpl := range templates {
			if tmpl.Service != webhook.Service {
				continue
			}
			webhook.PayloadTemplate = tmpl.PayloadTemplate
			webhook.WebhookConfig = withWebhookTemplateHeaders(webhook.WebhookConfig, tmpl.Headers)
			break
		}
	}

	if webhook.PayloadTemplate == "" {
		webhook.PayloadTemplate = `{
			"alert": {
				"id": "{{.ID}}",
				"type": "{{.Type}}",
				"level": "{{.Level}}",
				"resourceName": "{{.ResourceName}}",
				"node": "{{.Node}}",
				"message": "{{.Message}}",
				"value": {{.Value}},
				"threshold": {{.Threshold}}
			},
			"source": "pulse-monitoring",
			"timestamp": {{.Timestamp}}
		}`
	}

	return webhook
}

func (n *NotificationManager) prepareEnhancedWebhookExecution(webhook EnhancedWebhookConfig, alert *alerts.Alert) (EnhancedWebhookConfig, []byte, error) {
	data := n.prepareWebhookData(alert, webhook.CustomFields)
	canonical := canonicalWebhookConfigForEnhanced(webhook)

	var err error
	canonical, data, err = n.prepareWebhookDeliveryContext(canonical, data)
	if err != nil {
		return webhook, nil, fmt.Errorf("prepare enhanced webhook context: %w", err)
	}
	webhook.WebhookConfig = canonical

	if webhook.Service == "ntfy" && strings.TrimSpace(webhook.PayloadTemplate) != "" {
		tmpl, err := template.New("enhanced-webhook-ntfy").Funcs(templateFuncMap()).Parse(webhook.PayloadTemplate)
		if err != nil {
			return webhook, nil, fmt.Errorf("parse ntfy payload template: %w", err)
		}
		var rendered bytes.Buffer
		if err := tmpl.Execute(&rendered, data); err != nil {
			return webhook, nil, fmt.Errorf("render ntfy payload template: %w", err)
		}
		return webhook, rendered.Bytes(), nil
	}

	payload, err := n.renderWebhookPayloadJSON(canonical, data, webhookRenderModeSingle, func() ([]byte, error) {
		if strings.TrimSpace(webhook.PayloadTemplate) == "" {
			return nil, fmt.Errorf("enhanced webhook requires payload template or known service template")
		}
		return n.generatePayloadFromTemplateWithService(webhook.PayloadTemplate, data, webhook.Service)
	})
	if err != nil {
		return webhook, nil, fmt.Errorf("render enhanced webhook payload: %w", err)
	}

	return webhook, payload, nil
}

// SendEnhancedWebhook sends a webhook with template support
func (n *NotificationManager) SendEnhancedWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) error {
	// Check filters
	if !n.shouldSendWebhook(webhook, alert) {
		log.Debug().
			Str("webhook", webhook.Name).
			Str("alertID", alert.ID).
			Msg("alert filtered out by webhook rules")
		return nil
	}

	webhook, payload, err := n.prepareEnhancedWebhookExecution(webhook, alert)
	if err != nil {
		return err
	}

	if !n.checkWebhookRateLimit(webhook.URL) {
		log.Warn().
			Str("webhook", webhook.Name).
			Str("url", RedactWebhookURLSecrets(webhook.URL)).
			Msg("webhook request dropped due to rate limiting")
		return fmt.Errorf("rate limit exceeded for webhook %s", webhook.Name)
	}

	eventID := webhookEventID(alert.ID, "alert")

	// Send with retry logic
	if webhook.RetryEnabled {
		return n.sendWebhookWithRetry(webhook, payload, eventID)
	}

	_, err = n.executeEnhancedWebhookRequest(webhook, payload, WebhookTimeout, "Pulse-Monitoring/2.0", eventID)
	if err != nil {
		return fmt.Errorf("send webhook %q once: %w", webhook.Name, err)
	}
	return nil
}

// NOTE: prepareWebhookData is now defined in notifications.go to avoid duplication
// NOTE: generatePayloadFromTemplate is now defined in notifications.go to avoid duplication

// shouldSendWebhook checks if alert matches webhook filter rules
func (n *NotificationManager) shouldSendWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) bool {
	rules := webhook.FilterRules

	// Check level filter
	if len(rules.Levels) > 0 {
		found := false
		for _, level := range rules.Levels {
			if string(alert.Level) == level {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check type filter
	if len(rules.Types) > 0 {
		found := false
		for _, alertType := range rules.Types {
			if alert.Type == alertType {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check node filter
	if len(rules.Nodes) > 0 {
		found := false
		for _, node := range rules.Nodes {
			if alert.Node == node {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check resource type filter
	if len(rules.ResourceTypes) > 0 {
		resourceType, ok := alert.Metadata["resourceType"].(string)
		if !ok {
			return false
		}
		found := false
		for _, rt := range rules.ResourceTypes {
			if resourceType == rt {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	return true
}

// sendWebhookWithRetry implements exponential backoff retry with enhanced error tracking
// Note: When used with the persistent queue, retry behavior is layered:
// - Transport: one initial attempt plus RetryCount retries (non-positive counts use WebhookDefaultRetries)
// - Queue: up to MaxAttempts delivery attempts with exponential backoff
// The HTTP attempt ceiling is (effective RetryCount + 1) * MaxAttempts when
// each queue delivery uses this transport. Permanent errors can stop earlier;
// exhausting either retry budget does not guarantee delivery.
func (n *NotificationManager) sendWebhookWithRetry(webhook EnhancedWebhookConfig, payload []byte, eventID string) error {
	maxRetries := webhook.RetryCount
	if maxRetries <= 0 {
		maxRetries = WebhookDefaultRetries
	}

	var lastErr error
	var lastResp *webhookHTTPResult
	backoff := WebhookInitialBackoff
	retryableErrors := 0
	totalAttempts := 0

	for attempt := 0; attempt <= maxRetries; attempt++ {
		if attempt > 0 {
			// Check for Retry-After header from previous response (overrides backoff)
			// Service-unavailable responses can advertise recovery time too
			// (RFC 9110 section 10.2.3). Keep the same bounded parser and budget.
			if lastResp != nil && (lastResp.statusCode == http.StatusTooManyRequests || lastResp.statusCode == http.StatusServiceUnavailable) {
				usedBackoff := false
				if retryAfter := lastResp.headers.Get("Retry-After"); retryAfter != "" {
					if customBackoff, ok := parseRetryAfterBackoff(retryAfter, time.Now()); ok {
						log.Debug().
							Str("webhook", webhook.Name).
							Dur("retryAfter", customBackoff).
							Msg("using Retry-After header for backoff")
						time.Sleep(customBackoff)
						// The header overrides this wait only. Preserve the exponential
						// schedule so a zero delay cannot erase later failure backoff.
						usedBackoff = true
					} else {
						log.Debug().
							Str("webhook", webhook.Name).
							Msg("invalid Retry-After header; falling back to exponential backoff")
					}
				}
				if !usedBackoff {
					// No Retry-After, use exponential backoff
					log.Debug().
						Str("webhook", webhook.Name).
						Int("attempt", attempt).
						Int("maxRetries", maxRetries).
						Dur("backoff", backoff).
						Msg("retrying webhook after backoff")
					time.Sleep(backoff)
				}
			} else {
				// No supported retry hint, use exponential backoff
				log.Debug().
					Str("webhook", webhook.Name).
					Int("attempt", attempt).
					Int("maxRetries", maxRetries).
					Dur("backoff", backoff).
					Msg("retrying webhook after backoff")
				time.Sleep(backoff)
			}

			// Exponential backoff for next iteration
			backoff *= 2
			if backoff > WebhookMaxBackoff {
				backoff = WebhookMaxBackoff
			}
		}

		resp, err := n.executeEnhancedWebhookRequest(webhook, payload, WebhookTimeout, "Pulse-Monitoring/2.0", eventID)
		totalAttempts = attempt + 1
		lastResp = resp
		if err == nil {
			if attempt > 0 {
				log.Info().
					Str("webhook", webhook.Name).
					Int("attempt", attempt).
					Int("totalAttempts", attempt+1).
					Msg("webhook succeeded after retry")
			}
			// Log successful delivery
			log.Debug().
				Str("webhook", webhook.Name).
				Str("service", webhook.Service).
				Int("payloadSize", len(payload)).
				Msg("webhook delivered successfully")

			// Track successful delivery
			statusCode := http.StatusOK
			if resp != nil {
				statusCode = resp.statusCode
			}
			delivery := WebhookDelivery{
				WebhookName:     webhook.Name,
				WebhookURL:      webhook.URL,
				Service:         webhook.Service,
				AlertIdentifier: "enhanced", // Enhanced webhooks are service-level deliveries, not alert-bound.
				Timestamp:       time.Now(),
				StatusCode:      statusCode,
				Success:         true,
				RetryAttempts:   attempt,
				PayloadSize:     len(payload),
			}
			n.addWebhookDelivery(delivery)

			return nil
		}

		lastErr = err

		// Determine if error is retryable
		isRetryable := isRetryableWebhookError(err)
		if isRetryable {
			retryableErrors++
		}

		log.Warn().
			Err(err).
			Str("webhook", webhook.Name).
			Str("service", webhook.Service).
			Int("attempt", attempt+1).
			Int("maxRetries", maxRetries+1).
			Bool("retryable", isRetryable).
			Msg("webhook attempt failed")

		// If error is not retryable, stop immediately regardless of attempt.
		if !isRetryable {
			log.Error().
				Err(err).
				Str("webhook", webhook.Name).
				Msg("non-retryable webhook error - not attempting retry")
			break
		}
	}

	if totalAttempts == 0 {
		totalAttempts = maxRetries + 1
	}
	retryAttempts := totalAttempts - 1
	statusCode := 0
	if lastResp != nil {
		statusCode = lastResp.statusCode
	}

	// Final error logging with summary
	log.Error().
		Err(lastErr).
		Str("webhook", webhook.Name).
		Str("service", webhook.Service).
		Int("totalAttempts", totalAttempts).
		Int("retryableErrors", retryableErrors).
		Msg("webhook delivery failed after all retry attempts")

	// Track failed delivery
	delivery := WebhookDelivery{
		WebhookName:     webhook.Name,
		WebhookURL:      webhook.URL,
		Service:         webhook.Service,
		AlertIdentifier: "enhanced", // Enhanced webhooks are service-level deliveries, not alert-bound.
		Timestamp:       time.Now(),
		StatusCode:      statusCode,
		Success:         false,
		ErrorMessage:    lastErr.Error(),
		RetryAttempts:   retryAttempts,
		PayloadSize:     len(payload),
	}
	n.addWebhookDelivery(delivery)

	return fmt.Errorf("webhook failed after %d attempts: %w", totalAttempts, lastErr)
}

func parseRetryAfterBackoff(retryAfter string, now time.Time) (time.Duration, bool) {
	retryAfter = strings.TrimSpace(retryAfter)

	clampBackoff := func(backoff time.Duration) time.Duration {
		if backoff < 0 {
			return 0
		}
		if backoff > WebhookMaxBackoff {
			return WebhookMaxBackoff
		}
		return backoff
	}

	if seconds, err := strconv.ParseInt(retryAfter, 10, 64); err == nil {
		// Clamp in seconds before conversion: multiplying an untrusted value
		// by time.Second can overflow and turn a long delay into no delay.
		if seconds <= 0 {
			return 0, true
		}
		if seconds > int64(WebhookMaxBackoff/time.Second) {
			return WebhookMaxBackoff, true
		}
		return clampBackoff(time.Duration(seconds) * time.Second), true
	}

	if retryAt, err := http.ParseTime(retryAfter); err == nil {
		return clampBackoff(retryAt.Sub(now)), true
	}

	return 0, false
}

// isRetryableWebhookError determines if a webhook error should trigger a retry
func isRetryableWebhookError(err error) bool {
	// A local security/configuration refusal is authoritative too. In
	// particular, retrying an origin-changing redirect cannot make it safe.
	var declared *NotificationFailureError
	if errors.As(err, &declared) && declared.Class != "" {
		return declared.Class.Retryable()
	}
	// An explicit HTTP rejection takes precedence over diagnostic body text:
	// a 403 mentioning "timeout" is not a transport timeout. Errors without
	// a recognised status (including network failures) remain retryable below.
	if statusCode, ok := webhookErrorStatusCode(err); ok {
		switch statusCode {
		case http.StatusRequestTimeout, http.StatusMisdirectedRequest, http.StatusLocked, http.StatusTooEarly, http.StatusTooManyRequests:
			return true
		}
		if statusCode >= 500 && statusCode <= 599 {
			return true
		}
		if statusCode >= 400 && statusCode <= 499 {
			return false
		}
	}

	// Default to retryable for unknown errors
	return true
}

func webhookErrorStatusCode(err error) (int, bool) {
	if err == nil {
		return 0, false
	}

	match := webhookHTTPStatusPattern.FindStringSubmatch(strings.ToLower(err.Error()))
	if len(match) != 2 {
		return 0, false
	}

	statusCode, parseErr := strconv.Atoi(match[1])
	if parseErr != nil {
		return 0, false
	}
	return statusCode, true
}

// executeEnhancedWebhookRequest sends a single enhanced webhook request and returns response metadata.
func (n *NotificationManager) executeEnhancedWebhookRequest(webhook EnhancedWebhookConfig, payload []byte, timeout time.Duration, userAgent string, eventID string) (*webhookHTTPResult, error) {
	canonical := canonicalWebhookConfigForEnhanced(webhook)
	return n.executeWebhookRequest(canonical, payload, webhookRequestOptions{
		eventID:         eventID,
		timeout:         timeout,
		userAgent:       userAgent,
		responseLogging: webhook.ResponseLogging,
		validateURL:     true,
	})
}

// NOTE: formatWebhookDuration is now defined in notifications.go to avoid duplication

// TestEnhancedWebhook tests a webhook with a specific payload
func (n *NotificationManager) TestEnhancedWebhook(webhook EnhancedWebhookConfig) (int, string, error) {
	testAlert := buildNotificationTestAlert()
	testAlert.ID = "test-" + time.Now().Format("20060102-150405")
	if n.publicURL != "" {
		testAlert.Instance = n.publicURL
	}

	var err error
	webhook, payload, err := n.prepareEnhancedWebhookExecution(webhook, testAlert)
	if err != nil {
		return 0, "", err
	}

	if webhook.Service == "ntfy" {
		canonical := withNtfyAlertHeaders(canonicalWebhookConfigForEnhanced(webhook), []*alerts.Alert{testAlert})
		webhook.Headers = canonical.Headers
	}

	result, err := n.executeEnhancedWebhookRequest(webhook, payload, WebhookTestTimeout, "Pulse-Monitoring/2.0 (Test)", webhookEventID(testAlert.ID, "alert"))
	if err != nil {
		if result != nil {
			return result.statusCode, result.responseSummary(), err
		}
		return 0, "", err
	}

	return result.statusCode, result.responseSummary(), nil
}