pulse/internal/notifications/webhook_enhanced.go

package notifications

import (
	"bytes"
	"fmt"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rs/zerolog/log"
)

var webhookHTTPStatusPattern = regexp.MustCompile(`\b(?:status|http)\s+(\d{3})\b`)

// EnhancedWebhookConfig extends WebhookConfig with template support
type EnhancedWebhookConfig struct {
	WebhookConfig
	Service         string                 `json:"service"`         // discord, slack, teams, pagerduty, generic
	PayloadTemplate string                 `json:"payloadTemplate"` // Go template for payload
	RetryEnabled    bool                   `json:"retryEnabled"`
	RetryCount      int                    `json:"retryCount"`
	FilterRules     WebhookFilterRules     `json:"filterRules"`
	CustomFields    map[string]interface{} `json:"customFields"`    // For template variables
	ResponseLogging bool                   `json:"responseLogging"` // Log response for debugging
}

// WebhookFilterRules defines filtering for this webhook
type WebhookFilterRules struct {
	Levels        []string `json:"levels"`        // Only send these levels
	Types         []string `json:"types"`         // Only send these alert types
	Nodes         []string `json:"nodes"`         // Only send from these nodes
	ResourceTypes []string `json:"resourceTypes"` // vm, container, storage, etc
}

// WebhookPayloadData contains all data available for templates
type WebhookPayloadData struct {
	// Alert fields
	ID                 string
	Level              string
	Type               string
	ResourceName       string
	ResourceID         string
	Node               string
	NodeDisplayName    string
	Instance           string
	Message            string
	Value              float64
	Threshold          float64
	ValueFormatted     string
	ThresholdFormatted string
	StartTime          string
	Duration           string
	Timestamp          string
	ResourceType       string
	Acknowledged       bool
	AckTime            string
	AckUser            string

	// Event context (alert vs resolved)
	Event         string // "alert" or "resolved"
	ResolvedAt    string // RFC3339 timestamp when alert was resolved
	ResolvedAtISO string // Same as ResolvedAt (alias for template consistency)

	// Tenant identity (multi-tenant orgs and MSP client runtimes).
	// Empty on plain single-tenant installs.
	TenantID   string
	TenantName string

	// Additional context
	Metadata     map[string]interface{}
	CustomFields map[string]interface{}
	AlertCount   int
	Alerts       []*alerts.Alert // For grouped alerts
	ChatID       string          // For Telegram webhooks
	Mention      string          // Platform-specific mention text
}

func canonicalWebhookConfigForEnhanced(webhook EnhancedWebhookConfig) WebhookConfig {
	canonical := webhook.WebhookConfig
	if strings.TrimSpace(webhook.Service) != "" {
		canonical.Service = webhook.Service
	}
	if strings.TrimSpace(webhook.PayloadTemplate) != "" {
		canonical.Template = webhook.PayloadTemplate
	}
	return canonical
}

func BuildEnhancedWebhookTestConfig(basicWebhook WebhookConfig, requestedService string) EnhancedWebhookConfig {
	webhook := EnhancedWebhookConfig{
		WebhookConfig: basicWebhook,
		Service:       "generic",
		RetryEnabled:  false,
	}

	if len(basicWebhook.CustomFields) > 0 {
		customFields := make(map[string]interface{}, len(basicWebhook.CustomFields))
		for key, value := range basicWebhook.CustomFields {
			customFields[key] = value
		}
		webhook.CustomFields = customFields
	}

	if basicWebhook.Template != "" {
		webhook.PayloadTemplate = basicWebhook.Template
	}

	if strings.TrimSpace(requestedService) != "" {
		webhook.Service = requestedService
		webhook.WebhookConfig.Service = requestedService
	} else if strings.TrimSpace(basicWebhook.Service) != "" {
		webhook.Service = basicWebhook.Service
	}

	if webhook.PayloadTemplate == "" {
		templates := GetWebhookTemplates()
		for _, tmpl := range templates {
			if tmpl.Service != webhook.Service {
				continue
			}
			webhook.PayloadTemplate = tmpl.PayloadTemplate
			if webhook.Headers == nil {
				webhook.Headers = make(map[string]string)
			}
			for k, v := range tmpl.Headers {
				if !strings.Contains(v, "{{") {
					webhook.Headers[k] = v
				}
			}
			break
		}
	}

	if webhook.PayloadTemplate == "" {
		webhook.PayloadTemplate = `{
			"alert": {
				"id": "{{.ID}}",
				"type": "{{.Type}}",
				"level": "{{.Level}}",
				"resourceName": "{{.ResourceName}}",
				"node": "{{.Node}}",
				"message": "{{.Message}}",
				"value": {{.Value}},
				"threshold": {{.Threshold}}
			},
			"source": "pulse-monitoring",
			"timestamp": {{.Timestamp}}
		}`
	}

	return webhook
}

func (n *NotificationManager) prepareEnhancedWebhookExecution(webhook EnhancedWebhookConfig, alert *alerts.Alert) (EnhancedWebhookConfig, []byte, error) {
	data := n.prepareWebhookData(alert, webhook.CustomFields)
	canonical := canonicalWebhookConfigForEnhanced(webhook)

	var err error
	canonical, data, err = n.prepareWebhookDeliveryContext(canonical, data)
	if err != nil {
		return webhook, nil, fmt.Errorf("prepare enhanced webhook context: %w", err)
	}
	webhook.WebhookConfig = canonical

	if webhook.Service == "ntfy" && strings.TrimSpace(webhook.PayloadTemplate) != "" {
		tmpl, err := template.New("enhanced-webhook-ntfy").Funcs(templateFuncMap()).Parse(webhook.PayloadTemplate)
		if err != nil {
			return webhook, nil, fmt.Errorf("parse ntfy payload template: %w", err)
		}
		var rendered bytes.Buffer
		if err := tmpl.Execute(&rendered, data); err != nil {
			return webhook, nil, fmt.Errorf("render ntfy payload template: %w", err)
		}
		return webhook, rendered.Bytes(), nil
	}

	payload, err := n.renderWebhookPayloadJSON(canonical, data, webhookRenderModeSingle, func() ([]byte, error) {
		if strings.TrimSpace(webhook.PayloadTemplate) == "" {
			return nil, fmt.Errorf("enhanced webhook requires payload template or known service template")
		}
		return n.generatePayloadFromTemplateWithService(webhook.PayloadTemplate, data, webhook.Service)
	})
	if err != nil {
		return webhook, nil, fmt.Errorf("render enhanced webhook payload: %w", err)
	}

	return webhook, payload, nil
}

// SendEnhancedWebhook sends a webhook with template support
func (n *NotificationManager) SendEnhancedWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) error {
	// Check filters
	if !n.shouldSendWebhook(webhook, alert) {
		log.Debug().
			Str("webhook", webhook.Name).
			Str("alertID", alert.ID).
			Msg("alert filtered out by webhook rules")
		return nil
	}

	webhook, payload, err := n.prepareEnhancedWebhookExecution(webhook, alert)
	if err != nil {
		return err
	}

	if !n.checkWebhookRateLimit(webhook.URL) {
		log.Warn().
			Str("webhook", webhook.Name).
			Str("url", RedactWebhookURLSecrets(webhook.URL)).
			Msg("webhook request dropped due to rate limiting")
		return fmt.Errorf("rate limit exceeded for webhook %s", webhook.Name)
	}

	eventID := webhookEventID(alert.ID, "alert")

	// Send with retry logic
	if webhook.RetryEnabled {
		return n.sendWebhookWithRetry(webhook, payload, eventID)
	}

	_, err = n.executeEnhancedWebhookRequest(webhook, payload, WebhookTimeout, "Pulse-Monitoring/2.0", eventID)
	if err != nil {
		return fmt.Errorf("send webhook %q once: %w", webhook.Name, err)
	}
	return nil
}

// NOTE: prepareWebhookData is now defined in notifications.go to avoid duplication
// NOTE: generatePayloadFromTemplate is now defined in notifications.go to avoid duplication

// shouldSendWebhook checks if alert matches webhook filter rules
func (n *NotificationManager) shouldSendWebhook(webhook EnhancedWebhookConfig, alert *alerts.Alert) bool {
	rules := webhook.FilterRules

	// Check level filter
	if len(rules.Levels) > 0 {
		found := false
		for _, level := range rules.Levels {
			if string(alert.Level) == level {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check type filter
	if len(rules.Types) > 0 {
		found := false
		for _, alertType := range rules.Types {
			if alert.Type == alertType {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check node filter
	if len(rules.Nodes) > 0 {
		found := false
		for _, node := range rules.Nodes {
			if alert.Node == node {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	// Check resource type filter
	if len(rules.ResourceTypes) > 0 {
		resourceType, ok := alert.Metadata["resourceType"].(string)
		if !ok {
			return false
		}
		found := false
		for _, rt := range rules.ResourceTypes {
			if resourceType == rt {
				found = true
				break
			}
		}
		if !found {
			return false
		}
	}

	return true
}

// sendWebhookWithRetry implements exponential backoff retry with enhanced error tracking
// Note: When used with the persistent queue, retry behavior is layered:
// - Transport retries (this function): up to RetryCount attempts with exponential backoff
// - Queue retries: up to MaxAttempts (default 3) with exponential backoff
// Total attempts = RetryCount * MaxAttempts (e.g., 3 * 3 = 9 HTTP calls for a single notification)
// This ensures delivery even during transient failures at either layer.
func (n *NotificationManager) sendWebhookWithRetry(webhook EnhancedWebhookConfig, payload []byte, eventID string) error {
	maxRetries := webhook.RetryCount
	if maxRetries <= 0 {
		maxRetries = WebhookDefaultRetries
	}

	var lastErr error
	var lastResp *webhookHTTPResult
	backoff := WebhookInitialBackoff
	retryableErrors := 0
	totalAttempts := 0

	for attempt := 0; attempt <= maxRetries; attempt++ {
		if attempt > 0 {
			// Check for Retry-After header from previous response (overrides backoff)
			if lastResp != nil && lastResp.statusCode == 429 {
				usedBackoff := false
				if retryAfter := lastResp.headers.Get("Retry-After"); retryAfter != "" {
					if customBackoff, ok := parseRetryAfterBackoff(retryAfter, time.Now()); ok {
						log.Debug().
							Str("webhook", webhook.Name).
							Dur("retryAfter", customBackoff).
							Msg("using Retry-After header for backoff")
						time.Sleep(customBackoff)
						backoff = customBackoff // Use this for next iteration
						usedBackoff = true
					} else {
						log.Debug().
							Str("webhook", webhook.Name).
							Str("retryAfter", retryAfter).
							Msg("invalid Retry-After header; falling back to exponential backoff")
					}
				}
				if !usedBackoff {
					// No Retry-After, use exponential backoff
					log.Debug().
						Str("webhook", webhook.Name).
						Int("attempt", attempt).
						Int("maxRetries", maxRetries).
						Dur("backoff", backoff).
						Msg("retrying webhook after backoff")
					time.Sleep(backoff)
				}
			} else {
				// Not a 429, use exponential backoff
				log.Debug().
					Str("webhook", webhook.Name).
					Int("attempt", attempt).
					Int("maxRetries", maxRetries).
					Dur("backoff", backoff).
					Msg("retrying webhook after backoff")
				time.Sleep(backoff)
			}

			// Exponential backoff for next iteration
			backoff *= 2
			if backoff > WebhookMaxBackoff {
				backoff = WebhookMaxBackoff
			}
		}

		resp, err := n.executeEnhancedWebhookRequest(webhook, payload, WebhookTimeout, "Pulse-Monitoring/2.0", eventID)
		totalAttempts = attempt + 1
		lastResp = resp
		if err == nil {
			if attempt > 0 {
				log.Info().
					Str("webhook", webhook.Name).
					Int("attempt", attempt).
					Int("totalAttempts", attempt+1).
					Msg("webhook succeeded after retry")
			}
			// Log successful delivery
			log.Debug().
				Str("webhook", webhook.Name).
				Str("service", webhook.Service).
				Int("payloadSize", len(payload)).
				Msg("webhook delivered successfully")

			// Track successful delivery
			statusCode := http.StatusOK
			if resp != nil {
				statusCode = resp.statusCode
			}
			delivery := WebhookDelivery{
				WebhookName:     webhook.Name,
				WebhookURL:      webhook.URL,
				Service:         webhook.Service,
				AlertIdentifier: "enhanced", // Enhanced webhooks are service-level deliveries, not alert-bound.
				Timestamp:       time.Now(),
				StatusCode:      statusCode,
				Success:         true,
				RetryAttempts:   attempt,
				PayloadSize:     len(payload),
			}
			n.addWebhookDelivery(delivery)

			return nil
		}

		lastErr = err

		// Determine if error is retryable
		isRetryable := isRetryableWebhookError(err)
		if isRetryable {
			retryableErrors++
		}

		log.Warn().
			Err(err).
			Str("webhook", webhook.Name).
			Str("service", webhook.Service).
			Int("attempt", attempt+1).
			Int("maxRetries", maxRetries+1).
			Bool("retryable", isRetryable).
			Msg("webhook attempt failed")

		// If error is not retryable, stop immediately regardless of attempt.
		if !isRetryable {
			log.Error().
				Err(err).
				Str("webhook", webhook.Name).
				Msg("non-retryable webhook error - not attempting retry")
			break
		}
	}

	if totalAttempts == 0 {
		totalAttempts = maxRetries + 1
	}
	retryAttempts := totalAttempts - 1
	statusCode := 0
	if lastResp != nil {
		statusCode = lastResp.statusCode
	}

	// Final error logging with summary
	log.Error().
		Err(lastErr).
		Str("webhook", webhook.Name).
		Str("service", webhook.Service).
		Int("totalAttempts", totalAttempts).
		Int("retryableErrors", retryableErrors).
		Msg("webhook delivery failed after all retry attempts")

	// Track failed delivery
	delivery := WebhookDelivery{
		WebhookName:     webhook.Name,
		WebhookURL:      webhook.URL,
		Service:         webhook.Service,
		AlertIdentifier: "enhanced", // Enhanced webhooks are service-level deliveries, not alert-bound.
		Timestamp:       time.Now(),
		StatusCode:      statusCode,
		Success:         false,
		ErrorMessage:    lastErr.Error(),
		RetryAttempts:   retryAttempts,
		PayloadSize:     len(payload),
	}
	n.addWebhookDelivery(delivery)

	return fmt.Errorf("webhook failed after %d attempts: %w", totalAttempts, lastErr)
}

func parseRetryAfterBackoff(retryAfter string, now time.Time) (time.Duration, bool) {
	retryAfter = strings.TrimSpace(retryAfter)

	clampBackoff := func(backoff time.Duration) time.Duration {
		if backoff < 0 {
			return 0
		}
		if backoff > WebhookMaxBackoff {
			return WebhookMaxBackoff
		}
		return backoff
	}

	if seconds, err := strconv.Atoi(retryAfter); err == nil {
		return clampBackoff(time.Duration(seconds) * time.Second), true
	}

	if retryAt, err := http.ParseTime(retryAfter); err == nil {
		return clampBackoff(retryAt.Sub(now)), true
	}

	return 0, false
}

// isRetryableWebhookError determines if a webhook error should trigger a retry
func isRetryableWebhookError(err error) bool {
	errStr := strings.ToLower(err.Error())

	// Network-related errors that should be retried
	if strings.Contains(errStr, "timeout") ||
		strings.Contains(errStr, "connection refused") ||
		strings.Contains(errStr, "connection reset") ||
		strings.Contains(errStr, "no such host") ||
		strings.Contains(errStr, "network unreachable") {
		return true
	}

	if statusCode, ok := webhookErrorStatusCode(err); ok {
		switch statusCode {
		case http.StatusRequestTimeout, http.StatusMisdirectedRequest, http.StatusLocked, http.StatusTooEarly, http.StatusTooManyRequests:
			return true
		}
		if statusCode >= 500 && statusCode <= 599 {
			return true
		}
		if statusCode >= 400 && statusCode <= 499 {
			return false
		}
	}

	// Default to retryable for unknown errors
	return true
}

func webhookErrorStatusCode(err error) (int, bool) {
	if err == nil {
		return 0, false
	}

	match := webhookHTTPStatusPattern.FindStringSubmatch(strings.ToLower(err.Error()))
	if len(match) != 2 {
		return 0, false
	}

	statusCode, parseErr := strconv.Atoi(match[1])
	if parseErr != nil {
		return 0, false
	}
	return statusCode, true
}

// executeEnhancedWebhookRequest sends a single enhanced webhook request and returns response metadata.
func (n *NotificationManager) executeEnhancedWebhookRequest(webhook EnhancedWebhookConfig, payload []byte, timeout time.Duration, userAgent string, eventID string) (*webhookHTTPResult, error) {
	canonical := canonicalWebhookConfigForEnhanced(webhook)
	return n.executeWebhookRequest(canonical, payload, webhookRequestOptions{
		eventID:         eventID,
		timeout:         timeout,
		userAgent:       userAgent,
		responseLogging: webhook.ResponseLogging,
		validateURL:     true,
	})
}

// NOTE: formatWebhookDuration is now defined in notifications.go to avoid duplication

// TestEnhancedWebhook tests a webhook with a specific payload
func (n *NotificationManager) TestEnhancedWebhook(webhook EnhancedWebhookConfig) (int, string, error) {
	testAlert := buildNotificationTestAlert()
	testAlert.ID = "test-" + time.Now().Format("20060102-150405")
	if n.publicURL != "" {
		testAlert.Instance = n.publicURL
	}

	var err error
	webhook, payload, err := n.prepareEnhancedWebhookExecution(webhook, testAlert)
	if err != nil {
		return 0, "", err
	}

	if webhook.Service == "ntfy" {
		headers := make(map[string]string, len(webhook.Headers)+4)
		for key, value := range webhook.Headers {
			if !strings.Contains(value, "{{") {
				headers[key] = value
			}
		}
		webhook.Headers = headers
		webhook.Headers["Content-Type"] = "text/plain"

		// Set dynamic headers based on alert level
		title := fmt.Sprintf("%s: %s",
			func() string {
				switch testAlert.Level {
				case alerts.AlertLevelCritical:
					return "CRITICAL"
				case alerts.AlertLevelWarning:
					return "WARNING"
				default:
					return "INFO"
				}
			}(),
			testAlert.ResourceName,
		)
		webhook.Headers["Title"] = title

		priority := func() string {
			switch testAlert.Level {
			case alerts.AlertLevelCritical:
				return "urgent"
			case alerts.AlertLevelWarning:
				return "high"
			default:
				return "default"
			}
		}()
		webhook.Headers["Priority"] = priority

		tags := fmt.Sprintf("%s,pulse,%s",
			func() string {
				switch testAlert.Level {
				case alerts.AlertLevelCritical:
					return "rotating_light"
				case alerts.AlertLevelWarning:
					return "warning"
				default:
					return "white_check_mark"
				}
			}(),
			testAlert.Type,
		)
		webhook.Headers["Tags"] = tags
	}

	result, err := n.executeEnhancedWebhookRequest(webhook, payload, WebhookTestTimeout, "Pulse-Monitoring/2.0 (Test)", webhookEventID(testAlert.ID, "alert"))
	if err != nil {
		if result != nil {
			return result.statusCode, result.body, err
		}
		return 0, "", err
	}

	return result.statusCode, result.body, nil
}