pulse/internal/notifications/webhook_retry_test.go

package notifications

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"sync/atomic"
	"testing"
)

// A body interrupted after the response headers must not erase the server's
// rejection. These fixtures use only the HTTP transport, without queue workers.
func TestWebhookTruncatedResponseClassification(t *testing.T) {
	for _, code := range []int{200, 401, 403, 421, 422, 423, 425, 429, 503} {
		t.Run(fmt.Sprint(code), func(t *testing.T) {
			server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Content-Length", "100")
				w.Header().Set("Retry-After", "0")
				w.WriteHeader(code)
				fmt.Fprint(w, "short")
			}))
			defer server.Close()
			nm := createTestNotificationManager(t)
			nm.webhookClient = nm.createSecureWebhookClient(WebhookTimeout)
			defer nm.webhookClient.CloseIdleConnections()
			resp, err := nm.executeEnhancedWebhookRequest(EnhancedWebhookConfig{
				WebhookConfig: WebhookConfig{URL: server.URL},
			}, []byte(`{}`), WebhookTimeout, "test", "synthetic:alert")
			if !errors.Is(err, io.ErrUnexpectedEOF) {
				t.Fatalf("read error not retained: %v", err)
			}
			if resp == nil || resp.statusCode != code || resp.headers.Get("Retry-After") != "0" {
				t.Fatalf("response metadata lost: %+v", resp)
			}
			if code != 200 {
				if got := ClassifyNotificationFailureError(err); got != ClassFromHTTPStatus(code) {
					t.Errorf("class = %s, want %s", got, ClassFromHTTPStatus(code))
				}
			}
			wantRetry := code == 200 || code == 421 || code == 423 || code == 425 || code == 429 || code == 503
			if got := isRetryableWebhookError(err); got != wantRetry {
				t.Errorf("retryable = %v, want %v: %v", got, wantRetry, err)
			}
			if got := ClassifyNotificationFailureError(err).Retryable(); got != wantRetry {
				t.Errorf("queue classification retryable = %v, want %v", got, wantRetry)
			}
		})
	}
}

func TestWebhookRetryTruncatedResponse(t *testing.T) {
	for _, code := range []int{403, 421, 423, 425, 503} {
		t.Run(fmt.Sprint(code), func(t *testing.T) {
			var attempts atomic.Int32
			server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				if r.Header.Get("X-Pulse-Event-ID") != "synthetic:alert" {
					t.Error("lost event identity")
				}
				if attempts.Add(1) > 1 && code != 403 {
					w.WriteHeader(http.StatusNoContent)
					return
				}
				w.Header().Set("Content-Length", "100")
				w.Header().Set("Retry-After", "0")
				w.WriteHeader(code)
				fmt.Fprint(w, "short")
			}))
			defer server.Close()
			nm := createTestNotificationManager(t)
			nm.webhookClient = nm.createSecureWebhookClient(WebhookTimeout)
			defer nm.webhookClient.CloseIdleConnections()
			err := nm.sendWebhookWithRetry(EnhancedWebhookConfig{
				WebhookConfig: WebhookConfig{URL: server.URL}, RetryCount: 1,
			}, []byte(`{}`), "synthetic:alert")
			wantAttempts, wantStatus := int32(1), code
			wantSuccess := code != 403
			if wantSuccess {
				wantAttempts, wantStatus = 2, http.StatusNoContent
			}
			if (err == nil) != wantSuccess || attempts.Load() != wantAttempts {
				t.Errorf("attempts=%d error=%v, want attempts=%d success=%v", attempts.Load(), err, wantAttempts, wantSuccess)
			}
			history := nm.GetWebhookHistory()
			if len(history) != 1 {
				t.Fatalf("history length = %d, want 1", len(history))
			}
			if h := history[0]; h.StatusCode != wantStatus || h.Success != wantSuccess || h.RetryAttempts != int(wantAttempts-1) {
				t.Errorf("history status=%d success=%v retries=%d", h.StatusCode, h.Success, h.RetryAttempts)
			}
		})
	}
}

func TestIsRetryableWebhookError(t *testing.T) {
	tests := []struct {
		name     string
		err      error
		expected bool
	}{
		// Network errors - should be retryable
		{
			name:     "timeout error",
			err:      errors.New("connection timeout"),
			expected: true,
		},
		{
			name:     "timeout uppercase",
			err:      errors.New("TIMEOUT"),
			expected: true,
		},
		{
			name:     "connection refused",
			err:      errors.New("dial tcp: connection refused"),
			expected: true,
		},
		{
			name:     "connection reset",
			err:      errors.New("read: connection reset by peer"),
			expected: true,
		},
		{
			name:     "no such host",
			err:      errors.New("lookup webhook.example.com: no such host"),
			expected: true,
		},
		{
			name:     "network unreachable",
			err:      errors.New("dial tcp: network unreachable"),
			expected: true,
		},
		// HTTP 5xx errors - should be retryable
		{
			name:     "status 500 internal server error",
			err:      errors.New("webhook returned status 500: Internal Server Error"),
			expected: true,
		},
		{
			name:     "http 500 internal server error",
			err:      errors.New("webhook returned HTTP 500: Internal Server Error"),
			expected: true,
		},
		{
			name:     "status 502 bad gateway",
			err:      errors.New("webhook returned status 502: Bad Gateway"),
			expected: true,
		},
		{
			name:     "status 503 service unavailable",
			err:      errors.New("webhook returned status 503: Service Unavailable"),
			expected: true,
		},
		{
			name:     "status 504 gateway timeout",
			err:      errors.New("webhook returned status 504: Gateway Timeout"),
			expected: true,
		},
		{
			name:     "status 520 cloudflare error",
			err:      errors.New("webhook returned status 520"),
			expected: true,
		},
		{
			name:     "status 522 connection timed out",
			err:      errors.New("webhook returned status 522: Connection timed out"),
			expected: true,
		},
		{
			name:     "status 599 network connect timeout",
			err:      errors.New("webhook returned status 599"),
			expected: true,
		},
		// HTTP 429 rate limiting - should be retryable
		{
			name:     "status 408 request timeout",
			err:      errors.New("webhook returned status 408: Request Timeout"),
			expected: true,
		},
		{
			name:     "status 421 misdirected request",
			err:      errors.New("webhook returned status 421: Misdirected Request"),
			expected: true,
		},
		{
			name:     "status 423 locked",
			err:      errors.New("webhook returned status 423: Locked"),
			expected: true,
		},
		{
			name:     "status 425 too early",
			err:      errors.New("webhook returned status 425: Too Early"),
			expected: true,
		},
		{
			name:     "status 429 too many requests",
			err:      errors.New("webhook returned status 429: Too Many Requests"),
			expected: true,
		},
		{
			name:     "http 429 too many requests",
			err:      errors.New("webhook returned HTTP 429: Too Many Requests"),
			expected: true,
		},
		// HTTP 4xx client errors - should NOT be retryable
		{
			name:     "status 400 bad request",
			err:      errors.New("webhook returned status 400: Bad Request"),
			expected: false,
		},
		{
			name:     "http 400 bad request",
			err:      errors.New("webhook returned HTTP 400: Bad Request"),
			expected: false,
		},
		{
			name:     "status 401 unauthorized",
			err:      errors.New("webhook returned status 401: Unauthorized"),
			expected: false,
		},
		{
			name:     "status 403 forbidden",
			err:      errors.New("webhook returned status 403: Forbidden"),
			expected: false,
		},
		{
			name:     "status 404 not found",
			err:      errors.New("webhook returned status 404: Not Found"),
			expected: false,
		},
		{
			name:     "status 405 method not allowed",
			err:      errors.New("webhook returned status 405: Method Not Allowed"),
			expected: false,
		},
		{
			name:     "status 410 gone",
			err:      errors.New("webhook returned status 410: Gone"),
			expected: false,
		},
		{
			name:     "status 413 payload too large",
			err:      errors.New("webhook returned status 413: Payload Too Large"),
			expected: false,
		},
		{
			name:     "status 415 unsupported media type",
			err:      errors.New("webhook returned status 415: Unsupported Media Type"),
			expected: false,
		},
		{
			name:     "status 422 unprocessable entity",
			err:      errors.New("webhook returned status 422: Unprocessable Entity"),
			expected: false,
		},
		// Note: 429 is handled specially as retryable (checked before 4xx loop)
		// Edge cases
		{
			name:     "unknown error defaults to retryable",
			err:      errors.New("something unexpected happened"),
			expected: true,
		},
		{
			name:     "empty error message",
			err:      errors.New(""),
			expected: true,
		},
		{
			name:     "TLS handshake error",
			err:      errors.New("tls: handshake failure"),
			expected: true,
		},
		{
			name:     "EOF error",
			err:      errors.New("unexpected EOF"),
			expected: true,
		},
		{
			name:     "DNS resolution error",
			err:      errors.New("lookup failed: no such host"),
			expected: true,
		},
		// Case insensitivity tests
		{
			name:     "TIMEOUT uppercase",
			err:      errors.New("REQUEST TIMEOUT"),
			expected: true,
		},
		{
			name:     "Connection Refused mixed case",
			err:      errors.New("Connection Refused"),
			expected: true,
		},
		{
			name:     "Network Unreachable mixed case",
			err:      errors.New("Network Unreachable"),
			expected: true,
		},
		// Real-world error formats
		{
			name:     "dial tcp timeout format",
			err:      errors.New("dial tcp 192.168.1.1:443: i/o timeout"),
			expected: true,
		},
		{
			name:     "context deadline exceeded",
			err:      errors.New("context deadline exceeded"),
			expected: true,
		},
		{
			name:     "wrapped timeout error",
			err:      fmt.Errorf("request failed: %w", errors.New("timeout")),
			expected: true,
		},
		{
			name:     "status code in message format",
			err:      errors.New("HTTP error: status 503"),
			expected: true,
		},
		{
			name:     "http code in message format",
			err:      errors.New("webhook returned HTTP 503"),
			expected: true,
		},
		{
			name:     "status code only",
			err:      errors.New("status 401"),
			expected: false,
		},
		{
			name:     "http code only",
			err:      errors.New("HTTP 401"),
			expected: false,
		},
		// Boundary tests for status code ranges
		{
			name:     "status 499 last 4xx",
			err:      errors.New("webhook returned status 499"),
			expected: false,
		},
		{
			name:     "status 500 first 5xx",
			err:      errors.New("webhook returned status 500"),
			expected: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := isRetryableWebhookError(tt.err)
			if result != tt.expected {
				t.Errorf("isRetryableWebhookError(%q) = %v, want %v", tt.err.Error(), result, tt.expected)
			}
		})
	}
}

// TestIsRetryableWebhookError_AllStatusCodes verifies status code classification
func TestIsRetryableWebhookError_AllStatusCodes(t *testing.T) {
	// Test all 4xx codes are not retryable (except 429)
	for code := 400; code <= 499; code++ {
		err := fmt.Errorf("webhook returned status %d", code)
		result := isRetryableWebhookError(err)

		if code == 408 || code == 421 || code == 423 || code == 425 || code == 429 {
			// 408, 421, 423, 425, and 429 are special - should be retryable
			if !result {
				t.Errorf("status %d should be retryable (rate limited)", code)
			}
		} else {
			// Other 4xx should not be retryable
			if result {
				t.Errorf("status %d should NOT be retryable (client error)", code)
			}
		}
	}

	// Test all 5xx codes are retryable
	for code := 500; code <= 599; code++ {
		err := fmt.Errorf("webhook returned status %d", code)
		result := isRetryableWebhookError(err)
		if !result {
			t.Errorf("status %d should be retryable (server error)", code)
		}
	}
}

// TestIsRetryableWebhookError_NetworkPatterns tests various network error patterns
func TestIsRetryableWebhookError_NetworkPatterns(t *testing.T) {
	networkErrors := []string{
		"dial tcp 10.0.0.1:443: connect: connection refused",
		"dial tcp: lookup api.example.com: no such host",
		"read tcp 192.168.1.1:52345->10.0.0.1:443: read: connection reset by peer",
		"write tcp 192.168.1.1:52345->10.0.0.1:443: write: broken pipe",
		"dial tcp 10.0.0.1:443: i/o timeout",
		"net/http: request canceled while waiting for connection (Client.Timeout exceeded)",
		"dial tcp [::1]:443: connect: network unreachable",
		"Post \"https://api.example.com/webhook\": dial tcp: lookup api.example.com: no such host",
	}

	for _, errStr := range networkErrors {
		t.Run(errStr[:min(50, len(errStr))], func(t *testing.T) {
			err := errors.New(errStr)
			if !isRetryableWebhookError(err) {
				t.Errorf("network error %q should be retryable", errStr)
			}
		})
	}
}

// Provider diagnostics must not turn a terminal HTTP rejection into a network retry.
func TestIsRetryableWebhookError_StatusOverridesBody(t *testing.T) {
	for _, code := range []int{400, 401, 403, 404, 422} {
		for _, body := range []string{"authentication timeout", "connection refused", "connection reset", "no such host", "network unreachable"} {
			t.Run(fmt.Sprintf("%d/%s", code, body), func(t *testing.T) {
				err := fmt.Errorf("webhook returned HTTP %d: %s", code, body)
				if isRetryableWebhookError(err) {
					t.Fatalf("terminal HTTP %d was retried because of diagnostic %q", code, body)
				}
			})
		}
	}
}

// Exercise the HTTP transport without starting a persistent queue or workers.
func TestWebhookRetryRejectsForbiddenTimeoutBody(t *testing.T) {
	var attempts atomic.Int32
	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		attempts.Add(1)
		w.Header().Set("Retry-After", "0")
		w.WriteHeader(http.StatusForbidden)
		fmt.Fprint(w, "authentication timeout: replace credentials")
	}))
	defer server.Close()
	nm := &NotificationManager{}
	if err := nm.UpdateAllowedPrivateCIDRs("127.0.0.1"); err != nil {
		t.Fatal(err)
	}
	nm.webhookClient = nm.createSecureWebhookClient(WebhookTimeout)
	defer nm.webhookClient.CloseIdleConnections()
	err := nm.sendWebhookWithRetry(EnhancedWebhookConfig{
		WebhookConfig: WebhookConfig{Name: "synthetic rejection", URL: server.URL},
		RetryEnabled:  true, RetryCount: 1,
	}, []byte(`{"test":true}`), "synthetic:alert")
	if err == nil {
		t.Fatal("expected forbidden delivery failure")
	}
	if got := attempts.Load(); got != 1 {
		t.Errorf("HTTP attempts = %d, want 1", got)
	}
	history := nm.GetWebhookHistory()
	if len(history) != 1 {
		t.Fatalf("history length = %d, want 1", len(history))
	}
	if history[0].StatusCode != http.StatusForbidden || history[0].Success || history[0].RetryAttempts != 0 {
		t.Errorf("history lost terminal rejection: status=%d success=%v retries=%d", history[0].StatusCode, history[0].Success, history[0].RetryAttempts)
	}
}

// A retry hint from an earlier response must not outlive a terminal rejection.
func TestWebhookRetryRateLimitThenTerminalRejection(t *testing.T) {
	var attempts atomic.Int32
	payload := []byte(`{"test":true}`)
	server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("X-Pulse-Event-ID") != "synthetic:alert" {
			t.Error("retry lost event identity")
		}
		if attempts.Add(1) == 1 {
			w.Header().Set("Retry-After", "0")
			w.WriteHeader(http.StatusTooManyRequests)
			return
		}
		w.WriteHeader(http.StatusForbidden)
		fmt.Fprint(w, "authentication timeout: replace credentials")
	}))
	defer server.Close()
	// Transport-only fixture: no persistent queue or background workers.
	nm := &NotificationManager{}
	if err := nm.UpdateAllowedPrivateCIDRs("127.0.0.1"); err != nil {
		t.Fatal(err)
	}
	nm.webhookClient = nm.createSecureWebhookClient(WebhookTimeout)
	defer nm.webhookClient.CloseIdleConnections()
	err := nm.sendWebhookWithRetry(EnhancedWebhookConfig{
		WebhookConfig: WebhookConfig{Name: "synthetic rate limit", URL: server.URL},
		RetryEnabled:  true, RetryCount: 2,
	}, payload, "synthetic:alert")
	if err == nil {
		t.Fatal("expected terminal rejection after rate limit")
	}
	if got := attempts.Load(); got != 2 {
		t.Errorf("HTTP attempts = %d, want 2 despite remaining retry budget", got)
	}
	history := nm.GetWebhookHistory()
	if len(history) != 1 {
		t.Fatalf("history length = %d, want one final delivery outcome", len(history))
	}
	if history[0].StatusCode != http.StatusForbidden || history[0].Success || history[0].RetryAttempts != 1 || history[0].PayloadSize != len(payload) {
		t.Errorf("history lost final rejection or retry accounting: %+v", history[0])
	}
}

// Queue retry policy consumes the classified, wrapped transport error, not the
// HTTP response. Keep both retry layers consistent for every rejection status.
func TestWebhookHTTPRetryPolicyMatchesQueueClassification(t *testing.T) {
	for code := 400; code <= 599; code++ {
		t.Run(fmt.Sprint(code), func(t *testing.T) {
			nm := &NotificationManager{webhookClient: &http.Client{
				Transport: confidentialityTransport(func(*http.Request) (*http.Response, error) {
					return &http.Response{
						StatusCode: code, Header: make(http.Header),
						Body: io.NopCloser(strings.NewReader("unauthorized timeout rate limit")),
					}, nil
				}),
			}}
			_, err := nm.executeWebhookRequest(WebhookConfig{URL: "https://example.test/hook"},
				[]byte("{}"), webhookRequestOptions{})
			if err == nil {
				t.Fatal("expected HTTP rejection")
			}
			err = fmt.Errorf("webhook delivery exhausted: %w", err)
			wantRetry := code >= 500 || code == 408 || code == 421 || code == 423 || code == 425 || code == 429
			if got := isRetryableWebhookError(err); got != wantRetry {
				t.Errorf("transport retryable = %v, want %v", got, wantRetry)
			}
			class := ClassifyNotificationFailureError(err)
			if got := class.Retryable(); got != wantRetry {
				t.Errorf("queue class %s retryable = %v, want %v", class, got, wantRetry)
			}
		})
	}
}