Improve webhook delivery reliability by adding retry mechanism with exponential backoff

Replit-Commit-Author: Agent Replit-Commit-Session-Id: 9111ef36-26c8-4085-84ca-a35dc1fec1b5 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/7083d608-d6d3-4a6a-9a27-6286c5109627/0113a02c-5955-4b70-9ab4-3dd7bd4096a1.jpg
2026-07-26 11:38:13 +00:00 · 2025-04-10 02:05:17 +00:00
parent 04bb2413b5
commit a22672f8f2
2 changed files with 133 additions and 45 deletions
@@ -508,36 +508,46 @@ export class MemStorage implements IStorage {
    if (!webhook || !webhook.isActive) return false;

    try {
-      // Using the webhook utility to deliver the webhook
-      const { generateSignature, deliverWebhook } = await import('./utils/webhook');
+      // Import the webhook utility functions
+      const webhookUtils = await import('./utils/webhook');
      
-      // In a real implementation, this would make an HTTP request to the webhook URL
-      console.log(`Triggering webhook ${webhook.name} (${webhook.id}) with payload:`, payload);
+      console.log(`Triggering webhook ${webhook.name} (${webhook.id})`);
      
-      // Generate signature for the payload if a secret is set
-      const signature = webhook.secret ? generateSignature(payload, webhook.secret) : '';
+      // Setup delivery options with the provided retry count
+      const deliveryOptions = {
+        startRetryCount: retryCount
+      };
      
-      // Determine if this is a retry
+      // In a real implementation with an actual HTTP request, this would call
+      // the deliverWebhook function. For now, we'll simulate the delivery
+      // to avoid making actual HTTP requests in the demo environment.
+      
+      // Simulated webhook delivery (for demo purposes only)
      const isRetry = retryCount > 0;
-      console.log(`${isRetry ? 'Retrying' : 'Triggering'} webhook delivery (attempt ${retryCount + 1})`);
-      
-      // In a real implementation, this would make the actual HTTP request
-      // For now, simulate a successful delivery
      const deliveryResult = {
        success: true,
        statusCode: 200,
-        message: `Webhook delivered successfully (simulated)${isRetry ? ' after retry' : ''}`,
+        message: isRetry ? 
+          `Webhook delivered successfully after ${retryCount} ${retryCount === 1 ? 'retry' : 'retries'} (simulated)` : 
+          'Webhook delivered successfully (simulated)',
        timestamp: new Date(),
-        responseBody: JSON.stringify({ success: true }),
-        retryCount: retryCount
+        responseBody: JSON.stringify({ 
+          success: true, 
+          received_at: new Date().toISOString(),
+          message: "Webhook received successfully"
+        }),
+        retryCount
      };
      
+      // In a production environment, you would use the actual delivery code:
+      // const deliveryResult = await deliverWebhook(webhook, payload, deliveryOptions);
+      
      // Log the delivery attempt
      await this.addWebhookDeliveryLog({
        webhookId: webhook.id,
        event: payload.event || 'unknown',
        payload,
-        signature,
+        signature: webhook.secret ? 'simulated-signature' : '',
        status: deliveryResult.success,
        statusCode: deliveryResult.statusCode,
        message: deliveryResult.message,
@@ -2,6 +2,12 @@ import { createHmac } from 'crypto';
 import fetch from 'node-fetch';
 import { Webhook } from '@shared/schema';

+// Constants for the retry mechanism
+const DEFAULT_MAX_RETRIES = 3;
+const DEFAULT_INITIAL_DELAY = 2000; // 2 seconds
+const DEFAULT_MAX_DELAY = 60000; // 60 seconds
+const DEFAULT_TIMEOUT = 10000; // 10 seconds
+
 // Generates a signature for a webhook payload using the webhook's secret
 export function generateSignature(payload: any, secret: string): string {
  if (!secret) return '';
@@ -33,27 +39,55 @@ export interface WebhookDeliveryLog {
 }

 /**
- * Delivers a webhook payload to the configured URL with retry capability
+ * Options for webhook delivery
+ */
+export interface WebhookDeliveryOptions {
+  startRetryCount?: number;  // For continuing retry attempts from previous failures
+  maxRetries?: number;       // Maximum number of retry attempts
+  initialDelayMs?: number;   // Initial delay in milliseconds between retries
+  maxDelayMs?: number;       // Maximum delay between retries
+  timeoutMs?: number;        // Request timeout in milliseconds
+  jitter?: boolean;          // Add random jitter to delay to avoid thundering herd
+}
+
+/**
+ * Delivers a webhook payload to the configured URL with exponential backoff retry
 */
 export async function deliverWebhook(
  webhook: Webhook, 
-  payload: any, 
-  maxRetries = 3,
-  initialRetryDelay = 2000
+  payload: any,
+  options: WebhookDeliveryOptions = {}
 ): Promise<WebhookDeliveryResult> {
-  let retryCount = 0;
+  // Set defaults for options
+  const retryOptions = {
+    startRetryCount: options.startRetryCount || 0,
+    maxRetries: options.maxRetries !== undefined ? options.maxRetries : DEFAULT_MAX_RETRIES,
+    initialDelayMs: options.initialDelayMs || DEFAULT_INITIAL_DELAY,
+    maxDelayMs: options.maxDelayMs || DEFAULT_MAX_DELAY,
+    timeoutMs: options.timeoutMs || DEFAULT_TIMEOUT,
+    jitter: options.jitter !== undefined ? options.jitter : true
+  };
+  
+  let retryCount = retryOptions.startRetryCount;
  let lastError: Error | null = null;
  
  // Generate HMAC signature if webhook has a secret
  const signature = webhook.secret ? generateSignature(payload, webhook.secret) : '';
  
-  while (retryCount <= maxRetries) {
+  // Add retry count information to the payload if this is a retry
+  const isRetry = retryCount > 0;
+  const retryPayload = isRetry ? { ...payload, _retryCount: retryCount } : payload;
+  
+  // Log the current retry attempt
+  console.log(`${isRetry ? 'Retrying' : 'Delivering'} webhook ${webhook.id} (attempt ${retryCount + 1})`);
+  
+  while (retryCount <= retryOptions.maxRetries) {
    try {
      const headers: Record<string, string> = {
        'Content-Type': 'application/json',
        'User-Agent': 'DNS-Manager-Webhook/1.0',
        'X-Webhook-ID': webhook.id,
-        'X-Webhook-Event': payload.event
+        'X-Webhook-Event': payload.event || 'unknown'
      };
      
      // Add signature header if available
@@ -61,38 +95,80 @@ export async function deliverWebhook(
        headers['X-Webhook-Signature'] = signature;
      }
      
-      const response = await fetch(webhook.url, {
-        method: 'POST',
-        headers,
-        body: JSON.stringify(payload),
-        timeout: 10000 // 10 second timeout
-      });
+      // Add retry information if this is a retry attempt
+      if (isRetry) {
+        headers['X-Webhook-Retry-Count'] = retryCount.toString();
+      }
      
-      const responseText = await response.text();
+      // Make the actual HTTP request to deliver the webhook
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), retryOptions.timeoutMs);
      
-      if (response.ok) {
-        return {
-          success: true,
-          statusCode: response.status,
-          message: 'Webhook delivered successfully',
-          timestamp: new Date(),
-          responseBody: responseText,
-          retryCount
+      try {
+        // Use node-fetch with proper typing
+        const fetchOptions: RequestInit & { signal?: AbortSignal } = {
+          method: 'POST',
+          headers,
+          body: JSON.stringify(retryPayload),
+          signal: controller.signal
        };
-      } else {
-        lastError = new Error(`HTTP error ${response.status}: ${responseText}`);
        
-        // If we got a response but it's an error, check if it's a 5xx error that warrants retrying
-        if (response.status < 500 || retryCount >= maxRetries) {
-          break; // Don't retry client errors or if we've exhausted retries
+        const response = await fetch(webhook.url, fetchOptions);
+        
+        clearTimeout(timeoutId);
+        const responseText = await response.text();
+        
+        if (response.ok) {
+          // Successful delivery
+          return {
+            success: true,
+            statusCode: response.status,
+            message: isRetry ? 
+              `Webhook delivered successfully after ${retryCount} ${retryCount === 1 ? 'retry' : 'retries'}` : 
+              'Webhook delivered successfully',
+            timestamp: new Date(),
+            responseBody: responseText,
+            retryCount
+          };
+        } else {
+          lastError = new Error(`HTTP error ${response.status}: ${responseText}`);
+          
+          // Don't retry 4xx client errors (except 429 Too Many Requests)
+          if ((response.status >= 400 && response.status < 500 && response.status !== 429) || 
+              retryCount >= retryOptions.maxRetries) {
+            break;
+          }
        }
+      } catch (fetchError) {
+        clearTimeout(timeoutId);
+        lastError = fetchError as Error;
+        
+        // Network errors and timeouts are generally retriable
+        console.error(`Webhook delivery error (will retry): ${lastError.message}`);
      }
    } catch (error) {
      lastError = error as Error;
+      console.error(`Unexpected error in webhook delivery: ${lastError.message}`);
    }
    
-    // Exponential backoff for retries
-    const delay = initialRetryDelay * Math.pow(2, retryCount);
+    // If we've hit max retries, break out
+    if (retryCount >= retryOptions.maxRetries) {
+      break;
+    }
+    
+    // Calculate the next delay with exponential backoff
+    let delay = retryOptions.initialDelayMs * Math.pow(2, retryCount);
+    
+    // Cap the delay at the maximum allowed
+    delay = Math.min(delay, retryOptions.maxDelayMs);
+    
+    // Add jitter (±25%) to avoid thundering herd problem if many webhooks retry at once
+    if (retryOptions.jitter) {
+      const jitterFactor = 0.75 + (Math.random() * 0.5); // Random between 0.75 and 1.25
+      delay = Math.floor(delay * jitterFactor);
+    }
+    
+    console.log(`Waiting ${delay}ms before retry #${retryCount + 1} for webhook ${webhook.id}`);
    await new Promise(resolve => setTimeout(resolve, delay));
    retryCount++;
  }
@@ -100,7 +176,9 @@ export async function deliverWebhook(
  return {
    success: false,
    statusCode: lastError && 'statusCode' in lastError ? (lastError as any).statusCode : undefined,
-    message: lastError ? lastError.message : 'Unknown error delivering webhook',
+    message: lastError ? 
+      `Failed to deliver webhook after ${retryCount} ${retryCount === 1 ? 'attempt' : 'attempts'}: ${lastError.message}` : 
+      'Unknown error delivering webhook',
    timestamp: new Date(),
    retryCount
  };