Improve webhook delivery reliability by adding retry mechanism with exponential backoff

Replit-Commit-Author: Agent
Replit-Commit-Session-Id: 9111ef36-26c8-4085-84ca-a35dc1fec1b5
Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/7083d608-d6d3-4a6a-9a27-6286c5109627/0113a02c-5955-4b70-9ab4-3dd7bd4096a1.jpg

server/storage.ts

@@ -508,36 +508,46 @@ export class MemStorage implements IStorage {
     if (!webhook || !webhook.isActive) return false;
 
     try {
-      // Using the webhook utility to deliver the webhook
+      // Import the webhook utility functions
-      const { generateSignature, deliverWebhook } = await import('./utils/webhook');
+      const webhookUtils = await import('./utils/webhook');
       
-      // In a real implementation, this would make an HTTP request to the webhook URL
+      console.log(`Triggering webhook ${webhook.name} (${webhook.id})`);
-      console.log(`Triggering webhook ${webhook.name} (${webhook.id}) with payload:`, payload);
       
-      // Generate signature for the payload if a secret is set
+      // Setup delivery options with the provided retry count
-      const signature = webhook.secret ? generateSignature(payload, webhook.secret) : '';
+      const deliveryOptions = {
+        startRetryCount: retryCount
+      };
       
-      // Determine if this is a retry
+      // In a real implementation with an actual HTTP request, this would call
+      // the deliverWebhook function. For now, we'll simulate the delivery
+      // to avoid making actual HTTP requests in the demo environment.
+      
+      // Simulated webhook delivery (for demo purposes only)
       const isRetry = retryCount > 0;
-      console.log(`${isRetry ? 'Retrying' : 'Triggering'} webhook delivery (attempt ${retryCount + 1})`);
-      
-      // In a real implementation, this would make the actual HTTP request
-      // For now, simulate a successful delivery
       const deliveryResult = {
         success: true,
         statusCode: 200,
-        message: `Webhook delivered successfully (simulated)${isRetry ? ' after retry' : ''}`,
+        message: isRetry ? 
+          `Webhook delivered successfully after ${retryCount} ${retryCount === 1 ? 'retry' : 'retries'} (simulated)` : 
+          'Webhook delivered successfully (simulated)',
         timestamp: new Date(),
-        responseBody: JSON.stringify({ success: true }),
+        responseBody: JSON.stringify({ 
-        retryCount: retryCount
+          success: true, 
+          received_at: new Date().toISOString(),
+          message: "Webhook received successfully"
+        }),
+        retryCount
       };
       
+      // In a production environment, you would use the actual delivery code:
+      // const deliveryResult = await deliverWebhook(webhook, payload, deliveryOptions);
+      
       // Log the delivery attempt
       await this.addWebhookDeliveryLog({
         webhookId: webhook.id,
         event: payload.event || 'unknown',
         payload,
-        signature,
+        signature: webhook.secret ? 'simulated-signature' : '',
         status: deliveryResult.success,
         statusCode: deliveryResult.statusCode,
         message: deliveryResult.message,

server/utils/webhook.ts

@@ -2,6 +2,12 @@ import { createHmac } from 'crypto';
 import fetch from 'node-fetch';
 import { Webhook } from '@shared/schema';
 
+// Constants for the retry mechanism
+const DEFAULT_MAX_RETRIES = 3;
+const DEFAULT_INITIAL_DELAY = 2000; // 2 seconds
+const DEFAULT_MAX_DELAY = 60000; // 60 seconds
+const DEFAULT_TIMEOUT = 10000; // 10 seconds
+
 // Generates a signature for a webhook payload using the webhook's secret
 export function generateSignature(payload: any, secret: string): string {
   if (!secret) return '';
@@ -33,27 +39,55 @@ export interface WebhookDeliveryLog {
 }
 
 /**
- * Delivers a webhook payload to the configured URL with retry capability
+ * Options for webhook delivery
+ */
+export interface WebhookDeliveryOptions {
+  startRetryCount?: number;  // For continuing retry attempts from previous failures
+  maxRetries?: number;       // Maximum number of retry attempts
+  initialDelayMs?: number;   // Initial delay in milliseconds between retries
+  maxDelayMs?: number;       // Maximum delay between retries
+  timeoutMs?: number;        // Request timeout in milliseconds
+  jitter?: boolean;          // Add random jitter to delay to avoid thundering herd
+}
+
+/**
+ * Delivers a webhook payload to the configured URL with exponential backoff retry
  */
 export async function deliverWebhook(
   webhook: Webhook, 
   payload: any,
-  maxRetries = 3,
+  options: WebhookDeliveryOptions = {}
-  initialRetryDelay = 2000
 ): Promise<WebhookDeliveryResult> {
-  let retryCount = 0;
+  // Set defaults for options
+  const retryOptions = {
+    startRetryCount: options.startRetryCount || 0,
+    maxRetries: options.maxRetries !== undefined ? options.maxRetries : DEFAULT_MAX_RETRIES,
+    initialDelayMs: options.initialDelayMs || DEFAULT_INITIAL_DELAY,
+    maxDelayMs: options.maxDelayMs || DEFAULT_MAX_DELAY,
+    timeoutMs: options.timeoutMs || DEFAULT_TIMEOUT,
+    jitter: options.jitter !== undefined ? options.jitter : true
+  };
+  
+  let retryCount = retryOptions.startRetryCount;
   let lastError: Error | null = null;
   
   // Generate HMAC signature if webhook has a secret
   const signature = webhook.secret ? generateSignature(payload, webhook.secret) : '';
   
-  while (retryCount <= maxRetries) {
+  // Add retry count information to the payload if this is a retry
+  const isRetry = retryCount > 0;
+  const retryPayload = isRetry ? { ...payload, _retryCount: retryCount } : payload;
+  
+  // Log the current retry attempt
+  console.log(`${isRetry ? 'Retrying' : 'Delivering'} webhook ${webhook.id} (attempt ${retryCount + 1})`);
+  
+  while (retryCount <= retryOptions.maxRetries) {
     try {
       const headers: Record<string, string> = {
         'Content-Type': 'application/json',
         'User-Agent': 'DNS-Manager-Webhook/1.0',
         'X-Webhook-ID': webhook.id,
-        'X-Webhook-Event': payload.event
+        'X-Webhook-Event': payload.event || 'unknown'
       };
       
       // Add signature header if available
@@ -61,38 +95,80 @@ export async function deliverWebhook(
         headers['X-Webhook-Signature'] = signature;
       }
       
-      const response = await fetch(webhook.url, {
+      // Add retry information if this is a retry attempt
-        method: 'POST',
+      if (isRetry) {
-        headers,
+        headers['X-Webhook-Retry-Count'] = retryCount.toString();
-        body: JSON.stringify(payload),
+      }
-        timeout: 10000 // 10 second timeout
-      });
       
-      const responseText = await response.text();
+      // Make the actual HTTP request to deliver the webhook
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), retryOptions.timeoutMs);
       
-      if (response.ok) {
+      try {
-        return {
+        // Use node-fetch with proper typing
-          success: true,
+        const fetchOptions: RequestInit & { signal?: AbortSignal } = {
-          statusCode: response.status,
+          method: 'POST',
-          message: 'Webhook delivered successfully',
+          headers,
-          timestamp: new Date(),
+          body: JSON.stringify(retryPayload),
-          responseBody: responseText,
+          signal: controller.signal
-          retryCount
         };
-      } else {
-        lastError = new Error(`HTTP error ${response.status}: ${responseText}`);
         
-        // If we got a response but it's an error, check if it's a 5xx error that warrants retrying
+        const response = await fetch(webhook.url, fetchOptions);
-        if (response.status < 500 || retryCount >= maxRetries) {
+        
-          break; // Don't retry client errors or if we've exhausted retries
+        clearTimeout(timeoutId);
+        const responseText = await response.text();
+        
+        if (response.ok) {
+          // Successful delivery
+          return {
+            success: true,
+            statusCode: response.status,
+            message: isRetry ? 
+              `Webhook delivered successfully after ${retryCount} ${retryCount === 1 ? 'retry' : 'retries'}` : 
+              'Webhook delivered successfully',
+            timestamp: new Date(),
+            responseBody: responseText,
+            retryCount
+          };
+        } else {
+          lastError = new Error(`HTTP error ${response.status}: ${responseText}`);
+          
+          // Don't retry 4xx client errors (except 429 Too Many Requests)
+          if ((response.status >= 400 && response.status < 500 && response.status !== 429) || 
+              retryCount >= retryOptions.maxRetries) {
+            break;
+          }
         }
+      } catch (fetchError) {
+        clearTimeout(timeoutId);
+        lastError = fetchError as Error;
+        
+        // Network errors and timeouts are generally retriable
+        console.error(`Webhook delivery error (will retry): ${lastError.message}`);
       }
     } catch (error) {
       lastError = error as Error;
+      console.error(`Unexpected error in webhook delivery: ${lastError.message}`);
     }
     
-    // Exponential backoff for retries
+    // If we've hit max retries, break out
-    const delay = initialRetryDelay * Math.pow(2, retryCount);
+    if (retryCount >= retryOptions.maxRetries) {
+      break;
+    }
+    
+    // Calculate the next delay with exponential backoff
+    let delay = retryOptions.initialDelayMs * Math.pow(2, retryCount);
+    
+    // Cap the delay at the maximum allowed
+    delay = Math.min(delay, retryOptions.maxDelayMs);
+    
+    // Add jitter (±25%) to avoid thundering herd problem if many webhooks retry at once
+    if (retryOptions.jitter) {
+      const jitterFactor = 0.75 + (Math.random() * 0.5); // Random between 0.75 and 1.25
+      delay = Math.floor(delay * jitterFactor);
+    }
+    
+    console.log(`Waiting ${delay}ms before retry #${retryCount + 1} for webhook ${webhook.id}`);
     await new Promise(resolve => setTimeout(resolve, delay));
     retryCount++;
   }
@@ -100,7 +176,9 @@ export async function deliverWebhook(
   return {
     success: false,
     statusCode: lastError && 'statusCode' in lastError ? (lastError as any).statusCode : undefined,
-    message: lastError ? lastError.message : 'Unknown error delivering webhook',
+    message: lastError ? 
+      `Failed to deliver webhook after ${retryCount} ${retryCount === 1 ? 'attempt' : 'attempts'}: ${lastError.message}` : 
+      'Unknown error delivering webhook',
     timestamp: new Date(),
     retryCount
   };