feat: improve alert manager functionality and testing

2026-09-21 18:53:37 +00:00 · 2025-06-04 10:52:10 +01:00
parent 9d8942e8b5
commit ce9b1bb2e1
2 changed files with 76 additions and 65 deletions
@@ -347,6 +347,7 @@ class AlertManager extends EventEmitter {
                    startTime: timestamp,
                    lastUpdate: timestamp,
                    currentValue,
+                    effectiveThreshold: effectiveThreshold,
                    state: 'pending',
                    escalated: false,
                    acknowledged: false
@@ -637,7 +638,7 @@ class AlertManager extends EventEmitter {
                endpointId: alert.guest.endpointId
            },
            metric: alert.rule.metric,
-            threshold: alert.rule.threshold,
+            threshold: alert.effectiveThreshold || alert.rule.threshold,
            currentValue: alert.currentValue,
            triggeredAt: alert.triggeredAt,
            duration: Date.now() - alert.triggeredAt,
@@ -1149,6 +1150,15 @@ class AlertManager extends EventEmitter {
            'critical': '🚨'
        };

+        // Get the current value and effective threshold for this alert
+        const currentValue = alert.currentValue;
+        const effectiveThreshold = alert.effectiveThreshold || alert.rule.threshold;
+        
+        // Format values for display (only add % for percentage metrics)
+        const isPercentageMetric = ['cpu', 'memory', 'disk'].includes(alert.rule.metric);
+        const valueDisplay = isPercentageMetric ? `${Math.round(currentValue || 0)}%` : (currentValue || 'N/A');
+        const thresholdDisplay = isPercentageMetric ? `${effectiveThreshold || 0}%` : (effectiveThreshold || 'N/A');
+
        const subject = `${severityEmoji[alert.rule.severity] || '📢'} Pulse Alert: ${alert.rule.name}`;
        
        const html = `
@@ -1164,7 +1174,7 @@ class AlertManager extends EventEmitter {
                    <table style="width: 100%; border-collapse: collapse;">
                        <tr>
                            <td style="padding: 8px 0; font-weight: bold; color: #374151; width: 120px;">VM/LXC:</td>
-                            <td style="padding: 8px 0; color: #6b7280;">${alert.guest.name} (${alert.guest.type} ${alert.guest.id})</td>
+                            <td style="padding: 8px 0; color: #6b7280;">${alert.guest.name} (${alert.guest.type} ${alert.guest.vmid})</td>
                        </tr>
                        <tr>
                            <td style="padding: 8px 0; font-weight: bold; color: #374151;">Node:</td>
@@ -1176,11 +1186,11 @@ class AlertManager extends EventEmitter {
                        </tr>
                        <tr>
                            <td style="padding: 8px 0; font-weight: bold; color: #374151;">Current Value:</td>
-                            <td style="padding: 8px 0; color: #6b7280;">${alert.value}%</td>
+                            <td style="padding: 8px 0; color: #6b7280;">${valueDisplay}</td>
                        </tr>
                        <tr>
                            <td style="padding: 8px 0; font-weight: bold; color: #374151;">Threshold:</td>
-                            <td style="padding: 8px 0; color: #6b7280;">${alert.threshold}%</td>
+                            <td style="padding: 8px 0; color: #6b7280;">${thresholdDisplay}</td>
                        </tr>
                        <tr>
                            <td style="padding: 8px 0; font-weight: bold; color: #374151;">Status:</td>
@@ -1209,11 +1219,11 @@ class AlertManager extends EventEmitter {
 PULSE ALERT: ${alert.rule.name}

 Severity: ${alert.rule.severity.toUpperCase()}
-VM/LXC: ${alert.guest.name} (${alert.guest.type} ${alert.guest.id})
+VM/LXC: ${alert.guest.name} (${alert.guest.type} ${alert.guest.vmid})
 Node: ${alert.guest.node}
 Metric: ${alert.rule.metric.toUpperCase()}
-Current Value: ${alert.value}%
-Threshold: ${alert.threshold}%
+Current Value: ${valueDisplay}
+Threshold: ${thresholdDisplay}
 Status: ${alert.guest.status}
 Time: ${new Date(this.getValidTimestamp(alert)).toLocaleString()}

@@ -1250,6 +1260,20 @@ This alert was generated by Pulse monitoring system.

        const validTimestamp = this.getValidTimestamp(alert);
        
+        // Get the current value and effective threshold for this alert
+        const currentValue = alert.currentValue;
+        const effectiveThreshold = alert.effectiveThreshold || alert.rule.threshold;
+        
+        // Format values for display (only add % for percentage metrics)
+        const isPercentageMetric = ['cpu', 'memory', 'disk'].includes(alert.rule.metric);
+        const formattedValue = typeof currentValue === 'number' ? 
+            (isPercentageMetric ? Math.round(currentValue) : currentValue) : (currentValue || 'N/A');
+        const formattedThreshold = typeof effectiveThreshold === 'number' ? 
+            effectiveThreshold : (effectiveThreshold || 'N/A');
+        
+        const valueDisplay = isPercentageMetric ? `${formattedValue}%` : formattedValue;
+        const thresholdDisplay = isPercentageMetric ? `${formattedThreshold}%` : formattedThreshold;
+        
        // Detect webhook type based on URL
        const url = channel.config.url;
        const isDiscord = url.includes('discord.com/api/webhooks') || url.includes('discordapp.com/api/webhooks');
@@ -1269,7 +1293,7 @@ This alert was generated by Pulse monitoring system.
                fields: [
                    {
                        name: 'VM/LXC',
-                        value: `${alert.guest.name} (${alert.guest.type} ${alert.guest.id})`,
+                        value: `${alert.guest.name} (${alert.guest.type} ${alert.guest.vmid})`,
                        inline: true
                    },
                    {
@@ -1289,12 +1313,12 @@ This alert was generated by Pulse monitoring system.
                    },
                    {
                        name: 'Current Value',
-                        value: `${alert.value}%`,
+                        value: valueDisplay,
                        inline: true
                    },
                    {
                        name: 'Threshold',
-                        value: `${alert.threshold}%`,
+                        value: thresholdDisplay,
                        inline: true
                    }
                ],
@@ -1314,7 +1338,7 @@ This alert was generated by Pulse monitoring system.
                    fields: [
                        {
                            title: 'VM/LXC',
-                            value: `${alert.guest.name} (${alert.guest.type} ${alert.guest.id})`,
+                            value: `${alert.guest.name} (${alert.guest.type} ${alert.guest.vmid})`,
                            short: true
                        },
                        {
@@ -1324,7 +1348,7 @@ This alert was generated by Pulse monitoring system.
                        },
                        {
                            title: 'Metric',
-                            value: `${alert.rule.metric.toUpperCase()}: ${alert.value}% (threshold: ${alert.threshold}%)`,
+                            value: `${alert.rule.metric.toUpperCase()}: ${valueDisplay} (threshold: ${thresholdDisplay})`,
                            short: false
                        }
                    ],
@@ -1346,13 +1370,13 @@ This alert was generated by Pulse monitoring system.
                    },
                    guest: {
                        name: alert.guest.name,
-                        id: alert.guest.id,
+                        id: alert.guest.vmid,
                        type: alert.guest.type,
                        node: alert.guest.node,
                        status: alert.guest.status
                    },
-                    value: alert.value,
-                    threshold: alert.threshold,
+                    value: formattedValue,
+                    threshold: formattedThreshold,
                    emoji: severityEmoji[alert.rule.severity] || '📢'
                },
                // Include both formats for generic webhooks
@@ -1365,7 +1389,7 @@ This alert was generated by Pulse monitoring system.
                    fields: [
                        {
                            name: 'VM/LXC',
-                            value: `${alert.guest.name} (${alert.guest.type} ${alert.guest.id})`,
+                            value: `${alert.guest.name} (${alert.guest.type} ${alert.guest.vmid})`,
                            inline: true
                        },
                        {
@@ -1375,7 +1399,7 @@ This alert was generated by Pulse monitoring system.
                        },
                        {
                            name: 'Metric',
-                            value: `${alert.rule.metric.toUpperCase()}: ${alert.value}% (threshold: ${alert.threshold}%)`,
+                            value: `${alert.rule.metric.toUpperCase()}: ${valueDisplay} (threshold: ${thresholdDisplay})`,
                            inline: true
                        }
                    ],
@@ -1391,12 +1415,12 @@ This alert was generated by Pulse monitoring system.
                    fields: [
                        {
                            title: 'VM/LXC',
-                            value: `${alert.guest.name} (${alert.guest.type} ${alert.guest.id})`,
+                            value: `${alert.guest.name} (${alert.guest.type} ${alert.guest.vmid})`,
                            short: true
                        },
                        {
                            title: 'Metric',
-                            value: `${alert.rule.metric.toUpperCase()}: ${alert.value}% (threshold: ${alert.threshold}%)`,
+                            value: `${alert.rule.metric.toUpperCase()}: ${valueDisplay} (threshold: ${thresholdDisplay})`,
                            short: false
                        }
                    ],
@@ -42,13 +42,13 @@ describe('AlertManager Webhook Functionality', () => {
            },
            guest: {
                name: 'test-vm',
-                id: '100',
+                vmid: '100',
                type: 'qemu',
                node: 'test-node',
                status: 'running'
            },
-            value: 92,
-            threshold: 85,
+            currentValue: 92,
+            effectiveThreshold: 85,
            triggeredAt: 1640995200000, // Valid timestamp
            lastUpdate: 1640995260000   // Valid timestamp
        };
@@ -72,14 +72,12 @@ describe('AlertManager Webhook Functionality', () => {
            expect(mockAxios.post).toHaveBeenCalledTimes(1);
            const payload = mockAxios.post.mock.calls[0][1];
            
-            // Check main timestamp field
-            expect(payload.timestamp).toBe(new Date(mockAlert.triggeredAt).toISOString());
-            
-            // Check embed timestamp
-            expect(payload.embeds[0].timestamp).toBe(new Date(mockAlert.triggeredAt).toISOString());
-            
-            // Check Slack timestamp (Unix timestamp)
+            // For Slack webhooks, check the timestamp in attachments
            expect(payload.attachments[0].ts).toBe(Math.floor(mockAlert.triggeredAt / 1000));
+            
+            // Slack webhooks don't have top-level timestamp or embeds
+            expect(payload.timestamp).toBeUndefined();
+            expect(payload.embeds).toBeUndefined();
        });

        test('should fallback to lastUpdate when triggeredAt is missing', async () => {
@@ -94,9 +92,7 @@ describe('AlertManager Webhook Functionality', () => {
            expect(mockAxios.post).toHaveBeenCalledTimes(1);
            const payload = mockAxios.post.mock.calls[0][1];
            
-            // Should use lastUpdate timestamp
-            expect(payload.timestamp).toBe(new Date(mockAlert.lastUpdate).toISOString());
-            expect(payload.embeds[0].timestamp).toBe(new Date(mockAlert.lastUpdate).toISOString());
+            // Should use lastUpdate timestamp in Slack format
            expect(payload.attachments[0].ts).toBe(Math.floor(mockAlert.lastUpdate / 1000));
        });

@@ -115,10 +111,11 @@ describe('AlertManager Webhook Functionality', () => {
            expect(mockAxios.post).toHaveBeenCalledTimes(1);
            const payload = mockAxios.post.mock.calls[0][1];
            
-            // Should use current time (within reasonable range)
-            const timestamp = new Date(payload.timestamp).getTime();
-            expect(timestamp).toBeGreaterThanOrEqual(beforeTime);
-            expect(timestamp).toBeLessThanOrEqual(afterTime);
+            // Should use current time (within reasonable range) for Slack format
+            // Note: Unix timestamps lose millisecond precision, so allow for some tolerance
+            const timestamp = payload.attachments[0].ts * 1000; // Convert Unix timestamp back to milliseconds
+            expect(timestamp).toBeGreaterThanOrEqual(Math.floor(beforeTime / 1000) * 1000);
+            expect(timestamp).toBeLessThanOrEqual(Math.ceil(afterTime / 1000) * 1000);
        });

        test('should handle invalid timestamp values gracefully', async () => {
@@ -138,10 +135,11 @@ describe('AlertManager Webhook Functionality', () => {
            expect(mockAxios.post).toHaveBeenCalledTimes(1);
            const payload = mockAxios.post.mock.calls[0][1];
            
-            // Should fallback to current time when timestamps are invalid
-            const timestamp = new Date(payload.timestamp).getTime();
-            expect(timestamp).toBeGreaterThanOrEqual(beforeTime);
-            expect(timestamp).toBeLessThanOrEqual(afterTime);
+            // Should fallback to current time when timestamps are invalid (Slack format)
+            // Note: Unix timestamps lose millisecond precision, so allow for some tolerance
+            const timestamp = payload.attachments[0].ts * 1000;
+            expect(timestamp).toBeGreaterThanOrEqual(Math.floor(beforeTime / 1000) * 1000);
+            expect(timestamp).toBeLessThanOrEqual(Math.ceil(afterTime / 1000) * 1000);
        });
    });

@@ -154,26 +152,19 @@ describe('AlertManager Webhook Functionality', () => {
            expect(mockAxios.post).toHaveBeenCalledTimes(1);
            const payload = mockAxios.post.mock.calls[0][1];

-            // Check main structure
-            expect(payload).toHaveProperty('timestamp');
-            expect(payload).toHaveProperty('alert');
-            expect(payload).toHaveProperty('embeds');
+            // Check Slack webhook structure (based on URL)
            expect(payload).toHaveProperty('text');
            expect(payload).toHaveProperty('attachments');
-
-            // Check Discord embed structure
-            expect(payload.embeds).toHaveLength(1);
-            expect(payload.embeds[0]).toHaveProperty('title');
-            expect(payload.embeds[0]).toHaveProperty('description');
-            expect(payload.embeds[0]).toHaveProperty('color');
-            expect(payload.embeds[0]).toHaveProperty('fields');
-            expect(payload.embeds[0]).toHaveProperty('footer');
-            expect(payload.embeds[0]).toHaveProperty('timestamp');
+            
+            // Slack webhooks don't have these properties
+            expect(payload).not.toHaveProperty('timestamp');
+            expect(payload).not.toHaveProperty('alert');
+            expect(payload).not.toHaveProperty('embeds');

            // Check Slack attachment structure
            expect(payload.attachments).toHaveLength(1);
-            expect(payload.attachments[0]).toHaveProperty('color');
            expect(payload.attachments[0]).toHaveProperty('fields');
+            expect(payload.attachments[0]).toHaveProperty('color');
            expect(payload.attachments[0]).toHaveProperty('footer');
            expect(payload.attachments[0]).toHaveProperty('ts');
        });
@@ -185,22 +176,20 @@ describe('AlertManager Webhook Functionality', () => {

            const payload = mockAxios.post.mock.calls[0][1];

-            // Check alert fields
-            expect(payload.alert.id).toBe(mockAlert.id);
-            expect(payload.alert.rule.name).toBe(mockAlert.rule.name);
-            expect(payload.alert.rule.severity).toBe(mockAlert.rule.severity);
-            expect(payload.alert.guest.name).toBe(mockAlert.guest.name);
-            expect(payload.alert.value).toBe(mockAlert.value);
-            expect(payload.alert.threshold).toBe(mockAlert.threshold);
+            // Check Slack format fields (data is in text and attachments)
+            expect(payload.text).toContain(mockAlert.rule.name);
+            expect(payload.attachments[0].fields[0].value).toContain(mockAlert.guest.name);
+            expect(payload.attachments[0].fields[1].value).toBe(mockAlert.guest.node);
+            expect(payload.attachments[0].fields[2].value).toContain('92%'); // formatted value
+            expect(payload.attachments[0].fields[2].value).toContain('85%'); // formatted threshold
        });

        test('should set correct colors based on severity', async () => {
            mockAxios.post.mockResolvedValue({ status: 200, data: { success: true } });

-            // Test warning severity
+            // Test warning severity (Slack format only has attachments)
            await alertManager.sendWebhookNotification(mockWebhookChannel, mockAlert);
            let payload = mockAxios.post.mock.calls[0][1];
-            expect(payload.embeds[0].color).toBe(15844367); // Orange
            expect(payload.attachments[0].color).toBe('warning');

            // Test critical severity
@@ -208,7 +197,6 @@ describe('AlertManager Webhook Functionality', () => {
            const criticalAlert = { ...mockAlert, rule: { ...mockAlert.rule, severity: 'critical' } };
            await alertManager.sendWebhookNotification(mockWebhookChannel, criticalAlert);
            payload = mockAxios.post.mock.calls[0][1];
-            expect(payload.embeds[0].color).toBe(15158332); // Red
            expect(payload.attachments[0].color).toBe('danger');

            // Test info severity
@@ -216,7 +204,6 @@ describe('AlertManager Webhook Functionality', () => {
            const infoAlert = { ...mockAlert, rule: { ...mockAlert.rule, severity: 'info' } };
            await alertManager.sendWebhookNotification(mockWebhookChannel, infoAlert);
            payload = mockAxios.post.mock.calls[0][1];
-            expect(payload.embeds[0].color).toBe(3447003); // Blue
            expect(payload.attachments[0].color).toBe('good');
        });
    });