diff --git a/data/acknowledgements.json b/data/acknowledgements.json index f6d858a06..9e26dfeeb 100644 --- a/data/acknowledgements.json +++ b/data/acknowledgements.json @@ -1,482 +1 @@ -{ - "guest_down_primary_desktop_102": { - "id": "alert_1748685507709_c0qu2nodx", - "rule": { - "id": "guest_down", - "name": "Guest System Down", - "description": "Virtual machine or container has stopped unexpectedly", - "metric": "status", - "condition": "equals", - "threshold": "stopped", - "duration": 60000, - "severity": "critical", - "enabled": true, - "tags": [ - "availability", - "guest" - ], - "group": "availability_alerts", - "escalationTime": 600000, - "autoResolve": true, - "suppressionTime": 120000, - "notificationChannels": [ - "default", - "urgent" - ] - }, - "guest": { - "diskread": 0, - "serial": 1, - "vmid": 102, - "cpus": 4, - "name": "windows11", - "netout": 0, - "maxmem": 8589934592, - "uptime": 0, - "diskwrite": 0, - "disk": 0, - "cpu": 0, - "netin": 0, - "status": "stopped", - "mem": 0, - "maxdisk": 68719476736, - "node": "desktop", - "endpointId": "primary", - "type": "qemu", - "id": "primary-desktop-102" - }, - "startTime": 1748685507709, - "lastUpdate": 1748688477886, - "currentValue": "stopped", - "state": "active", - "escalated": true, - "acknowledged": true, - "triggeredAt": 1748685568177, - "escalatedAt": 1748686208308, - "acknowledgedBy": "bulk-operation", - "acknowledgedAt": 1748688478340, - "acknowledgeNote": "Bulk acknowledged via dropdown", - "note": "Bulk acknowledged via dropdown" - }, - "guest_down_primary_desktop_400": { - "id": "alert_1748685507709_1k93uym8r", - "rule": { - "id": "guest_down", - "name": "Guest System Down", - "description": "Virtual machine or container has stopped unexpectedly", - "metric": "status", - "condition": "equals", - "threshold": "stopped", - "duration": 60000, - "severity": "critical", - "enabled": true, - "tags": [ - "availability", - "guest" - ], - "group": "availability_alerts", - "escalationTime": 600000, - "autoResolve": true, - "suppressionTime": 120000, - "notificationChannels": [ - "default", - "urgent" - ] - }, - "guest": { - "cpu": 0, - "disk": 0, - "maxdisk": 34359738368, - "mem": 0, - "status": "stopped", - "netin": 0, - "vmid": 400, - "diskread": 0, - "diskwrite": 0, - "uptime": 0, - "maxmem": 2147483648, - "netout": 0, - "name": "ubuntu-gpu-vm", - "cpus": 2, - "node": "desktop", - "endpointId": "primary", - "type": "qemu", - "id": "primary-desktop-400" - }, - "startTime": 1748685507709, - "lastUpdate": 1748688477886, - "currentValue": "stopped", - "state": "active", - "escalated": true, - "acknowledged": true, - "triggeredAt": 1748685568177, - "escalatedAt": 1748686208307, - "acknowledgedBy": "bulk-operation", - "acknowledgedAt": 1748688478335, - "acknowledgeNote": "Bulk acknowledged via dropdown", - "note": "Bulk acknowledged via dropdown" - }, - "guest_down_primary_desktop_200": { - "id": "alert_1748685507709_i4z21lcww", - "rule": { - "id": "guest_down", - "name": "Guest System Down", - "description": "Virtual machine or container has stopped unexpectedly", - "metric": "status", - "condition": "equals", - "threshold": "stopped", - "duration": 60000, - "severity": "critical", - "enabled": true, - "tags": [ - "availability", - "guest" - ], - "group": "availability_alerts", - "escalationTime": 600000, - "autoResolve": true, - "suppressionTime": 120000, - "notificationChannels": [ - "default", - "urgent" - ] - }, - "guest": { - "diskread": 0, - "vmid": 200, - "diskwrite": 0, - "name": "UnraidServer", - "cpus": 6, - "maxmem": 8589934592, - "netout": 0, - "uptime": 0, - "cpu": 0, - "disk": 0, - "maxdisk": 0, - "netin": 0, - "status": "stopped", - "mem": 0, - "node": "desktop", - "endpointId": "primary", - "type": "qemu", - "id": "primary-desktop-200" - }, - "startTime": 1748685507709, - "lastUpdate": 1748688477886, - "currentValue": "stopped", - "state": "active", - "escalated": true, - "acknowledged": true, - "triggeredAt": 1748685568177, - "escalatedAt": 1748686208308, - "acknowledgedBy": "bulk-operation", - "acknowledgedAt": 1748688478343, - "acknowledgeNote": "Bulk acknowledged via dropdown", - "note": "Bulk acknowledged via dropdown" - }, - "guest_down_primary_desktop_109": { - "id": "alert_1748685507710_i0hk9rnyu", - "rule": { - "id": "guest_down", - "name": "Guest System Down", - "description": "Virtual machine or container has stopped unexpectedly", - "metric": "status", - "condition": "equals", - "threshold": "stopped", - "duration": 60000, - "severity": "critical", - "enabled": true, - "tags": [ - "availability", - "guest" - ], - "group": "availability_alerts", - "escalationTime": 600000, - "autoResolve": true, - "suppressionTime": 120000, - "notificationChannels": [ - "default", - "urgent" - ] - }, - "guest": { - "diskwrite": 0, - "type": "lxc", - "swap": 0, - "maxswap": 536870912, - "cpus": 4, - "name": "pbs2", - "maxmem": 4294967296, - "netout": 0, - "uptime": 0, - "diskread": 0, - "vmid": 109, - "tags": "backup;community-script", - "maxdisk": 10737418240, - "netin": 0, - "mem": 0, - "status": "stopped", - "cpu": 0, - "disk": 0, - "node": "desktop", - "endpointId": "primary", - "id": "primary-desktop-109" - }, - "startTime": 1748685507709, - "lastUpdate": 1748688477886, - "currentValue": "stopped", - "state": "active", - "escalated": true, - "acknowledged": true, - "triggeredAt": 1748685568177, - "escalatedAt": 1748686208308, - "acknowledgedBy": "bulk-operation", - "acknowledgedAt": 1748688478351, - "acknowledgeNote": "Bulk acknowledged via dropdown", - "note": "Bulk acknowledged via dropdown" - }, - "guest_down_primary_desktop_111": { - "id": "alert_1748685507710_sgmqk37dq", - "rule": { - "id": "guest_down", - "name": "Guest System Down", - "description": "Virtual machine or container has stopped unexpectedly", - "metric": "status", - "condition": "equals", - "threshold": "stopped", - "duration": 60000, - "severity": "critical", - "enabled": true, - "tags": [ - "availability", - "guest" - ], - "group": "availability_alerts", - "escalationTime": 600000, - "autoResolve": true, - "suppressionTime": 120000, - "notificationChannels": [ - "default", - "urgent" - ] - }, - "guest": { - "maxdisk": 2147483648, - "netin": 0, - "mem": 0, - "status": "stopped", - "cpu": 0, - "disk": 0, - "diskwrite": 0, - "type": "lxc", - "swap": 0, - "maxswap": 536870912, - "name": "debian", - "cpus": 1, - "netout": 0, - "maxmem": 536870912, - "uptime": 0, - "diskread": 0, - "vmid": 111, - "tags": "community-script;os", - "node": "desktop", - "endpointId": "primary", - "id": "primary-desktop-111" - }, - "startTime": 1748685507709, - "lastUpdate": 1748688477886, - "currentValue": "stopped", - "state": "active", - "escalated": true, - "acknowledged": true, - "triggeredAt": 1748685568177, - "escalatedAt": 1748686208308, - "acknowledgedBy": "bulk-operation", - "acknowledgedAt": 1748688478347, - "acknowledgeNote": "Bulk acknowledged via dropdown", - "note": "Bulk acknowledged via dropdown" - }, - "guest_down_endpoint_2_pi_100": { - "id": "alert_1748685507710_lujuqfuca", - "rule": { - "id": "guest_down", - "name": "Guest System Down", - "description": "Virtual machine or container has stopped unexpectedly", - "metric": "status", - "condition": "equals", - "threshold": "stopped", - "duration": 60000, - "severity": "critical", - "enabled": true, - "tags": [ - "availability", - "guest" - ], - "group": "availability_alerts", - "escalationTime": 600000, - "autoResolve": true, - "suppressionTime": 120000, - "notificationChannels": [ - "default", - "urgent" - ] - }, - "guest": { - "maxdisk": 6442450944, - "disk": 0, - "name": "pihole", - "uptime": 0, - "diskread": 0, - "netin": 0, - "maxswap": 536870912, - "type": "lxc", - "netout": 0, - "vmid": 100, - "cpu": 0, - "diskwrite": 0, - "cpus": 1, - "swap": 0, - "tags": "proxmox-helper-scripts", - "status": "stopped", - "maxmem": 536870912, - "mem": 0, - "node": "pi", - "endpointId": "endpoint_2", - "id": "endpoint_2-pi-100" - }, - "startTime": 1748685507709, - "lastUpdate": 1748688477886, - "currentValue": "stopped", - "state": "active", - "escalated": true, - "acknowledged": true, - "triggeredAt": 1748685568177, - "escalatedAt": 1748686208308, - "acknowledgedBy": "bulk-operation", - "acknowledgedAt": 1748688478362, - "acknowledgeNote": "Bulk acknowledged via dropdown", - "note": "Bulk acknowledged via dropdown" - }, - "guest_down_endpoint_2_pi_102": { - "id": "alert_1748685507710_64sv65hre", - "rule": { - "id": "guest_down", - "name": "Guest System Down", - "description": "Virtual machine or container has stopped unexpectedly", - "metric": "status", - "condition": "equals", - "threshold": "stopped", - "duration": 60000, - "severity": "critical", - "enabled": true, - "tags": [ - "availability", - "guest" - ], - "group": "availability_alerts", - "escalationTime": 600000, - "autoResolve": true, - "suppressionTime": 120000, - "notificationChannels": [ - "default", - "urgent" - ] - }, - "guest": { - "diskwrite": 0, - "cpu": 0, - "tags": "proxmox-helper-scripts", - "swap": 0, - "cpus": 1, - "mem": 0, - "maxmem": 536870912, - "status": "stopped", - "name": "pi-docker", - "disk": 0, - "maxdisk": 4401922048, - "netin": 0, - "uptime": 0, - "diskread": 0, - "type": "lxc", - "maxswap": 536870912, - "vmid": 102, - "netout": 0, - "node": "pi", - "endpointId": "endpoint_2", - "id": "endpoint_2-pi-102" - }, - "startTime": 1748685507709, - "lastUpdate": 1748688477886, - "currentValue": "stopped", - "state": "active", - "escalated": true, - "acknowledged": true, - "triggeredAt": 1748685568177, - "escalatedAt": 1748686208308, - "acknowledgedBy": "bulk-operation", - "acknowledgedAt": 1748688478358, - "acknowledgeNote": "Bulk acknowledged via dropdown", - "note": "Bulk acknowledged via dropdown" - }, - "guest_down_endpoint_2_pi_101": { - "id": "alert_1748685507710_qh85y8kjf", - "rule": { - "id": "guest_down", - "name": "Guest System Down", - "description": "Virtual machine or container has stopped unexpectedly", - "metric": "status", - "condition": "equals", - "threshold": "stopped", - "duration": 60000, - "severity": "critical", - "enabled": true, - "tags": [ - "availability", - "guest" - ], - "group": "availability_alerts", - "escalationTime": 600000, - "autoResolve": true, - "suppressionTime": 120000, - "notificationChannels": [ - "default", - "urgent" - ] - }, - "guest": { - "diskwrite": 0, - "cpu": 0, - "tags": "alpine;community-script;os", - "cpus": 1, - "swap": 0, - "status": "stopped", - "maxmem": 536870912, - "mem": 0, - "disk": 0, - "name": "pi-influxdb", - "maxdisk": 2254438400, - "netin": 0, - "diskread": 0, - "uptime": 0, - "maxswap": 536870912, - "type": "lxc", - "vmid": 101, - "netout": 0, - "node": "pi", - "endpointId": "endpoint_2", - "id": "endpoint_2-pi-101" - }, - "startTime": 1748685507709, - "lastUpdate": 1748688477886, - "currentValue": "stopped", - "state": "active", - "escalated": true, - "acknowledged": true, - "triggeredAt": 1748685568177, - "escalatedAt": 1748686208308, - "acknowledgedBy": "bulk-operation", - "acknowledgedAt": 1748688478354, - "acknowledgeNote": "Bulk acknowledged via dropdown", - "note": "Bulk acknowledged via dropdown" - } -} \ No newline at end of file +{} \ No newline at end of file diff --git a/data/alert-rules.json b/data/alert-rules.json index 9e26dfeeb..27f3d3958 100644 --- a/data/alert-rules.json +++ b/data/alert-rules.json @@ -1 +1,31 @@ -{} \ No newline at end of file +{ + "compound_1749159848654_h011bw96f": { + "id": "compound_1749159848654_h011bw96f", + "condition": "greater_than", + "duration": 5000, + "severity": "warning", + "enabled": true, + "tags": [], + "group": "compound_threshold", + "escalationTime": 900000, + "autoResolve": false, + "suppressionTime": 300000, + "notificationChannels": [ + "default" + ], + "type": "compound_threshold", + "name": "Alert for CPU: 2%", + "targetType": "all", + "specificTarget": "", + "thresholds": [ + { + "type": "cpu", + "operator": ">", + "value": 2 + } + ], + "sendEmail": true, + "sendWebhook": false, + "createdAt": 1749159848648 + } +} \ No newline at end of file diff --git a/docs/resilient-dns.md b/docs/resilient-dns.md new file mode 100644 index 000000000..4745264d4 --- /dev/null +++ b/docs/resilient-dns.md @@ -0,0 +1,139 @@ +# Resilient DNS Resolution for Pulse + +## Overview + +Pulse now includes a resilient DNS resolution feature that helps handle DNS failures when connecting to Proxmox VE or PBS instances. This is particularly useful for environments with: + +- Round-robin DNS configurations +- Multiple A records for a single hostname +- DNS servers that may occasionally fail +- Hosts with `.lan` domains that use local DNS servers + +## How It Works + +When resilient DNS is enabled for an endpoint, Pulse will: + +1. **Resolve all IP addresses** for the hostname +2. **Cache DNS results** for 1 minute to reduce DNS queries +3. **Try each IP address** in sequence if one fails +4. **Mark failed IPs** and skip them for 30 seconds +5. **Use stale cache** if DNS resolution completely fails +6. **Remember working IPs** and prioritize them for future requests + +## Configuration + +### Automatic Enablement + +Resilient DNS is automatically enabled for any endpoint with a hostname ending in `.lan`. + +### Manual Configuration + +You can manually enable resilient DNS for any endpoint by setting environment variables: + +#### For Proxmox VE Endpoints + +```bash +# Primary endpoint +PROXMOX_RESILIENT_DNS=true + +# Additional endpoints +PROXMOX_RESILIENT_DNS_1=true +PROXMOX_RESILIENT_DNS_2=true +``` + +#### For PBS Endpoints + +```bash +# Primary PBS +PBS_RESILIENT_DNS=true + +# Additional PBS instances +PBS_RESILIENT_DNS_1=true +PBS_RESILIENT_DNS_2=true +``` + +## Example Configuration + +```bash +# Example with round-robin DNS +PROXMOX_HOST=proxmox.lan +PROXMOX_PORT=8006 +PROXMOX_TOKEN_ID=user@pam!token +PROXMOX_TOKEN_SECRET=secret-uuid +PROXMOX_RESILIENT_DNS=true # Optional for .lan domains + +# Example with custom domain +PROXMOX_HOST_1=cluster.example.com +PROXMOX_PORT_1=8006 +PROXMOX_TOKEN_ID_1=user@pam!token +PROXMOX_TOKEN_SECRET_1=secret-uuid +PROXMOX_RESILIENT_DNS_1=true # Required for non-.lan domains +``` + +## Troubleshooting + +### Testing DNS Resolution + +You can test DNS resolution for your hostname using the included test script: + +```bash +node scripts/test-dns-resolver.js proxmox.lan +``` + +This will show: +- All resolved IP addresses +- Cache behavior +- Failed IP handling +- Hostname extraction from URLs + +### Log Messages + +When resilient DNS is enabled, you'll see messages like: + +``` +[ApiClients] Creating resilient client for hostname: proxmox.lan +[DnsResolver] Resolved proxmox.lan to: 192.168.1.10, 192.168.1.11, 192.168.1.12 +[ResilientApiClient] Attempting request to proxmox.lan via IP 192.168.1.10 +[ResilientApiClient] Request failed for proxmox.lan via IP 192.168.1.10: ECONNREFUSED +[DnsResolver] Marking host as failed: 192.168.1.10 +[ResilientApiClient] Attempting request to proxmox.lan via IP 192.168.1.11 +``` + +### Common Issues + +1. **"DNS resolution failed" errors** + - Verify the hostname is correct + - Check your DNS server is responding + - Try using `nslookup` or `dig` to test DNS resolution + +2. **All IPs marked as failed** + - Check if the Proxmox/PBS service is running + - Verify firewall rules allow access + - Wait 30 seconds for IPs to be retried + +3. **SSL certificate errors with IP addresses** + - The resilient client adds the proper `Host` header + - Ensure your Proxmox certificates include the hostname + - Consider using `PROXMOX_ALLOW_SELF_SIGNED_CERTS=true` for testing + +## Performance Considerations + +- DNS results are cached for 1 minute +- Failed IPs are skipped for 30 seconds +- Each IP attempt has its own timeout +- The last working IP is tried first on subsequent requests + +## Disabling Resilient DNS + +If you need to disable resilient DNS for a `.lan` domain: + +```bash +PROXMOX_HOST=proxmox.lan +PROXMOX_RESILIENT_DNS=false # Explicitly disable +``` + +## Security Notes + +- The resilient DNS client maintains proper SSL/TLS validation +- The original hostname is used for certificate validation via the `Host` header +- Self-signed certificate settings are preserved \ No newline at end of file diff --git a/scripts/test-dns-resolver.js b/scripts/test-dns-resolver.js new file mode 100755 index 000000000..9d364c8a1 --- /dev/null +++ b/scripts/test-dns-resolver.js @@ -0,0 +1,84 @@ +#!/usr/bin/env node + +/** + * Test script for the resilient DNS resolver + * Usage: node test-dns-resolver.js + */ + +const dnsResolver = require('../server/dnsResolver'); + +async function testDnsResolution(hostname) { + console.log(`\n=== Testing DNS Resolution for: ${hostname} ===\n`); + + try { + // Test basic resolution + console.log('1. Testing basic DNS resolution...'); + const addresses = await dnsResolver.resolveHostname(hostname); + console.log(` ✓ Resolved to ${addresses.length} addresses:`); + addresses.forEach((addr, idx) => { + console.log(` ${idx + 1}. ${addr}`); + }); + + // Test cache + console.log('\n2. Testing cached resolution...'); + const cachedAddresses = await dnsResolver.resolveHostname(hostname); + console.log(` ✓ Got ${cachedAddresses.length} addresses from cache`); + + // Test marking IPs as failed + if (addresses.length > 1) { + console.log('\n3. Testing failed IP handling...'); + const firstIp = addresses[0]; + dnsResolver.markHostFailed(firstIp); + console.log(` - Marked ${firstIp} as failed`); + + const filteredAddresses = await dnsResolver.resolveHostname(hostname); + console.log(` ✓ After filtering: ${filteredAddresses.length} working addresses`); + + // Wait for retry delay + console.log('\n4. Testing retry delay...'); + console.log(` - Waiting for failed IP to be retryable...`); + + const isStillFailed = dnsResolver.isHostFailed(firstIp); + console.log(` - IP ${firstIp} is ${isStillFailed ? 'still marked as failed' : 'available again'}`); + } + + // Test hostname extraction + console.log('\n5. Testing hostname extraction...'); + const testUrls = [ + `https://${hostname}:8006`, + `${hostname}:8006`, + `https://${hostname}/api2/json`, + hostname + ]; + + testUrls.forEach(url => { + const extracted = dnsResolver.extractHostname(url); + console.log(` - "${url}" -> "${extracted}"`); + }); + + // Test canResolve + console.log('\n6. Testing canResolve...'); + const canResolve = await dnsResolver.canResolve(hostname); + console.log(` ✓ Can resolve ${hostname}: ${canResolve}`); + + console.log('\n=== Test completed successfully ===\n'); + + } catch (error) { + console.error(`\n✗ DNS resolution failed: ${error.message}\n`); + process.exit(1); + } +} + +// Main execution +const hostname = process.argv[2]; + +if (!hostname) { + console.error('Usage: node test-dns-resolver.js '); + console.error('Example: node test-dns-resolver.js proxmox.lan'); + process.exit(1); +} + +testDnsResolution(hostname).catch(error => { + console.error('Unexpected error:', error); + process.exit(1); +}); \ No newline at end of file diff --git a/server/alertManager.js b/server/alertManager.js index 458c461be..eeba8c043 100644 --- a/server/alertManager.js +++ b/server/alertManager.js @@ -54,6 +54,33 @@ class AlertManager extends EventEmitter { this.escalationInterval = setInterval(() => { this.checkEscalations(); }, 60000); // Every minute + + // Watch alert rules file for changes + this.setupAlertRulesWatcher(); + } + + setupAlertRulesWatcher() { + const fs = require('fs'); + const path = require('path'); + + try { + console.log('[AlertManager] Setting up alert rules file watcher...'); + + fs.watchFile(this.alertRulesFile, { interval: 1000 }, async (curr, prev) => { + if (curr.mtime !== prev.mtime) { + console.log('[AlertManager] Alert rules file changed, reloading...'); + await this.loadAlertRules(); + console.log('[AlertManager] Alert rules reloaded successfully'); + + // Immediately evaluate current state with new rules + this.evaluateCurrentState(); + } + }); + + console.log('[AlertManager] Alert rules file watcher active'); + } catch (error) { + console.error('[AlertManager] Failed to setup alert rules watcher:', error); + } } initializeDefaultRules() { @@ -638,7 +665,9 @@ class AlertManager extends EventEmitter { getActiveAlerts(filters = {}) { const active = []; for (const alert of this.activeAlerts.values()) { - if (alert.state === 'active' && this.matchesFilters(alert, filters)) { + // Include both 'active' alerts and 'resolved' alerts that have autoResolve=false + if ((alert.state === 'active' || (alert.state === 'resolved' && !alert.rule.autoResolve)) + && this.matchesFilters(alert, filters)) { active.push(this.formatAlertForAPI(alert)); } } @@ -674,7 +703,7 @@ class AlertManager extends EventEmitter { type: alert.guest.type, endpointId: alert.guest.endpointId }, - metric: alert.rule.metric, + metric: alert.rule.metric || (alert.rule.type === 'compound_threshold' ? 'compound' : null), threshold: alert.effectiveThreshold || alert.rule.threshold, currentValue: alert.currentValue, triggeredAt: alert.triggeredAt, @@ -683,7 +712,9 @@ class AlertManager extends EventEmitter { acknowledgedBy: alert.acknowledgedBy, acknowledgedAt: alert.acknowledgedAt, escalated: alert.escalated || false, - message: this.generateAlertMessage(alert) + message: this.generateAlertMessage(alert), + type: alert.rule.type || 'single_metric', + thresholds: alert.rule.thresholds || null }; } @@ -700,7 +731,7 @@ class AlertManager extends EventEmitter { ); const activeCount = Array.from(this.activeAlerts.values()) - .filter(a => a.state === 'active').length; + .filter(a => a.state === 'active' || (a.state === 'resolved' && !a.rule.autoResolve)).length; const acknowledgedCount = Array.from(this.activeAlerts.values()) .filter(a => a.acknowledged).length; @@ -818,6 +849,21 @@ class AlertManager extends EventEmitter { const { guest, rule, currentValue } = alert; let valueStr = ''; + // Handle compound threshold rules + if (rule.type === 'compound_threshold' && rule.thresholds) { + // For compound rules, show all threshold values + const conditions = rule.thresholds.map(threshold => { + const value = currentValue && typeof currentValue === 'object' ? currentValue[threshold.type] : null; + const displayName = this.getThresholdDisplayName(threshold.type); + const unit = ['cpu', 'memory', 'disk'].includes(threshold.type) ? '%' : ' bytes/s'; + const formattedValue = typeof value === 'number' ? Math.round(value * 10) / 10 : value; + return `${displayName}: ${formattedValue}${unit}`; + }).join(', '); + + return `${rule.name} - ${guest.name} (${guest.type.toUpperCase()} ${guest.vmid}) on ${guest.node} - ${conditions}`; + } + + // Handle single-metric rules if (rule.metric === 'status') { valueStr = `Status: ${currentValue}`; } else if (rule.condition === 'anomaly') { @@ -1033,14 +1079,11 @@ class AlertManager extends EventEmitter { */ evaluateCurrentState() { try { - console.log('[AlertManager] evaluateCurrentState() called'); - // Get current state from state manager const stateManager = require('./state'); const currentState = stateManager.getState(); if (!currentState) { - console.log('[AlertManager] No current state available for evaluation'); return; } @@ -1049,36 +1092,30 @@ class AlertManager extends EventEmitter { const currentMetrics = currentState.metrics || []; if (allGuests.length === 0) { - console.log('[AlertManager] No guests found in current state'); return; } - console.log(`[AlertManager] Evaluating current state: ${allGuests.length} guests against ${this.alertRules.size} rules`); - - // Log some sample guests for debugging - const sampleGuests = allGuests.slice(0, 3).map(g => `${g.name}(${g.status})`).join(', '); - console.log(`[AlertManager] Sample guests: ${sampleGuests}`); - - // Check which rules are enabled for down alerts - const downRules = Array.from(this.alertRules.values()).filter(r => r.metric === 'status'); - console.log(`[AlertManager] Found ${downRules.length} down alert rules enabled`); - // For immediate evaluation, we need to check existing conditions and create alerts immediately // This bypasses the normal duration-based pending state const timestamp = Date.now(); allGuests.forEach(guest => { this.alertRules.forEach(rule => { - if (this.isRuleSuppressed(rule.id, guest)) return; + if (this.isRuleSuppressed(rule.id, guest)) { + return; + } const alertKey = `${rule.id}_${guest.endpointId}_${guest.node}_${guest.vmid}`; const existingAlert = this.activeAlerts.get(alertKey); // Skip if alert already exists - if (existingAlert) return; + if (existingAlert) { + return; + } - // For immediate evaluation, we only care about status-based rules (down alerts) + // Handle different rule types if (rule.metric === 'status') { + // Handle status-based rules (down alerts) const effectiveThreshold = this.getEffectiveThreshold(rule, guest); const isTriggered = this.evaluateCondition(guest.status, rule.condition, effectiveThreshold); @@ -1101,6 +1138,11 @@ class AlertManager extends EventEmitter { this.activeAlerts.set(alertKey, newAlert); this.triggerAlert(newAlert); } + } else if (rule.type === 'compound_threshold' && rule.thresholds) { + // Handle compound threshold rules + // We need current metrics for compound threshold evaluation + // Use the regular compound threshold evaluation but bypass duration for immediate evaluation + this.evaluateCompoundThresholdRuleImmediate(rule, guest, alertKey, timestamp); } }); }); @@ -1111,8 +1153,59 @@ class AlertManager extends EventEmitter { } } + evaluateCompoundThresholdRuleImmediate(rule, guest, alertKey, timestamp) { + // Get current metrics from state manager + const stateManager = require('./state'); + const currentState = stateManager.getState(); + + const metrics = currentState.metrics || []; + + // Find metrics for this guest (metrics is an array, not an object) + const guestMetrics = metrics.find(m => + m.endpointId === guest.endpointId && + m.node === guest.node && + m.id === guest.vmid + ); + + if (!guestMetrics || !guestMetrics.current) { + return; + } + + // Check if ALL threshold conditions are met (AND logic) + const thresholdsMet = rule.thresholds.every(threshold => { + return this.evaluateThresholdCondition(threshold, guestMetrics.current, guest); + }); + + if (thresholdsMet) { + // Create alert immediately without waiting for duration + const newAlert = { + id: this.generateAlertId(), + ruleId: rule.id, + rule: rule, + guest: guest, + severity: rule.severity, + message: this.formatCompoundThresholdMessage(rule, guestMetrics.current, guest), + startTime: timestamp, + lastUpdate: timestamp, + triggeredAt: timestamp, // Set immediately for instant alerts + currentValue: this.getCurrentThresholdValues(rule.thresholds, guestMetrics.current, guest), + state: 'active', // Make it active immediately + escalated: false, + acknowledged: false + }; + + this.activeAlerts.set(alertKey, newAlert); + this.triggerAlert(newAlert); + } + } + evaluateCompoundThresholdRule(rule, guest, metrics, alertKey, timestamp) { - const guestMetrics = metrics[guest.vmid] || metrics[guest.name]; + // Find metrics for this guest (metrics is an array, not an object) + const guestMetrics = metrics.find(m => + m.endpointId === guest.endpointId && + m.node === guest.node && + m.id === guest.vmid + ); if (!guestMetrics || !guestMetrics.current) return; // Check if ALL threshold conditions are met (AND logic) @@ -1124,24 +1217,48 @@ class AlertManager extends EventEmitter { if (thresholdsMet) { if (!existingAlert) { - // Create new alert - const alert = { - id: alertKey, - ruleId: rule.id, - rule: rule, - guest: guest, - severity: rule.severity, - message: this.formatCompoundThresholdMessage(rule, guestMetrics.current, guest), - timestamp: timestamp, - state: 'firing', - values: this.getCurrentThresholdValues(rule.thresholds, guestMetrics.current, guest) + // Create new alert with permanent ID + const newAlert = { + id: this.generateAlertId(), // Generate ID once when alert is created + rule, + guest, + startTime: timestamp, + lastUpdate: timestamp, + currentValue: this.getCurrentThresholdValues(rule.thresholds, guestMetrics.current, guest), + effectiveThreshold: rule.thresholds, + state: 'pending', + escalated: false, + acknowledged: false }; - - this.fireAlert(alert); + this.activeAlerts.set(alertKey, newAlert); + } else if (existingAlert.state === 'pending') { + // Check if duration threshold is met + const duration = timestamp - existingAlert.startTime; + if (duration >= rule.duration) { + // Trigger alert + existingAlert.state = 'active'; + existingAlert.triggeredAt = timestamp; + this.triggerAlert(existingAlert); + } + existingAlert.lastUpdate = timestamp; + existingAlert.currentValue = this.getCurrentThresholdValues(rule.thresholds, guestMetrics.current, guest); + } else if (existingAlert.state === 'active') { + // Update existing active alert + existingAlert.lastUpdate = timestamp; + existingAlert.currentValue = this.getCurrentThresholdValues(rule.thresholds, guestMetrics.current, guest); + } + } else { + if (existingAlert && existingAlert.state === 'active') { + // Resolve alert + existingAlert.state = 'resolved'; + existingAlert.resolvedAt = timestamp; + if (existingAlert.rule.autoResolve) { + this.resolveAlert(existingAlert); + } + } else if (existingAlert && existingAlert.state === 'pending') { + // Remove pending alert that didn't trigger + this.activeAlerts.delete(alertKey); } - } else if (existingAlert) { - // Resolve existing alert - this.resolveAlert(alertKey, timestamp, 'Thresholds no longer exceeded'); } } @@ -1151,6 +1268,11 @@ class AlertManager extends EventEmitter { switch (threshold.type) { case 'cpu': metricValue = currentMetrics.cpu; + // CPU values from Proxmox might be in decimal format (0.0-1.0) + // Convert to percentage if needed + if (metricValue !== undefined && metricValue !== null && metricValue <= 1.0) { + metricValue = metricValue * 100; + } break; case 'memory': metricValue = currentMetrics.memory; @@ -1178,7 +1300,25 @@ class AlertManager extends EventEmitter { return false; } - return metricValue >= threshold.value; + // Apply the specified operator + switch (threshold.operator) { + case '>': + return metricValue > threshold.value; + case '>=': + return metricValue >= threshold.value; + case '<': + return metricValue < threshold.value; + case '<=': + return metricValue <= threshold.value; + case '==': + case '=': + return metricValue == threshold.value; + case '!=': + return metricValue != threshold.value; + default: + // Default to >= for backward compatibility + return metricValue >= threshold.value; + } } formatCompoundThresholdMessage(rule, currentMetrics, guest) { @@ -1203,7 +1343,14 @@ class AlertManager extends EventEmitter { getThresholdCurrentValue(threshold, currentMetrics) { switch (threshold.type) { - case 'cpu': return currentMetrics.cpu || 0; + case 'cpu': + const cpuValue = currentMetrics.cpu || 0; + // CPU values from Proxmox might be in decimal format (0.0-1.0) + // Convert to percentage if needed + if (cpuValue <= 1.0) { + return Math.round(cpuValue * 100 * 10) / 10; // Round to 1 decimal place + } + return Math.round(cpuValue * 10) / 10; case 'memory': return currentMetrics.memory || 0; case 'disk': return currentMetrics.disk || 0; case 'diskread': return currentMetrics.diskread || 0; @@ -1382,6 +1529,15 @@ class AlertManager extends EventEmitter { const data = await fs.readFile(this.alertRulesFile, 'utf-8'); const savedRules = JSON.parse(data); + // First, remove all existing compound threshold and custom rules + const rulesToRemove = []; + for (const [key, rule] of this.alertRules) { + if (rule.type === 'compound_threshold' || rule.group === 'custom') { + rulesToRemove.push(key); + } + } + rulesToRemove.forEach(key => this.alertRules.delete(key)); + // Load saved alert rules into the map for (const [key, rule] of Object.entries(savedRules)) { // Only load non-default rules (compound threshold rules and custom rules) @@ -1391,6 +1547,14 @@ class AlertManager extends EventEmitter { } console.log(`[AlertManager] Loaded ${Object.keys(savedRules).length} persisted alert rules`); + + // Clear any active alerts for rules that no longer exist or have been modified + for (const [alertKey, alert] of this.activeAlerts) { + const ruleStillExists = this.alertRules.has(alert.rule.id); + if (!ruleStillExists || this.alertRules.get(alert.rule.id) !== alert.rule) { + this.activeAlerts.delete(alertKey); + } + } } catch (error) { if (error.code !== 'ENOENT') { console.error('[AlertManager] Error loading alert rules:', error); @@ -1687,7 +1851,9 @@ This alert was generated by Pulse monitoring system. }, { title: 'Metric', - value: `${alert.rule.metric.toUpperCase()}: ${valueDisplay} (threshold: ${thresholdDisplay})`, + value: alert.rule.type === 'compound_threshold' ? + `Compound Rule: ${valueDisplay}` : + `${alert.rule.metric.toUpperCase()}: ${valueDisplay} (threshold: ${thresholdDisplay})`, short: false } ], @@ -1738,7 +1904,9 @@ This alert was generated by Pulse monitoring system. }, { name: 'Metric', - value: `${alert.rule.metric.toUpperCase()}: ${valueDisplay} (threshold: ${thresholdDisplay})`, + value: alert.rule.type === 'compound_threshold' ? + `Compound Rule: ${valueDisplay}` : + `${alert.rule.metric.toUpperCase()}: ${valueDisplay} (threshold: ${thresholdDisplay})`, inline: true } ], @@ -1759,7 +1927,9 @@ This alert was generated by Pulse monitoring system. }, { title: 'Metric', - value: `${alert.rule.metric.toUpperCase()}: ${valueDisplay} (threshold: ${thresholdDisplay})`, + value: alert.rule.type === 'compound_threshold' ? + `Compound Rule: ${valueDisplay}` : + `${alert.rule.metric.toUpperCase()}: ${valueDisplay} (threshold: ${thresholdDisplay})`, short: false } ], @@ -1802,6 +1972,11 @@ This alert was generated by Pulse monitoring system. if (this.escalationInterval) { clearInterval(this.escalationInterval); } + + // Stop watching alert rules file + const fs = require('fs'); + fs.unwatchFile(this.alertRulesFile); + this.removeAllListeners(); this.activeAlerts.clear(); this.alertRules.clear(); diff --git a/server/apiClients.js b/server/apiClients.js index df39a31d0..61c794d98 100644 --- a/server/apiClients.js +++ b/server/apiClients.js @@ -1,6 +1,8 @@ const axios = require('axios'); const https = require('https'); const axiosRetry = require('axios-retry').default; +const ResilientApiClient = require('./resilientApiClient'); +const dnsResolver = require('./dnsResolver'); /** * Creates a request interceptor for PVE API authentication. @@ -69,18 +71,21 @@ function initializePveClients(endpoints) { retryConditionChecker: pveRetryConditionChecker, }; - const apiClient = createApiClientInstance(baseURL, endpoint.allowSelfSignedCerts, authInterceptor, retryConfig); + // Enable resilient DNS handling if specified in endpoint config or if host ends with .lan + const useResilientDns = endpoint.useResilientDns || endpoint.host.includes('.lan'); + + const apiClient = createApiClientInstance(baseURL, endpoint.allowSelfSignedCerts, authInterceptor, retryConfig, useResilientDns); apiClients[endpoint.id] = { client: apiClient, config: endpoint }; - console.log(`INFO: Initialized PVE API client for endpoint: ${endpoint.name} (${endpoint.host})`); + console.log(`INFO: Initialized PVE API client for endpoint: ${endpoint.name} (${endpoint.host})${useResilientDns ? ' with resilient DNS' : ''}`); }); return apiClients; } // Generic function to create an Axios API client instance -function createApiClientInstance(baseURL, allowSelfSignedCerts, authInterceptor, retryConfig) { - const apiClient = axios.create({ +function createApiClientInstance(baseURL, allowSelfSignedCerts, authInterceptor, retryConfig, useResilientClient = false) { + const baseConfig = { baseURL: baseURL, timeout: 30000, // 30 second timeout httpsAgent: new https.Agent({ @@ -89,7 +94,20 @@ function createApiClientInstance(baseURL, allowSelfSignedCerts, authInterceptor, headers: { 'Content-Type': 'application/json' } - }); + }; + + // If resilient client is requested and the host appears to be a hostname (not IP) + if (useResilientClient) { + const hostname = dnsResolver.extractHostname(baseURL); + // Check if it's likely a hostname (not an IP) + if (hostname && !hostname.match(/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/)) { + console.log(`[ApiClients] Creating resilient client for hostname: ${hostname}`); + return new ResilientApiClient(baseConfig, authInterceptor); + } + } + + // Standard axios client + const apiClient = axios.create(baseConfig); if (authInterceptor) { apiClient.interceptors.request.use(authInterceptor); @@ -166,10 +184,13 @@ async function initializePbsClients(pbsConfigs) { retryConditionChecker: pbsRetryConditionChecker, }; - const pbsAxiosInstance = createApiClientInstance(pbsBaseURL, config.allowSelfSignedCerts, authInterceptor, retryConfig); + // Enable resilient DNS handling if specified in config or if host ends with .lan + const useResilientDns = config.useResilientDns || config.host.includes('.lan'); + + const pbsAxiosInstance = createApiClientInstance(pbsBaseURL, config.allowSelfSignedCerts, authInterceptor, retryConfig, useResilientDns); clientData = { client: pbsAxiosInstance, config: config }; - console.log(`INFO: [PBS Init] Successfully initialized client for instance '${config.name}' (Token Auth)`); + console.log(`INFO: [PBS Init] Successfully initialized client for instance '${config.name}' (Token Auth)${useResilientDns ? ' with resilient DNS' : ''}`); } else { console.error(`ERROR: Unexpected authMethod '${config.authMethod}' found during PBS client initialization for: ${config.name}`); } diff --git a/server/configLoader.js b/server/configLoader.js index 2a6dca116..e7a0b78f9 100644 --- a/server/configLoader.js +++ b/server/configLoader.js @@ -73,6 +73,7 @@ function loadPbsConfig(index = null) { const nodeNameVar = `PBS_NODE_NAME${suffix}`; const portVar = `PBS_PORT${suffix}`; const selfSignedVar = `PBS_ALLOW_SELF_SIGNED_CERTS${suffix}`; + const resilientDnsVar = `PBS_RESILIENT_DNS${suffix}`; const pbsHostUrl = process.env[hostVar]; if (!pbsHostUrl) { @@ -110,7 +111,8 @@ function loadPbsConfig(index = null) { tokenSecret: pbsTokenSecret, nodeName: process.env[nodeNameVar], // Keep nodeName field allowSelfSignedCerts: process.env[selfSignedVar] !== 'false', - enabled: true + enabled: true, + useResilientDns: process.env[resilientDnsVar] === 'true' }; console.log(`INFO: Found PBS configuration ${index || 'primary'} with ID: ${config.id}, name: ${config.name}, host: ${config.host}`); } @@ -209,6 +211,10 @@ function loadConfiguration() { return null; } + // Check for resilient DNS configuration + const resilientDnsEnv = index ? `PROXMOX_RESILIENT_DNS_${index}` : 'PROXMOX_RESILIENT_DNS'; + const useResilientDns = process.env[resilientDnsEnv] === 'true'; + return { id: index ? `${idPrefix}_${index}` : idPrefix, name: nodeName || null, // Only use explicitly configured names @@ -218,6 +224,7 @@ function loadConfiguration() { tokenSecret: tokenSecret, enabled: process.env[enabledEnv] !== 'false', allowSelfSignedCerts: process.env[selfSignedEnv] !== 'false', + useResilientDns: useResilientDns, }; } diff --git a/server/dataFetcher.js b/server/dataFetcher.js index 122c70acf..69b07515b 100644 --- a/server/dataFetcher.js +++ b/server/dataFetcher.js @@ -2,6 +2,7 @@ const { processPbsTasks } = require('./pbsUtils'); // Assuming pbsUtils.js exist const { createApiClientInstance } = require('./apiClients'); const axios = require('axios'); const https = require('https'); +const dnsResolver = require('./dnsResolver'); let pLimit; let requestLimiter; @@ -253,28 +254,47 @@ async function fetchDataForPveEndpoint(endpointId, apiClientInstance, config) { return { nodes: [], vms: [], containers: [] }; } - // Get node IP addresses from cluster status (reuse the result we already have) + // Get node IP addresses and online status from cluster status (reuse the result we already have) const nodeIpMap = new Map(); + const nodeStatusMap = new Map(); // Track online/offline status from cluster if (clusterStatusResult.status === 'fulfilled' && clusterStatusResult.value.data?.data) { const clusterStatus = clusterStatusResult.value.data.data; clusterStatus.forEach(item => { - if (item.type === 'node' && item.ip) { - nodeIpMap.set(item.name, item.ip); + if (item.type === 'node') { + if (item.ip) { + nodeIpMap.set(item.name, item.ip); + } + // Check if node is online (1) or offline (0) + nodeStatusMap.set(item.name, item.online === 1 ? 'online' : 'offline'); } }); if (nodeIpMap.size > 0) { console.log(`[DataFetcher - ${endpointName}] Found IP addresses for ${nodeIpMap.size} nodes`); } + + // Log offline nodes + const offlineNodes = Array.from(nodeStatusMap.entries()) + .filter(([_, status]) => status === 'offline') + .map(([name, _]) => name); + if (offlineNodes.length > 0) { + console.log(`[DataFetcher - ${endpointName}] Detected offline nodes: ${offlineNodes.join(', ')}`); + } } else { console.warn(`[DataFetcher - ${endpointName}] Could not get cluster status for node IPs`); } // Pass the correct endpointId to fetchDataForNode with concurrency limiting - const guestPromises = nodes.map(node => - requestLimiter(() => fetchDataForNode(apiClientInstance, endpointId, node.node)) - ); + // Skip fetching data for offline nodes to prevent timeouts + const guestPromises = nodes.map(node => { + const isOffline = nodeStatusMap.get(node.node) === 'offline'; + if (isOffline) { + console.log(`[DataFetcher - ${endpointName}] Skipping data fetch for offline node: ${node.node}`); + return Promise.resolve({ skipped: true, reason: 'offline' }); + } + return requestLimiter(() => fetchDataForNode(apiClientInstance, endpointId, node.node)); + }); const guestResults = await Promise.allSettled(guestPromises); let endpointVms = []; @@ -295,6 +315,10 @@ async function fetchDataForPveEndpoint(endpointId, apiClientInstance, config) { nodeDisplayName = `${config.name} - ${correspondingNodeInfo.node}`; } + // Check cluster status first for offline nodes + const clusterNodeStatus = nodeStatusMap.get(correspondingNodeInfo.node); + const isNodeOffline = clusterNodeStatus === 'offline'; + const finalNode = { cpu: null, mem: null, disk: null, maxdisk: null, uptime: 0, loadavg: null, storage: [], node: correspondingNodeInfo.node, @@ -302,7 +326,7 @@ async function fetchDataForPveEndpoint(endpointId, apiClientInstance, config) { maxcpu: correspondingNodeInfo.maxcpu, maxmem: correspondingNodeInfo.maxmem, level: correspondingNodeInfo.level, - status: correspondingNodeInfo.status || 'unknown', + status: isNodeOffline ? 'offline' : (correspondingNodeInfo.status || 'unknown'), id: `${endpointId}-${correspondingNodeInfo.node}`, // Use endpointId for node ID endpointId: endpointId, // Use endpointId for tagging node clusterIdentifier: actualClusterName, // Use actual cluster name or endpoint name @@ -310,7 +334,7 @@ async function fetchDataForPveEndpoint(endpointId, apiClientInstance, config) { ip: nodeIpMap.get(correspondingNodeInfo.node) || null, // Add IP address for direct connections }; - if (result.status === 'fulfilled' && result.value) { + if (result.status === 'fulfilled' && result.value && !result.value.skipped) { const nodeData = result.value; // Use endpointId (the actual key) for constructing IDs and tagging endpointVms.push(...(nodeData.vms || []).map(vm => ({ @@ -343,10 +367,12 @@ async function fetchDataForPveEndpoint(endpointId, apiClientInstance, config) { } else { if (result.status === 'rejected') { console.error(`[DataFetcher - ${endpointName}-${correspondingNodeInfo.node}] Error fetching Node status: ${result.reason?.message || result.reason}`); + } else if (result.value?.skipped && result.value.reason === 'offline') { + console.log(`[DataFetcher - ${endpointName}-${correspondingNodeInfo.node}] Node is offline, showing with offline status`); } else { // console.warn(`[DataFetcher - ${endpointName}-${correspondingNodeInfo.node}] Unexpected result status: ${result.status}`); } - processedNodes.push(finalNode); // Push node with defaults on failure + processedNodes.push(finalNode); // Push node with defaults on failure or offline } }); @@ -1393,6 +1419,21 @@ async function fetchPbsData(currentPbsApiClients) { }; try { + // Quick connectivity check for PBS to fail fast + try { + await Promise.race([ + pbsClient.client.get('/api2/json/version'), + new Promise((_, reject) => + setTimeout(() => reject(new Error('PBS connectivity check timeout')), 3000) + ) + ]); + } catch (connectError) { + console.warn(`[DataFetcher] PBS instance ${instanceName} appears to be offline: ${connectError.message}`); + instanceData.status = 'offline'; + instanceData.error = `PBS server unreachable: ${connectError.message}`; + return instanceData; + } + const nodeName = pbsClient.config.nodeName || await fetchPbsNodeName(pbsClient); // console.log(`[DataFetcher - PBS Data Debug - ${instanceName}] Fetched nodeName: '${nodeName}' (Configured: '${pbsClient.config.nodeName}')`); // REMOVED DEBUG LOG @@ -1473,6 +1514,12 @@ async function fetchPveBackupData(currentApiClients, nodes, vms, containers) { const endpointId = node.endpointId; const nodeName = node.node; + // Skip offline nodes to prevent timeouts + if (node.status === 'offline') { + console.log(`[DataFetcher] Skipping backup fetch for offline node: ${nodeName}`); + return; + } + if (!currentApiClients[endpointId]) { console.warn(`[DataFetcher] No API client found for endpoint: ${endpointId}`); return; diff --git a/server/dnsResolver.js b/server/dnsResolver.js new file mode 100644 index 000000000..47849c368 --- /dev/null +++ b/server/dnsResolver.js @@ -0,0 +1,146 @@ +const dns = require('dns').promises; +const { promisify } = require('util'); +const lookup = promisify(require('dns').lookup); + +// Cache for DNS resolutions +const dnsCache = new Map(); +const DNS_CACHE_TTL = 60000; // 1 minute cache + +/** + * Custom DNS resolver with caching and round-robin support + */ +class DnsResolver { + constructor() { + this.failedHosts = new Map(); // Track failed hosts with timestamps + this.FAILED_HOST_RETRY_DELAY = 30000; // 30 seconds before retrying a failed host + } + + /** + * Clear DNS cache + */ + clearCache() { + dnsCache.clear(); + this.failedHosts.clear(); + } + + /** + * Mark a host/IP as failed + */ + markHostFailed(hostOrIp) { + console.log(`[DnsResolver] Marking host as failed: ${hostOrIp}`); + this.failedHosts.set(hostOrIp, Date.now()); + } + + /** + * Check if a host is marked as failed and still in the retry delay period + */ + isHostFailed(hostOrIp) { + const failedTime = this.failedHosts.get(hostOrIp); + if (!failedTime) return false; + + const elapsed = Date.now() - failedTime; + if (elapsed > this.FAILED_HOST_RETRY_DELAY) { + // Remove from failed list after delay period + this.failedHosts.delete(hostOrIp); + return false; + } + + return true; + } + + /** + * Resolve hostname to IP addresses with caching + * @param {string} hostname - The hostname to resolve + * @returns {Promise} - Array of IP addresses + */ + async resolveHostname(hostname) { + // Check cache first + const cached = dnsCache.get(hostname); + if (cached && (Date.now() - cached.timestamp < DNS_CACHE_TTL)) { + console.log(`[DnsResolver] Using cached DNS resolution for ${hostname}: ${cached.addresses.length} addresses`); + return cached.addresses; + } + + try { + // Try DNS resolution with both IPv4 and IPv6 + const addresses = await dns.resolve4(hostname).catch(() => []); + const addresses6 = await dns.resolve6(hostname).catch(() => []); + + const allAddresses = [...addresses, ...addresses6]; + + if (allAddresses.length === 0) { + // Fallback to lookup (uses system resolver) + console.log(`[DnsResolver] DNS resolve failed for ${hostname}, trying system lookup`); + const result = await lookup(hostname, { all: true }); + const lookupAddresses = result.map(r => r.address); + + if (lookupAddresses.length > 0) { + dnsCache.set(hostname, { addresses: lookupAddresses, timestamp: Date.now() }); + console.log(`[DnsResolver] System lookup resolved ${hostname} to: ${lookupAddresses.join(', ')}`); + return lookupAddresses; + } + + throw new Error(`No IP addresses found for ${hostname}`); + } + + // Cache the results + dnsCache.set(hostname, { addresses: allAddresses, timestamp: Date.now() }); + console.log(`[DnsResolver] Resolved ${hostname} to: ${allAddresses.join(', ')}`); + + // Filter out failed IPs + const workingAddresses = allAddresses.filter(ip => !this.isHostFailed(ip)); + + if (workingAddresses.length === 0) { + console.warn(`[DnsResolver] All ${allAddresses.length} IPs for ${hostname} are marked as failed, using all anyway`); + return allAddresses; + } + + return workingAddresses; + + } catch (error) { + console.error(`[DnsResolver] Failed to resolve ${hostname}: ${error.message}`); + + // Check if we have a stale cache entry we can use + const staleCache = dnsCache.get(hostname); + if (staleCache) { + console.warn(`[DnsResolver] Using stale DNS cache for ${hostname} due to resolution failure`); + return staleCache.addresses; + } + + throw error; + } + } + + /** + * Extract hostname from a URL or host:port string + */ + extractHostname(hostString) { + try { + if (hostString.includes('://')) { + const url = new URL(hostString); + return url.hostname; + } else { + // Handle host:port format + const parts = hostString.split(':'); + return parts[0]; + } + } catch (error) { + return hostString; // Return as-is if parsing fails + } + } + + /** + * Test if a hostname can be resolved + */ + async canResolve(hostname) { + try { + const addresses = await this.resolveHostname(hostname); + return addresses.length > 0; + } catch (error) { + return false; + } + } +} + +// Export singleton instance +module.exports = new DnsResolver(); \ No newline at end of file diff --git a/server/index.js b/server/index.js index d7bd78d85..7cf640161 100644 --- a/server/index.js +++ b/server/index.js @@ -536,7 +536,12 @@ app.post('/api/alerts/rules/reload', async (req, res) => { // Endpoint to trigger immediate alert evaluation app.post('/api/alerts/evaluate', async (req, res) => { try { + console.log('====== API ENDPOINT HIT: /api/alerts/evaluate ======'); console.log('[AlertManager] Triggering immediate alert evaluation...'); + console.log('[DEBUG] stateManager exists:', !!stateManager); + console.log('[DEBUG] stateManager.alertManager exists:', !!stateManager.alertManager); + console.log('[DEBUG] evaluateCurrentState method exists:', !!stateManager.alertManager?.evaluateCurrentState); + stateManager.alertManager.evaluateCurrentState(); res.json({ success: true, message: "Alert evaluation triggered" }); } catch (error) { diff --git a/server/resilientApiClient.js b/server/resilientApiClient.js new file mode 100644 index 000000000..62bf01f4d --- /dev/null +++ b/server/resilientApiClient.js @@ -0,0 +1,164 @@ +const axios = require('axios'); +const https = require('https'); +const dnsResolver = require('./dnsResolver'); + +/** + * Creates a resilient API client that handles DNS failures gracefully + */ +class ResilientApiClient { + constructor(baseConfig, authInterceptor) { + this.baseConfig = baseConfig; + this.authInterceptor = authInterceptor; + this.hostname = this.extractHostname(baseConfig.baseURL); + this.clients = new Map(); // Map of IP -> axios instance + this.lastWorkingIp = null; + } + + extractHostname(url) { + try { + const urlObj = new URL(url); + return urlObj.hostname; + } catch (error) { + // Handle cases where baseURL might not be a full URL + const match = url.match(/^https?:\/\/([^:\/]+)/); + return match ? match[1] : null; + } + } + + /** + * Create an axios instance for a specific IP + */ + createClientForIp(ip) { + // Replace hostname with IP in the base URL + const baseURL = this.baseConfig.baseURL.replace(this.hostname, ip); + + const client = axios.create({ + ...this.baseConfig, + baseURL: baseURL, + httpsAgent: new https.Agent({ + rejectUnauthorized: this.baseConfig.httpsAgent?.options?.rejectUnauthorized ?? true, + // Set a shorter keepAlive timeout for failed connections + keepAlive: true, + keepAliveMsecs: 1000, + timeout: this.baseConfig.timeout || 30000 + }), + headers: { + ...this.baseConfig.headers, + // Add Host header to ensure proper SSL certificate validation + 'Host': this.hostname + } + }); + + // Apply auth interceptor if provided + if (this.authInterceptor) { + client.interceptors.request.use(this.authInterceptor); + } + + // Add response interceptor to track working IPs + client.interceptors.response.use( + (response) => { + // Mark this IP as working + this.lastWorkingIp = ip; + return response; + }, + (error) => { + // Mark IP as failed on certain errors + if (error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT' || error.code === 'EHOSTUNREACH') { + dnsResolver.markHostFailed(ip); + } + return Promise.reject(error); + } + ); + + return client; + } + + /** + * Get or create a client for a specific IP + */ + getClientForIp(ip) { + if (!this.clients.has(ip)) { + this.clients.set(ip, this.createClientForIp(ip)); + } + return this.clients.get(ip); + } + + /** + * Try to make a request with automatic failover + */ + async request(config) { + // First, try to resolve the hostname + let ips; + try { + ips = await dnsResolver.resolveHostname(this.hostname); + } catch (error) { + console.error(`[ResilientApiClient] DNS resolution failed for ${this.hostname}: ${error.message}`); + throw new Error(`DNS resolution failed for ${this.hostname}: ${error.message}`); + } + + if (ips.length === 0) { + throw new Error(`No IP addresses available for ${this.hostname}`); + } + + // If we have a last working IP, try it first + if (this.lastWorkingIp && ips.includes(this.lastWorkingIp)) { + ips = [this.lastWorkingIp, ...ips.filter(ip => ip !== this.lastWorkingIp)]; + } + + let lastError; + + // Try each IP in sequence + for (const ip of ips) { + if (dnsResolver.isHostFailed(ip)) { + console.log(`[ResilientApiClient] Skipping failed IP ${ip} for ${this.hostname}`); + continue; + } + + try { + const client = this.getClientForIp(ip); + console.log(`[ResilientApiClient] Attempting request to ${this.hostname} via IP ${ip}`); + + const response = await client.request(config); + return response; + + } catch (error) { + lastError = error; + console.warn(`[ResilientApiClient] Request failed for ${this.hostname} via IP ${ip}: ${error.message}`); + + // If this is a connection error, mark the IP as failed + if (error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT' || error.code === 'EHOSTUNREACH') { + dnsResolver.markHostFailed(ip); + } + + // Continue to next IP + } + } + + // All IPs failed + console.error(`[ResilientApiClient] All ${ips.length} IPs failed for ${this.hostname}`); + throw lastError || new Error(`All IPs failed for ${this.hostname}`); + } + + // Proxy common axios methods + async get(url, config) { + return this.request({ ...config, method: 'GET', url }); + } + + async post(url, data, config) { + return this.request({ ...config, method: 'POST', url, data }); + } + + async put(url, data, config) { + return this.request({ ...config, method: 'PUT', url, data }); + } + + async delete(url, config) { + return this.request({ ...config, method: 'DELETE', url }); + } + + async patch(url, data, config) { + return this.request({ ...config, method: 'PATCH', url, data }); + } +} + +module.exports = ResilientApiClient; \ No newline at end of file diff --git a/server/tests/dnsResolver.test.js b/server/tests/dnsResolver.test.js new file mode 100644 index 000000000..f31f3c0e1 --- /dev/null +++ b/server/tests/dnsResolver.test.js @@ -0,0 +1,123 @@ +const dnsResolver = require('../dnsResolver'); +const dns = require('dns').promises; + +// Mock the dns module +jest.mock('dns', () => ({ + promises: { + resolve4: jest.fn(), + resolve6: jest.fn() + } +})); + +// Mock the util.promisify +jest.mock('util', () => ({ + promisify: () => jest.fn() +})); + +describe('DnsResolver', () => { + beforeEach(() => { + // Clear all mocks and caches + jest.clearAllMocks(); + dnsResolver.clearCache(); + }); + + describe('resolveHostname', () => { + it('should resolve hostname to IP addresses', async () => { + const mockIPs = ['192.168.1.10', '192.168.1.11', '192.168.1.12']; + dns.resolve4.mockResolvedValue(mockIPs); + dns.resolve6.mockResolvedValue([]); + + const result = await dnsResolver.resolveHostname('proxmox.lan'); + + expect(result).toEqual(mockIPs); + expect(dns.resolve4).toHaveBeenCalledWith('proxmox.lan'); + }); + + it('should cache DNS results', async () => { + const mockIPs = ['192.168.1.10']; + dns.resolve4.mockResolvedValue(mockIPs); + dns.resolve6.mockResolvedValue([]); + + // First call + await dnsResolver.resolveHostname('test.lan'); + expect(dns.resolve4).toHaveBeenCalledTimes(1); + + // Second call should use cache + await dnsResolver.resolveHostname('test.lan'); + expect(dns.resolve4).toHaveBeenCalledTimes(1); // Still only called once + }); + + it('should filter out failed IPs', async () => { + const mockIPs = ['192.168.1.10', '192.168.1.11', '192.168.1.12']; + dns.resolve4.mockResolvedValue(mockIPs); + dns.resolve6.mockResolvedValue([]); + + // Mark one IP as failed + dnsResolver.markHostFailed('192.168.1.11'); + + const result = await dnsResolver.resolveHostname('proxmox.lan'); + + expect(result).toEqual(['192.168.1.10', '192.168.1.12']); + expect(result).not.toContain('192.168.1.11'); + }); + + it('should handle DNS resolution failures gracefully', async () => { + dns.resolve4.mockRejectedValue(new Error('DNS resolution failed')); + dns.resolve6.mockRejectedValue(new Error('DNS resolution failed')); + + // Mock lookup to also fail + const lookup = require('util').promisify(); + lookup.mockRejectedValue(new Error('Lookup failed')); + + await expect(dnsResolver.resolveHostname('invalid.lan')) + .rejects.toThrow('No IP addresses found'); + }); + }); + + describe('markHostFailed and isHostFailed', () => { + it('should mark host as failed temporarily', async () => { + const testIP = '192.168.1.10'; + + expect(dnsResolver.isHostFailed(testIP)).toBe(false); + + dnsResolver.markHostFailed(testIP); + expect(dnsResolver.isHostFailed(testIP)).toBe(true); + }); + }); + + describe('extractHostname', () => { + it('should extract hostname from various URL formats', () => { + const testCases = [ + { input: 'https://proxmox.lan:8006', expected: 'proxmox.lan' }, + { input: 'http://test.local:3000/path', expected: 'test.local' }, + { input: 'server.domain:8080', expected: 'server.domain' }, + { input: 'simple-hostname', expected: 'simple-hostname' } + ]; + + testCases.forEach(({ input, expected }) => { + expect(dnsResolver.extractHostname(input)).toBe(expected); + }); + }); + }); + + describe('canResolve', () => { + it('should return true for resolvable hostnames', async () => { + dns.resolve4.mockResolvedValue(['192.168.1.10']); + dns.resolve6.mockResolvedValue([]); + + const result = await dnsResolver.canResolve('valid.lan'); + expect(result).toBe(true); + }); + + it('should return false for unresolvable hostnames', async () => { + dns.resolve4.mockRejectedValue(new Error('Not found')); + dns.resolve6.mockRejectedValue(new Error('Not found')); + + const lookup = require('util').promisify(); + lookup.mockRejectedValue(new Error('Not found')); + + const result = await dnsResolver.canResolve('invalid.lan'); + expect(result).toBe(false); + }); + }); +}); \ No newline at end of file diff --git a/src/public/js/alertsHandler.js b/src/public/js/alertsHandler.js index 45bb06184..e4424a9a3 100644 --- a/src/public/js/alertsHandler.js +++ b/src/public/js/alertsHandler.js @@ -370,8 +370,19 @@ PulseApp.alerts = (() => { } else if (typeof alert.currentValue === 'number') { const isPercentageMetric = ['cpu', 'memory', 'disk'].includes(alert.metric); currentValueDisplay = `${Math.round(alert.currentValue)}${isPercentageMetric ? '%' : ''}`; + } else if (typeof alert.currentValue === 'object' && alert.currentValue !== null) { + // Handle compound threshold alerts (multiple metrics) + const values = []; + for (const [metric, value] of Object.entries(alert.currentValue)) { + const isPercentageMetric = ['cpu', 'memory', 'disk'].includes(metric); + const formattedValue = typeof value === 'number' + ? `${Math.round(value)}${isPercentageMetric ? '%' : ''}` + : value; + values.push(`${metric}: ${formattedValue}`); + } + currentValueDisplay = values.join(', '); } else { - currentValueDisplay = alert.currentValue; + currentValueDisplay = alert.currentValue || ''; } // Muted text classes for acknowledged alerts diff --git a/src/public/js/ui/alertManagementModal.js b/src/public/js/ui/alertManagementModal.js index 62d1fc318..42165e7af 100644 --- a/src/public/js/ui/alertManagementModal.js +++ b/src/public/js/ui/alertManagementModal.js @@ -41,22 +41,12 @@ PulseApp.ui.alertManagementModal = (() => {
@@ -1055,9 +1045,53 @@ PulseApp.ui.alertManagementModal = (() => { const emailConfigSection = document.getElementById('email-config-section'); if (!emailConfigSection) return; + // Get email configuration from currentConfig + const config = currentConfig || {}; + const smtp = config.advanced?.smtp || {}; + + // Populate email fields with values from configuration + const emailFromInput = document.querySelector('input[name="ALERT_FROM_EMAIL"]'); + if (emailFromInput && smtp.from) { + emailFromInput.value = smtp.from; + } + + const emailToInput = document.querySelector('input[name="ALERT_TO_EMAIL"]'); + if (emailToInput && smtp.to) { + emailToInput.value = smtp.to; + } + + const smtpHostInput = document.querySelector('input[name="ALERT_SMTP_HOST"]'); + if (smtpHostInput && smtp.host) { + smtpHostInput.value = smtp.host; + } + + const smtpPortInput = document.querySelector('input[name="ALERT_SMTP_PORT"]'); + if (smtpPortInput && smtp.port) { + smtpPortInput.value = smtp.port; + } + + const smtpUserInput = document.querySelector('input[name="ALERT_SMTP_USER"]'); + if (smtpUserInput && smtp.user) { + smtpUserInput.value = smtp.user; + } + + const smtpSecureInput = document.querySelector('input[name="ALERT_SMTP_SECURE"]'); + if (smtpSecureInput && smtp.secure !== undefined) { + smtpSecureInput.checked = smtp.secure; + } + + const smtpPassInput = document.querySelector('input[name="ALERT_EMAIL_PASSWORD"]'); + if (smtpPassInput && smtp.pass) { + smtpPassInput.value = smtp.pass; + } + + // Auto-detect email provider if from email is set + if (emailFromInput && emailFromInput.value) { + autoDetectEmailProvider(emailFromInput.value); + } + // Use existing email configuration from settings if available if (PulseApp.ui.settings && PulseApp.ui.settings.renderAlertsTab) { - const config = currentConfig || {}; const fullContent = PulseApp.ui.settings.renderAlertsTab(config.alerts || {}, config); // Extract just the email configuration section @@ -1267,12 +1301,12 @@ PulseApp.ui.alertManagementModal = (() => { } } - function testEmailConnection() { + async function testEmailConnection() { // Collect email configuration const emailConfig = { from: document.querySelector('input[name="ALERT_FROM_EMAIL"]')?.value, to: document.querySelector('input[name="ALERT_TO_EMAIL"]')?.value, - password: document.querySelector('input[name="ALERT_EMAIL_PASSWORD"]')?.value, + pass: document.querySelector('input[name="ALERT_EMAIL_PASSWORD"]')?.value, // Backend expects 'pass' not 'password' host: document.querySelector('input[name="ALERT_SMTP_HOST"]')?.value, port: document.querySelector('input[name="ALERT_SMTP_PORT"]')?.value, user: document.querySelector('input[name="ALERT_SMTP_USER"]')?.value, @@ -1284,23 +1318,33 @@ PulseApp.ui.alertManagementModal = (() => { return; } - // TODO: Implement actual email test - PulseApp.ui.toast.info('Test email functionality will be implemented with backend integration'); + // Send test email via API + try { + const response = await fetch('/api/test-email', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(emailConfig) + }); + + const result = await response.json(); + + if (response.ok && result.success) { + PulseApp.ui.toast.success('Test email sent successfully! Please check your inbox.'); + } else { + PulseApp.ui.toast.error('Test email failed: ' + (result.error || 'Unknown error')); + } + } catch (error) { + console.error('[Email Test] Error:', error); + PulseApp.ui.toast.error('Error sending test email: ' + error.message); + } } - function saveEmailConfiguration() { - // Collect email configuration - const emailConfig = { - from: document.querySelector('input[name="ALERT_FROM_EMAIL"]')?.value, - to: document.querySelector('input[name="ALERT_TO_EMAIL"]')?.value, - password: document.querySelector('input[name="ALERT_EMAIL_PASSWORD"]')?.value, - host: document.querySelector('input[name="ALERT_SMTP_HOST"]')?.value, - port: document.querySelector('input[name="ALERT_SMTP_PORT"]')?.value, - user: document.querySelector('input[name="ALERT_SMTP_USER"]')?.value, - secure: document.querySelector('input[name="ALERT_SMTP_SECURE"]')?.checked - }; - - // TODO: Implement saving to backend + async function saveEmailConfiguration() { + // This function is called by the specific save button in the email section + // It will trigger the main saveConfiguration function + await saveConfiguration(); PulseApp.ui.toast.success('Email configuration saved successfully!'); } @@ -1719,6 +1763,24 @@ PulseApp.ui.alertManagementModal = (() => {
+
+ +
+ + +
+

+ Alert will only trigger if the condition persists for this duration. Prevents false alarms from brief spikes. + Range: 5 seconds to 60 minutes. +

+
+