Files
pulse/server/state.js
T
rcourtman 4452b27655 fix: resolve circular reference errors in alert system
- Fixed undefined guest.type errors in generateAlertMessage by adding proper null checks
- Separated notification status from alert objects to prevent circular references during serialization
- Added comprehensive error handling with fallback data when alert serialization fails
- Improved alert state management to track notification status separately
- Fixed alert display issues in UI by ensuring proper data structure
- Added debugging helpers for troubleshooting serialization issues
- Cleaned up console logging and removed development artifacts

These changes resolve the "Converting circular structure to JSON" errors that were
preventing alerts from being properly displayed and managed in the UI.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-06-09 16:15:56 +01:00

511 lines
15 KiB
JavaScript

const AlertManager = require('./alertManager');
const state = {
nodes: [],
vms: [],
containers: [],
metrics: [],
pbs: [], // Array to hold data for each PBS instance
pveBackups: { // Add PVE backup data
backupTasks: [],
storageBackups: [],
guestSnapshots: []
},
isConfigPlaceholder: false, // Add this flag
endpoints: [], // Add endpoint configurations for client
pbsConfigs: [], // Add PBS configurations for client
// Enhanced monitoring data
performance: {
lastDiscoveryTime: null,
lastMetricsTime: null,
discoveryDuration: 0,
metricsDuration: 0,
errorCount: 0,
successCount: 0,
avgResponseTime: 0,
peakMemoryUsage: 0
},
// Connection health tracking
connections: new Map(), // endpointId -> { status, lastSeen, errorCount, responseTime }
// Runtime statistics
stats: {
totalGuests: 0,
runningGuests: 0,
stoppedGuests: 0,
totalNodes: 0,
healthyNodes: 0,
warningNodes: 0,
errorNodes: 0,
avgCpuUsage: 0,
avgMemoryUsage: 0,
avgDiskUsage: 0,
lastUpdated: null
},
// These were pbsInstances, allPbsTasks, aggregatedPbsTaskSummary.
// state.pbs now holds the primary list of PBS instances.
// Task-related data might be part of each PBS instance object in state.pbs,
// or handled separately if needed by the client for global views.
allPbsTasks: [],
aggregatedPbsTaskSummary: {}
};
// Initialize alert manager
const alertManager = new AlertManager();
// Track performance metrics
let performanceHistory = [];
const MAX_PERFORMANCE_HISTORY = 288; // 24 hours at 5-minute intervals
function init() {
console.log('[State Manager] Initializing enhanced state management...');
// Set up alert event handlers
alertManager.on('alert', (alert) => {
// Could emit to websockets, send notifications, etc.
updateAlertStats();
});
alertManager.on('alertResolved', (alert) => {
updateAlertStats();
});
// Initialize performance tracking
state.performance.lastDiscoveryTime = Date.now();
state.performance.lastMetricsTime = Date.now();
console.log('[State Manager] Enhanced state management initialized');
}
function getState() {
// Return a comprehensive state object
return {
nodes: state.nodes,
vms: state.vms,
containers: state.containers,
metrics: state.metrics, // Assuming metrics are updated elsewhere
pbs: state.pbs, // This is what's sent to the client and should now be correct
pveBackups: state.pveBackups, // Add PVE backup data
isConfigPlaceholder: state.isConfigPlaceholder,
endpoints: state.endpoints, // Add endpoint configurations
pbsConfigs: state.pbsConfigs, // Add PBS configurations
// Enhanced monitoring data
performance: state.performance,
connections: Object.fromEntries(state.connections), // Convert Map to object for serialization
stats: state.stats,
// PBS specific data structures - these might be derived or directly set
// If they are part of each item in state.pbs, the client can aggregate them.
// If they are global summaries, ensure they are updated correctly in updateDiscoveryData.
allPbsTasks: state.allPbsTasks,
aggregatedPbsTaskSummary: state.aggregatedPbsTaskSummary,
// Alerts
alerts: getAlertInfo(), // Corrected: Call the local getAlertInfo function
// Custom Thresholds
customThresholds: getCustomThresholds()
};
}
function updateDiscoveryData({ nodes, vms, containers, pbs, pveBackups, allPbsTasks, aggregatedPbsTaskSummary }, duration = 0, errors = []) {
const startTime = Date.now();
try {
// Update main data
state.nodes = nodes || [];
state.vms = vms || [];
state.containers = containers || [];
state.pbs = pbs || [];
// Update PVE backup data
if (pveBackups) {
state.pveBackups = {
backupTasks: pveBackups.backupTasks || [],
storageBackups: pveBackups.storageBackups || [],
guestSnapshots: pveBackups.guestSnapshots || []
};
}
// If the discovery data structure nests these under the main 'pbs' array (e.g., from fetchPbsData),
// they might not be separate top-level items in the discoveryData object passed here.
// If they are indeed separate, this update is fine.
// If they are part of the main 'pbs' array items, this might be redundant or need adjustment
// based on how fetchDiscoveryData structures the final object.
if (allPbsTasks !== undefined) { // Check for presence before assigning
state.allPbsTasks = allPbsTasks;
}
if (aggregatedPbsTaskSummary !== undefined) { // Check for presence before assigning
state.aggregatedPbsTaskSummary = aggregatedPbsTaskSummary;
}
// Update performance metrics
state.performance.lastDiscoveryTime = startTime;
state.performance.discoveryDuration = duration;
if (errors.length > 0) {
state.performance.errorCount += errors.length;
updateConnectionHealth(errors);
} else {
state.performance.successCount++;
}
// Calculate runtime statistics
calculateRuntimeStats();
// Track performance history
addPerformanceSnapshot('discovery', duration, errors.length);
console.log(`[State Manager] Discovery update completed. Duration: ${duration}ms, Errors: ${errors.length}`);
} catch (error) {
console.error('[State Manager] Error updating discovery data:', error);
state.performance.errorCount++;
}
}
function updateMetricsData(metrics, duration = 0, errors = []) {
const startTime = Date.now();
try {
state.metrics = metrics || [];
// Update performance metrics
state.performance.lastMetricsTime = startTime;
state.performance.metricsDuration = duration;
if (errors.length > 0) {
state.performance.errorCount += errors.length;
updateConnectionHealth(errors);
} else {
state.performance.successCount++;
}
// Calculate average response time
updateAverageResponseTime(duration);
// Check metrics against alert rules
checkAlertsForMetrics();
// Update runtime statistics with latest metrics
calculateRuntimeStats();
// Track performance history
addPerformanceSnapshot('metrics', duration, errors.length);
console.log(`[State Manager] Metrics update completed. Duration: ${duration}ms, Metrics: ${metrics.length}, Errors: ${errors.length}`);
} catch (error) {
console.error('[State Manager] Error updating metrics data:', error);
state.performance.errorCount++;
}
}
function calculateRuntimeStats() {
try {
const allGuests = [...state.vms, ...state.containers];
// Guest statistics
state.stats.totalGuests = allGuests.length;
state.stats.runningGuests = allGuests.filter(g => g.status === 'running').length;
state.stats.stoppedGuests = allGuests.filter(g => g.status === 'stopped').length;
// Node statistics
state.stats.totalNodes = state.nodes.length;
state.stats.healthyNodes = state.nodes.filter(n => n.status === 'online').length;
state.stats.warningNodes = state.nodes.filter(n =>
n.status === 'online' && (
(n.cpu && n.cpu > 80) ||
(n.mem && n.maxmem && (n.mem / n.maxmem) > 0.9)
)
).length;
state.stats.errorNodes = state.nodes.filter(n => n.status !== 'online').length;
// Average usage calculations
const runningGuests = allGuests.filter(g => g.status === 'running');
if (runningGuests.length > 0 && state.metrics.length > 0) {
let totalCpu = 0, totalMemory = 0, totalDisk = 0, count = 0;
state.metrics.forEach(metric => {
if (metric.current) {
const guest = runningGuests.find(g =>
g.vmid === metric.id && g.node === metric.node && g.endpointId === metric.endpointId
);
if (guest) {
if (metric.current.cpu !== undefined) {
totalCpu += metric.current.cpu * 100;
}
if (metric.current.mem !== undefined && guest.maxmem) {
totalMemory += (metric.current.mem / guest.maxmem) * 100;
}
if (metric.current.disk !== undefined && guest.maxdisk) {
totalDisk += (metric.current.disk / guest.maxdisk) * 100;
}
count++;
}
}
});
if (count > 0) {
state.stats.avgCpuUsage = Math.round(totalCpu / count);
state.stats.avgMemoryUsage = Math.round(totalMemory / count);
state.stats.avgDiskUsage = Math.round(totalDisk / count);
}
}
state.stats.lastUpdated = Date.now();
} catch (error) {
console.error('[State Manager] Error calculating runtime stats:', error);
}
}
function updateConnectionHealth(errors) {
errors.forEach(error => {
const endpointId = error.endpointId || 'unknown';
const connection = state.connections.get(endpointId) || {
status: 'healthy',
lastSeen: Date.now(),
errorCount: 0,
responseTime: 0
};
connection.errorCount++;
connection.lastSeen = Date.now();
connection.status = connection.errorCount > 5 ? 'error' : 'warning';
state.connections.set(endpointId, connection);
});
}
function updateAverageResponseTime(newDuration) {
const alpha = 0.1; // Exponential moving average factor
if (state.performance.avgResponseTime === 0) {
state.performance.avgResponseTime = newDuration;
} else {
state.performance.avgResponseTime = (alpha * newDuration) + ((1 - alpha) * state.performance.avgResponseTime);
}
}
function addPerformanceSnapshot(type, duration, errorCount) {
const snapshot = {
timestamp: Date.now(),
type,
duration,
errorCount,
memoryUsage: process.memoryUsage().heapUsed,
guestCount: state.stats.totalGuests,
nodeCount: state.stats.totalNodes
};
performanceHistory.unshift(snapshot);
if (performanceHistory.length > MAX_PERFORMANCE_HISTORY) {
performanceHistory = performanceHistory.slice(0, MAX_PERFORMANCE_HISTORY);
}
// Update peak memory usage
if (snapshot.memoryUsage > state.performance.peakMemoryUsage) {
state.performance.peakMemoryUsage = snapshot.memoryUsage;
}
}
async function checkAlertsForMetrics() {
try {
const allGuests = [...state.vms, ...state.containers];
await alertManager.checkMetrics(allGuests, state.metrics);
} catch (error) {
console.error('[State Manager] Error checking alerts:', error);
}
}
function updateAlertStats() {
// This could update dashboard counters, etc.
const alertStats = alertManager.getEnhancedAlertStats();
state.stats.activeAlerts = alertStats.active;
state.stats.alertsLast24h = alertStats.last24Hours;
}
function getAlertInfo() {
try {
const alertInfo = {
active: alertManager.getActiveAlerts(),
stats: alertManager.getEnhancedAlertStats(),
rules: alertManager.getRules()
};
// Test serialization to catch any circular references early
JSON.stringify(alertInfo);
return alertInfo;
} catch (error) {
console.error('[State Manager] Error serializing alert info:', error.message);
// Try to identify which part is causing the circular reference
try {
JSON.stringify(alertManager.getActiveAlerts());
console.log('[State Manager] Active alerts serialization: OK');
} catch (activeError) {
console.error('[State Manager] Active alerts circular reference detected');
}
try {
JSON.stringify(alertManager.getEnhancedAlertStats());
console.log('[State Manager] Alert stats serialization: OK');
} catch (statsError) {
console.error('[State Manager] Alert stats circular reference detected');
}
try {
JSON.stringify(alertManager.getRules());
console.log('[State Manager] Alert rules serialization: OK');
} catch (rulesError) {
console.error('[State Manager] Alert rules circular reference detected');
}
// Return safe empty alert data if serialization fails
return {
active: [],
stats: {
active: 0,
acknowledged: 0,
escalated: 0,
last24Hours: 0,
lastHour: 0,
totalRules: 0,
suppressedRules: 0,
metrics: { totalFired: 0, totalResolved: 0, totalAcknowledged: 0, averageResolutionTime: 0, falsePositiveRate: 0 },
groups: []
},
rules: []
};
}
}
function getCustomThresholds() {
try {
const customThresholdManager = require('./customThresholds');
return customThresholdManager.getAllThresholds();
} catch (error) {
console.error('[State Manager] Error getting custom thresholds:', error);
return [];
}
}
function clearMetricsData() {
state.metrics = [];
console.log('[State Manager] Metrics data cleared');
}
function hasData() {
return state.nodes.length > 0 || state.vms.length > 0 || state.containers.length > 0 || state.pbs.length > 0;
}
function setConfigPlaceholderStatus(isPlaceholder) {
state.isConfigPlaceholder = isPlaceholder;
}
function setEndpointConfigurations(endpoints, pbsConfigs) {
// Store endpoint configurations for client use
state.endpoints = endpoints.map(endpoint => ({
id: endpoint.id,
name: endpoint.name,
host: endpoint.host,
port: endpoint.port,
enabled: endpoint.enabled
}));
state.pbsConfigs = pbsConfigs.map(config => ({
id: config.id,
name: config.name,
host: config.host,
port: config.port
}));
}
function getPerformanceHistory(limit = 50) {
return performanceHistory.slice(0, limit);
}
function getConnectionHealth() {
const connections = {};
for (const [endpointId, health] of state.connections) {
connections[endpointId] = { ...health };
}
return connections;
}
function getHealthSummary() {
const now = Date.now();
const fiveMinutesAgo = now - (5 * 60 * 1000);
// Recent performance
const recentSnapshots = performanceHistory.filter(s => s.timestamp >= fiveMinutesAgo);
const avgDuration = recentSnapshots.length > 0
? recentSnapshots.reduce((sum, s) => sum + s.duration, 0) / recentSnapshots.length
: 0;
const recentErrors = recentSnapshots.reduce((sum, s) => sum + s.errorCount, 0);
// Connection health
let healthyConnections = 0;
let totalConnections = state.connections.size;
for (const connection of state.connections.values()) {
if (connection.status === 'healthy') healthyConnections++;
}
return {
overall: recentErrors === 0 && healthyConnections === totalConnections ? 'healthy' :
recentErrors < 3 ? 'warning' : 'error',
performance: {
avgResponseTime: Math.round(avgDuration),
recentErrors,
memoryUsage: process.memoryUsage().heapUsed,
uptime: process.uptime()
},
connections: {
healthy: healthyConnections,
total: totalConnections
},
alerts: alertManager.getEnhancedAlertStats(),
lastUpdate: state.stats.lastUpdated
};
}
// Graceful cleanup
function destroy() {
console.log('[State Manager] Cleaning up...');
if (alertManager) {
alertManager.destroy();
}
state.connections.clear();
performanceHistory = [];
}
function getAlertManager() {
return alertManager;
}
module.exports = {
init,
getState,
setConfigPlaceholderStatus,
setEndpointConfigurations,
updateDiscoveryData,
updateMetricsData,
clearMetricsData,
hasData,
// Enhanced monitoring functions
getPerformanceHistory,
getConnectionHealth,
getHealthSummary,
getAlertInfo,
destroy,
// Alert manager access - make non-enumerable to prevent accidental serialization
get alertManager() { return alertManager; },
getAlertManager
};