feat: add dynamic alert rule management with cleanup for disabled types

- Add refreshRules() method to AlertManager for dynamic rule updates
- Add cleanupAlertsForRule() to remove active alerts when rules are disabled
- Modify configuration reload to trigger alert rule refresh
- Add getAlertManager() method to state module for consistent access
- Existing alerts are now properly cleaned up when global alert types are disabled
- Changes take effect immediately without requiring server restart

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
courtmanr@gmail.com
2025-06-01 22:35:30 +01:00
parent 2b1157c8ca
commit b7e93f7fd8
3 changed files with 105 additions and 1 deletions
+85
View File
@@ -908,6 +908,91 @@ class AlertManager extends EventEmitter {
return success;
}
/**
* Refresh alert rules based on current environment variables
* This should be called after configuration changes
*/
refreshRules() {
console.log('[AlertManager] Refreshing alert rules based on current environment variables');
// Store currently disabled rule IDs to clean up their alerts
const previouslyActiveRules = new Set(this.alertRules.keys());
// Clear existing rules
this.alertRules.clear();
// Re-initialize rules with current environment variables
this.initializeDefaultRules();
// Find rules that were disabled
const nowActiveRules = new Set(this.alertRules.keys());
const disabledRules = [...previouslyActiveRules].filter(ruleId => !nowActiveRules.has(ruleId));
// Clean up alerts for disabled rules
disabledRules.forEach(ruleId => {
this.cleanupAlertsForRule(ruleId);
});
console.log(`[AlertManager] Rules refreshed. Active: ${this.alertRules.size}, Disabled: ${disabledRules.length}`);
if (disabledRules.length > 0) {
console.log(`[AlertManager] Cleaned up alerts for disabled rules: ${disabledRules.join(', ')}`);
}
this.emit('rulesRefreshed', { activeRules: nowActiveRules.size, disabledRules });
}
/**
* Clean up active alerts for a specific rule type
*/
cleanupAlertsForRule(ruleId) {
const alertsToRemove = [];
// Find all active alerts for this rule
for (const [alertKey, alert] of this.activeAlerts) {
if (alert.rule.id === ruleId) {
alertsToRemove.push(alertKey);
}
}
// Remove the alerts
alertsToRemove.forEach(alertKey => {
const alert = this.activeAlerts.get(alertKey);
if (alert) {
// Mark as resolved due to rule disable
const resolvedAlert = {
id: alert.id,
ruleId: alert.rule.id,
ruleName: alert.rule.name,
severity: 'resolved',
guest: {
name: alert.guest.name,
vmid: alert.guest.vmid,
node: alert.guest.node,
type: alert.guest.type,
endpointId: alert.guest.endpointId
},
metric: alert.rule.metric,
resolvedAt: Date.now(),
duration: alert.triggeredAt ? Date.now() - alert.triggeredAt : 0,
message: `${alert.rule.name} - Alert cleared due to rule type being disabled`,
resolvedReason: 'rule_disabled'
};
// Add to history
this.addToHistory(resolvedAlert);
// Emit event
this.emit('alertResolved', resolvedAlert);
console.info(`[ALERT CLEARED] ${resolvedAlert.message}`);
}
this.activeAlerts.delete(alertKey);
});
return alertsToRemove.length;
}
getRules(filters = {}) {
const rules = Array.from(this.alertRules.values());
if (filters.group) {
+14
View File
@@ -711,6 +711,20 @@ class ConfigApi {
global.lastReloadTime = Date.now();
}
// Refresh AlertManager rules based on new environment variables
try {
const alertManager = stateManager.getAlertManager();
if (alertManager && typeof alertManager.refreshRules === 'function') {
alertManager.refreshRules();
console.log('Alert rules refreshed after configuration reload');
} else {
console.warn('AlertManager not available or refreshRules method not found');
}
} catch (alertError) {
console.error('Error refreshing alert rules:', alertError);
// Don't fail the entire reload if alert refresh fails
}
// Trigger a discovery cycle if we have any endpoints configured (PVE or PBS)
if (endpoints.length > 0 || pbsConfigs.length > 0) {
console.log('Triggering discovery cycle after configuration reload...');
+6 -1
View File
@@ -436,6 +436,10 @@ function destroy() {
performanceHistory = [];
}
function getAlertManager() {
return alertManager;
}
module.exports = {
init,
getState,
@@ -454,5 +458,6 @@ module.exports = {
destroy,
// Alert manager access
alertManager
alertManager,
getAlertManager
};