gsadmin/pulse
1
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-09-23 11:46:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement per-resource-type alert delays

Browse Source 
- VMs/Containers default to 10 seconds
- Nodes default to 15 seconds
- Storage defaults to 30 seconds
- PBS servers default to 30 seconds

This allows more appropriate delays for different resource types instead of a single global delay that doesn't fit all use cases. Storage and PBS can have longer delays since they're less critical and more prone to transient spikes during operations.
This commit is contained in:
Pulse Monitor
2025-09-10 19:45:42 +00:00
parent 39e82f26c4
commit 2dbd9c4e36
5 changed files with 162 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend-modern/src/components/Alerts/ThresholdsTable.tsx
+82 -23
View File
@@ -53,6 +53,8 @@ interface ThresholdsTableProps {
setStorageDefault: (value: number) => void;
timeThreshold: () => number;
setTimeThreshold: (value: number) => void;
timeThresholds: () => { guest: number; node: number; storage: number; pbs: number };
setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void;
setHasUnsavedChanges: (value: boolean) => void;
activeAlerts?: Record<string, Alert>;
}
@@ -793,29 +795,84 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
</div>
</div>
{/* Additional settings row */}
<div class="flex items-center justify-between pt-2 border-t border-gray-200 dark:border-gray-700">
<div class="flex items-center gap-2">
<label class="text-xs text-gray-500 dark:text-gray-400">
Alert delay:
</label>
<input
type="number"
min="0"
max="300"
value={props.timeThreshold()}
onInput={(e) => {
props.setTimeThreshold(parseInt(e.currentTarget.value) || 0);
props.setHasUnsavedChanges(true);
}}
class="w-14 px-1 py-0.5 text-xs border border-gray-300 dark:border-gray-600 rounded
bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100"
/>
<span class="text-xs text-gray-500 dark:text-gray-400">
seconds before alerting
</span>
</div>
{/* Alert delay settings per resource type */}
<div class="pt-3 border-t border-gray-200 dark:border-gray-700">
<div class="mb-2">
<h4 class="text-xs font-medium text-gray-600 dark:text-gray-400 mb-2">Alert Delay (seconds before triggering)</h4>
<div class="grid grid-cols-2 md:grid-cols-4 gap-3">
<div class="flex items-center gap-2">
<label class="text-xs text-gray-500 dark:text-gray-400 min-w-[80px]">
VMs/Containers:
</label>
<input
type="number"
min="0"
max="300"
value={props.timeThresholds().guest}
onInput={(e) => {
props.setTimeThresholds({...props.timeThresholds(), guest: parseInt(e.currentTarget.value) || 0});
props.setHasUnsavedChanges(true);
}}
class="w-14 px-1 py-0.5 text-xs text-center border border-gray-300 dark:border-gray-600 rounded
bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100"
/>
</div>
<div class="flex items-center gap-2">
<label class="text-xs text-gray-500 dark:text-gray-400 min-w-[80px]">
Nodes:
</label>
<input
type="number"
min="0"
max="300"
value={props.timeThresholds().node}
onInput={(e) => {
props.setTimeThresholds({...props.timeThresholds(), node: parseInt(e.currentTarget.value) || 0});
props.setHasUnsavedChanges(true);
}}
class="w-14 px-1 py-0.5 text-xs text-center border border-gray-300 dark:border-gray-600 rounded
bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100"
/>
</div>
<div class="flex items-center gap-2">
<label class="text-xs text-gray-500 dark:text-gray-400 min-w-[80px]">
Storage:
</label>
<input
type="number"
min="0"
max="300"
value={props.timeThresholds().storage}
onInput={(e) => {
props.setTimeThresholds({...props.timeThresholds(), storage: parseInt(e.currentTarget.value) || 0});
props.setHasUnsavedChanges(true);
}}
class="w-14 px-1 py-0.5 text-xs text-center border border-gray-300 dark:border-gray-600 rounded
bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100"
/>
</div>
<div class="flex items-center gap-2">
<label class="text-xs text-gray-500 dark:text-gray-400 min-w-[80px]">
PBS:
</label>
<input
type="number"
min="0"
max="300"
value={props.timeThresholds().pbs}
onInput={(e) => {
props.setTimeThresholds({...props.timeThresholds(), pbs: parseInt(e.currentTarget.value) || 0});
props.setHasUnsavedChanges(true);
}}
class="w-14 px-1 py-0.5 text-xs text-center border border-gray-300 dark:border-gray-600 rounded
bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100"
/>
</div>
</div>
</div>
{/* Reset button row */}
<div class="flex justify-end pt-2">
<button
onClick={() => {
props.setGuestDefaults({
@@ -834,6 +891,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
});
props.setStorageDefault(85);
props.setTimeThreshold(0);
props.setTimeThresholds({ guest: 10, node: 15, storage: 30, pbs: 30 });
props.setHasUnsavedChanges(true);
}}
class="flex items-center gap-1 px-2 py-0.5 text-xs text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-200 hover:bg-gray-100 dark:hover:bg-gray-800 rounded transition-colors"
@@ -846,6 +904,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
</button>
</div>
</div>
</div>
</div>
{/* Search Bar */}
frontend-modern/src/pages/Alerts.tsx
+32 -2
View File
@@ -282,6 +282,23 @@ export function Alerts() {
if (config.timeThreshold !== undefined) {
setTimeThreshold(config.timeThreshold);
}
// Load per-type time thresholds if available
if (config.timeThresholds) {
setTimeThresholds({
guest: config.timeThresholds.guest ?? 10,
node: config.timeThresholds.node ?? 15,
storage: config.timeThresholds.storage ?? 30,
pbs: config.timeThresholds.pbs ?? 30
});
} else if (config.timeThreshold !== undefined) {
// Fallback to legacy single threshold for all types
setTimeThresholds({
guest: config.timeThreshold,
node: config.timeThreshold,
storage: config.timeThreshold,
pbs: config.timeThreshold
});
}
if (config.overrides) {
// Store raw config to be processed when state is available
setRawOverridesConfig(config.overrides);
@@ -461,7 +478,13 @@ export function Alerts() {
});
const [storageDefault, setStorageDefault] = createSignal(85);
const [timeThreshold, setTimeThreshold] = createSignal(0);
const [timeThreshold, setTimeThreshold] = createSignal(0); // Legacy
const [timeThresholds, setTimeThresholds] = createSignal({
guest: 10,
node: 15,
storage: 30,
pbs: 30
});
const tabs: { id: AlertTab; label: string; icon: string }[] = [
{
@@ -546,7 +569,8 @@ export function Alerts() {
minimumDelta: 2.0,
suppressionWindow: 5,
hysteresisMargin: 5.0,
timeThreshold: timeThreshold() || 0,
timeThreshold: timeThreshold() || 0, // Legacy
timeThresholds: timeThresholds(),
// Use rawOverridesConfig which is already properly formatted with disabled flags
overrides: rawOverridesConfig(),
schedule: {
@@ -668,6 +692,8 @@ export function Alerts() {
setStorageDefault={setStorageDefault}
timeThreshold={timeThreshold}
setTimeThreshold={setTimeThreshold}
timeThresholds={timeThresholds}
setTimeThresholds={setTimeThresholds}
activeAlerts={activeAlerts}
setHasUnsavedChanges={setHasUnsavedChanges}
/>
@@ -991,12 +1017,14 @@ interface ThresholdsTabProps {
nodeDefaults: () => Record<string, number>;
storageDefault: () => number;
timeThreshold: () => number;
timeThresholds: () => { guest: number; node: number; storage: number; pbs: number };
overrides: () => Override[];
rawOverridesConfig: () => Record<string, unknown>;
setGuestDefaults: (value: Record<string, number> | ((prev: Record<string, number>) => Record<string, number>)) => void;
setNodeDefaults: (value: Record<string, number> | ((prev: Record<string, number>) => Record<string, number>)) => void;
setStorageDefault: (value: number) => void;
setTimeThreshold: (value: number) => void;
setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void;
setOverrides: (value: Override[]) => void;
setRawOverridesConfig: (value: Record<string, unknown>) => void;
activeAlerts: Record<string, Alert>;
@@ -1023,6 +1051,8 @@ function ThresholdsTab(props: ThresholdsTabProps) {
setStorageDefault={props.setStorageDefault}
timeThreshold={props.timeThreshold}
setTimeThreshold={props.setTimeThreshold}
timeThresholds={props.timeThresholds}
setTimeThresholds={props.setTimeThresholds}
setHasUnsavedChanges={props.setHasUnsavedChanges}
activeAlerts={props.activeAlerts}
/>
frontend-modern/src/types/alerts.ts
+7 -1
View File
@@ -57,7 +57,13 @@ export interface AlertConfig {
minimumDelta?: number;
suppressionWindow?: number;
hysteresisMargin?: number;
timeThreshold?: number;
timeThreshold?: number; // Legacy single global delay
timeThresholds?: {
guest?: number;
node?: number;
storage?: number;
pbs?: number;
};
aggregation?: {
enabled: boolean;
timeWindow: number;
internal/alerts/alerts.go
+39 -6
View File
@@ -69,6 +69,7 @@ type ThresholdConfig struct {
DiskWrite *HysteresisThreshold `json:"diskWrite,omitempty"`
NetworkIn *HysteresisThreshold `json:"networkIn,omitempty"`
NetworkOut *HysteresisThreshold `json:"networkOut,omitempty"`
Usage *HysteresisThreshold `json:"usage,omitempty"` // For storage devices
// Legacy fields for backward compatibility
CPULegacy *float64 `json:"cpuLegacy,omitempty"`
MemoryLegacy *float64 `json:"memoryLegacy,omitempty"`
@@ -169,7 +170,8 @@ type AlertConfig struct {
MinimumDelta float64 `json:"minimumDelta"` // Minimum % change to trigger new alert
SuppressionWindow int `json:"suppressionWindow"` // Minutes to suppress duplicate alerts
HysteresisMargin float64 `json:"hysteresisMargin"` // Default margin for legacy thresholds
TimeThreshold int `json:"timeThreshold"` // Seconds that threshold must be exceeded before triggering
TimeThreshold int `json:"timeThreshold"` // Legacy: Seconds that threshold must be exceeded before triggering
TimeThresholds map[string]int `json:"timeThresholds"` // Per-type delays: guest, node, storage, pbs
}
// Manager handles alert monitoring and state
@@ -922,6 +924,34 @@ func (m *Manager) clearAlert(alertID string) {
}
}
// getTimeThresholdForType returns the appropriate time threshold for the resource type
func (m *Manager) getTimeThresholdForType(resourceType string) int {
// Use per-type thresholds if available
if m.config.TimeThresholds != nil {
switch resourceType {
case "qemu", "lxc", "guest":
if delay, ok := m.config.TimeThresholds["guest"]; ok {
return delay
}
case "node":
if delay, ok := m.config.TimeThresholds["node"]; ok {
return delay
}
case "storage":
if delay, ok := m.config.TimeThresholds["storage"]; ok {
return delay
}
case "pbs":
if delay, ok := m.config.TimeThresholds["pbs"]; ok {
return delay
}
}
}
// Fall back to legacy single threshold
return m.config.TimeThreshold
}
// checkMetric checks a single metric against its threshold with hysteresis
func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resourceType, metricType string, value float64, threshold *HysteresisThreshold) {
if threshold == nil || threshold.Trigger <= 0 {
@@ -956,24 +986,27 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
if value >= threshold.Trigger {
// Threshold exceeded
if !exists {
// Determine the appropriate time threshold based on resource type
timeThreshold := m.getTimeThresholdForType(resourceType)
// Check if we have a time threshold configured
if m.config.TimeThreshold > 0 {
if timeThreshold > 0 {
// Check if this threshold was already pending
if pendingTime, isPending := m.pendingAlerts[alertID]; isPending {
// Check if enough time has passed
if time.Since(pendingTime) >= time.Duration(m.config.TimeThreshold)*time.Second {
if time.Since(pendingTime) >= time.Duration(timeThreshold)*time.Second {
// Time threshold met, proceed with alert
delete(m.pendingAlerts, alertID)
log.Debug().
Str("alertID", alertID).
Int("timeThreshold", m.config.TimeThreshold).
Int("timeThreshold", timeThreshold).
Dur("elapsed", time.Since(pendingTime)).
Msg("Time threshold met, triggering alert")
} else {
// Still waiting for time threshold
log.Debug().
Str("alertID", alertID).
Int("timeThreshold", m.config.TimeThreshold).
Int("timeThreshold", timeThreshold).
Dur("elapsed", time.Since(pendingTime)).
Msg("Threshold exceeded but waiting for time threshold")
return
@@ -983,7 +1016,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
m.pendingAlerts[alertID] = time.Now()
log.Debug().
Str("alertID", alertID).
Int("timeThreshold", m.config.TimeThreshold).
Int("timeThreshold", timeThreshold).
Msg("Threshold exceeded, starting time threshold tracking")
return
}
internal/api/config_handlers.go
+2 -2
View File
@@ -1175,7 +1175,7 @@ func (h *ConfigHandlers) HandleUpdateNode(w http.ResponseWriter, r *http.Request
// Check if there are overrides for this PBS node
if alertConfig.Overrides != nil {
if override, exists := alertConfig.Overrides[monitoringID]; exists {
if _, exists := alertConfig.Overrides[monitoringID]; exists {
log.Debug().
Str("nodeID", nodeID).
Str("monitoringID", monitoringID).
@@ -2855,7 +2855,7 @@ func (h *ConfigHandlers) HandleSetupScriptURL(w http.ResponseWriter, r *http.Req
host := r.Host
// Dev environment fix: if we detect localhost from vite proxy AND we're in dev mode, use the actual IP
if host == "127.0.0.1:7656" {
if host == "127.0.0.1:7656" || host == "localhost:7656" {
// Check if we're in development mode
if _, err := os.Stat("/opt/pulse/.dev-mode"); err == nil {
// This is the dev backend being proxied through vite on the dev machine