diff --git a/frontend-modern/src/components/Alerts/ResourceTable.tsx b/frontend-modern/src/components/Alerts/ResourceTable.tsx index e2c28373b..42ab2667c 100644 --- a/frontend-modern/src/components/Alerts/ResourceTable.tsx +++ b/frontend-modern/src/components/Alerts/ResourceTable.tsx @@ -87,7 +87,6 @@ interface ResourceTableProps { onSetOfflineState?: (resourceId: string, state: OfflineState) => void; showDelayColumn?: boolean; globalDelaySeconds?: number; - onGlobalDelayChange?: (value: number) => void; editingId: () => string | null; editingThresholds: () => Record; setEditingThresholds: (value: Record) => void; @@ -100,6 +99,8 @@ interface ResourceTableProps { onToggleGlobalDisable?: () => void; globalDisableOfflineFlag?: () => boolean; onToggleGlobalDisableOffline?: () => void; + metricDelaySeconds?: Record; + onMetricDelayChange?: (metricKey: string, value: number | null) => void; groupHeaderMeta?: Record; } @@ -194,37 +195,21 @@ export function ResourceTable(props: ResourceTableProps) { return 80; }; - const resourceDelaySeconds = (resource: Resource): number => { - if (typeof resource.delaySeconds === 'number' && Number.isFinite(resource.delaySeconds)) { - return resource.delaySeconds; - } - if (typeof props.globalDelaySeconds === 'number' && Number.isFinite(props.globalDelaySeconds)) { - return props.globalDelaySeconds; - } - return 0; - }; - const formatDelayLabel = (delay: number): string => (delay <= 0 ? 'Instant' : `${delay}s`); - const isCustomDelay = (resource: Resource): boolean => { - if (typeof resource.delaySeconds !== 'number') { - return false; + const metricDelayOverride = (metric: string): number | undefined => { + const normalized = metric.trim().toLowerCase(); + const value = props.metricDelaySeconds?.[normalized] ?? props.metricDelaySeconds?.[metric]; + if (typeof value !== 'number' || !Number.isFinite(value)) { + return undefined; } - if (!Number.isFinite(resource.delaySeconds)) { - return false; - } - const globalDelay = - typeof props.globalDelaySeconds === 'number' && Number.isFinite(props.globalDelaySeconds) - ? props.globalDelaySeconds - : 0; - return resource.delaySeconds !== globalDelay; + return value; }; const totalColumnCount = () => props.columns.length + 3 + - (props.showOfflineAlertsColumn ? 1 : 0) + - (props.showDelayColumn ? 1 : 0); + (props.showOfflineAlertsColumn ? 1 : 0); const getColumnHeaderTooltip = (column: string): string | undefined => { const normalized = column.trim().toLowerCase(); @@ -421,14 +406,6 @@ export function ResourceTable(props: ResourceTableProps) { Offline Alerts - - - Alert Delay (s) - - Actions @@ -469,7 +446,7 @@ export function ResourceTable(props: ResourceTableProps) { return ( -
+
- + + - + + + + + + + - + + + + Alert Delay (s) + + + + {(column) => { + const metric = normalizeMetricKey(column); + const typeDefaultDelay = props.globalDelaySeconds ?? 5; + const overrideDelay = metricDelayOverride(metric); + + return ( + +
+ { + return overrideDelay !== undefined ? overrideDelay : ''; + })()} + placeholder={formatDelayLabel(typeDefaultDelay)} + class="w-20 rounded border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 px-2 py-0.5 text-xs text-gray-700 dark:text-gray-100 focus:border-blue-500 focus:ring-1 focus:ring-blue-500" + onInput={(e) => { + const raw = e.currentTarget.value; + if (raw === '') { + props.onMetricDelayChange?.(metric, null); + props.setHasUnsavedChanges?.(true); + } else { + const parsed = parseInt(raw, 10); + if (Number.isNaN(parsed)) { + return; + } + props.onMetricDelayChange?.(metric, Math.max(0, parsed)); + props.setHasUnsavedChanges?.(true); + } + }} + /> + + + +
+ + ); + }} + + -
- { - const raw = parseInt(e.currentTarget.value, 10); - if (Number.isNaN(raw)) return; - props.onGlobalDelayChange?.(Math.max(0, raw)); - props.setHasUnsavedChanges?.(true); - }} - class="w-20 rounded border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 text-sm text-gray-700 dark:text-gray-100 px-2 py-0.5 focus:border-blue-500 focus:ring-1 focus:ring-blue-500" - /> -
+ - @@ -765,74 +806,76 @@ export function ResourceTable(props: ResourceTableProps) { } > - - { + openMetricEditor(event); + }} + class="cursor-pointer hover:bg-gray-100 dark:hover:bg-gray-800 rounded px-1 py-0.5 transition-colors" + title="Click to edit this metric" + > + +
+ } + > +
+ { + if ( + isEditing() && + activeMetricInput()?.resourceId === resource.id && + activeMetricInput()?.metric === metric + ) { + queueMicrotask(() => { + el.focus(); + el.select(); + }); + } + }} + onInput={(e) => { + const raw = e.currentTarget.value; + if (raw === '') { + props.setEditingThresholds({ + ...props.editingThresholds(), + [metric]: undefined, + }); + return; + } + const val = parseInt(raw, 10); + if (!Number.isNaN(val)) { + props.setEditingThresholds({ + ...props.editingThresholds(), + [metric]: val, + }); + } + }} + onBlur={() => { + if (props.editingId() === resource.id) { + props.onSaveEdit(resource.id); + } + setActiveMetricInput(null); + }} + class={`w-16 px-2 py-0.5 text-sm text-center border rounded ${ + isDisabled() + ? 'bg-gray-100 dark:bg-gray-800 text-gray-400 dark:text-gray-600 border-gray-300 dark:border-gray-600' + : 'bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 border-gray-300 dark:border-gray-600' + }`} />
- } - > -
- { - if ( - isEditing() && - activeMetricInput()?.resourceId === resource.id && - activeMetricInput()?.metric === metric - ) { - queueMicrotask(() => { - el.focus(); - el.select(); - }); - } - }} - onInput={(e) => { - const raw = e.currentTarget.value; - if (raw === '') { - props.setEditingThresholds({ - ...props.editingThresholds(), - [metric]: undefined, - }); - return; - } - const val = parseInt(raw, 10); - if (!Number.isNaN(val)) { - props.setEditingThresholds({ - ...props.editingThresholds(), - [metric]: val, - }); - } - }} - onBlur={() => { - if (props.editingId() === resource.id) { - props.onSaveEdit(resource.id); - } - setActiveMetricInput(null); - }} - class={`w-16 px-2 py-0.5 text-sm text-center border rounded ${ - isDisabled() - ? 'bg-gray-100 dark:bg-gray-800 text-gray-400 dark:text-gray-600 border-gray-300 dark:border-gray-600' - : 'bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 border-gray-300 dark:border-gray-600' - }`} - /> -
- - + + ); }} @@ -889,32 +932,6 @@ export function ResourceTable(props: ResourceTableProps) { - - - {(() => { - const delay = resourceDelaySeconds(resource); - const label = formatDelayLabel(delay); - const custom = isCustomDelay(resource); - const title = custom - ? 'Custom delay applied to this resource' - : 'Using global alert delay'; - - return ( - - {label} - - ); - })()} - - - {/* Actions column */}
@@ -1310,32 +1327,6 @@ export function ResourceTable(props: ResourceTableProps) { - - - {(() => { - const delay = resourceDelaySeconds(resource); - const label = formatDelayLabel(delay); - const custom = isCustomDelay(resource); - const title = custom - ? 'Custom delay applied to this resource' - : 'Using global alert delay'; - - return ( - - {label} - - ); - })()} - - - {/* Actions column */}
diff --git a/frontend-modern/src/components/Alerts/ThresholdsTable.tsx b/frontend-modern/src/components/Alerts/ThresholdsTable.tsx index c65fa5b3a..c23adf97b 100644 --- a/frontend-modern/src/components/Alerts/ThresholdsTable.tsx +++ b/frontend-modern/src/components/Alerts/ThresholdsTable.tsx @@ -91,7 +91,12 @@ interface ThresholdsTableProps { storageDefault: () => number; setStorageDefault: (value: number) => void; timeThresholds: () => { guest: number; node: number; storage: number; pbs: number }; - setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void; + metricTimeThresholds: () => Record>; + setMetricTimeThresholds: ( + value: + | Record> + | ((prev: Record>) => Record>), + ) => void; setHasUnsavedChanges: (value: boolean) => void; activeAlerts?: Record; removeAlerts?: (predicate: (alert: Alert) => boolean) => void; @@ -130,30 +135,37 @@ export function ThresholdsTable(props: ThresholdsTableProps) { // Set up keyboard shortcuts onMount(() => { - const handleKeyDown = (e: KeyboardEvent) => { - // Skip if user is typing in an input or textarea (unless it's Escape) - const target = e.target as HTMLElement; - const isInInput = - target.tagName === 'INPUT' || - target.tagName === 'TEXTAREA' || - target.contentEditable === 'true'; + const isEditableElement = (el: HTMLElement | null | undefined): boolean => { + if (!el) return false; + const tag = el.tagName; + return ( + tag === 'INPUT' || + tag === 'TEXTAREA' || + tag === 'SELECT' || + el.contentEditable === 'true' + ); + }; + + const handleKeyDown = (e: KeyboardEvent) => { + const target = e.target as HTMLElement | null; + const activeElement = (document.activeElement as HTMLElement) ?? null; + const inEditable = isEditableElement(target); - // Escape clears search from anywhere if (e.key === 'Escape') { - e.preventDefault(); - setSearchTerm(''); - if (searchInputRef && isInInput) { + if (searchTerm()) { + e.preventDefault(); + setSearchTerm(''); + } + if (searchInputRef && document.activeElement === searchInputRef) { searchInputRef.blur(); } return; } - // Skip other shortcuts if already in an input - if (isInInput) { + if (e.defaultPrevented || inEditable || isEditableElement(activeElement) || editingId()) { return; } - // Any letter/number focuses search and starts typing if (e.key.length === 1 && e.key.match(/[a-z0-9]/i)) { e.preventDefault(); if (searchInputRef) { @@ -293,6 +305,30 @@ const buildNodeHeaderMeta = (node: Node) => { return { headerMeta, keys }; }; +const overrideHasContent = (override: Override): boolean => { + const hasThresholds = Object.values(override.thresholds || {}).some( + (value) => value !== undefined, + ); + const hasStateFlags = Boolean( + override.disabled || override.disableConnectivity || override.poweredOffSeverity, + ); + return hasThresholds || hasStateFlags; +}; + +const buildOverrideFromResource = (resource: Resource): Override => ({ + id: resource.id, + name: resource.name, + type: resource.type as OverrideType, + resourceType: resource.resourceType, + vmid: (resource as unknown as { vmid?: number }).vmid, + node: (resource as unknown as { node?: string }).node, + instance: (resource as unknown as { instance?: string }).instance, + disabled: resource.disabled, + disableConnectivity: resource.disableConnectivity, + poweredOffSeverity: resource.poweredOffSeverity, + thresholds: resource.thresholds ? { ...resource.thresholds } : {}, +}); + const nodesWithOverrides = createMemo((prev = []) => { // If we're currently editing, return the previous value to avoid re-renders if (editingId()) { @@ -316,6 +352,7 @@ const buildNodeHeaderMeta = (node: Node) => { ); }); + const originalDisplayName = node.displayName?.trim() || node.name; const friendlyName = getFriendlyNodeName(originalDisplayName, node.clusterName); const rawName = node.name; @@ -341,8 +378,8 @@ const buildNodeHeaderMeta = (node: Node) => { hasOverride: hasCustomThresholds || false, disabled: false, disableConnectivity: override?.disableConnectivity || false, - thresholds: override?.thresholds || {}, - defaults: props.nodeDefaults, + thresholds: override?.thresholds || {}, + defaults: props.nodeDefaults, clusterName: node.isClusterMember ? node.clusterName?.trim() : undefined, isClusterMember: node.isClusterMember ?? false, instance: node.instance, @@ -465,6 +502,7 @@ const dockerContainersGroupedByHost = createMemo>((pr ); }); + const hasOverride = hasCustomThresholds || override?.disabled || override?.disableConnectivity || overrideSeverity !== undefined || false; @@ -621,6 +659,7 @@ const dockerContainersGroupedByHost = createMemo>((pr ); }); + // A guest has an override if it has custom thresholds OR is disabled OR has connectivity disabled const hasOverride = hasCustomThresholds || override?.disabled || override?.disableConnectivity || overrideSeverity !== undefined || false; @@ -720,6 +759,7 @@ const dockerContainersGroupedByHost = createMemo>((pr ); }); + return { id: pbsId, name: pbs.name, @@ -897,6 +937,7 @@ const dockerContainersGroupedByHost = createMemo>((pr const editedThresholds = editingThresholds(); const defaultThresholds = (resource.defaults ?? {}) as Record; + const existingOverride = props.overrides().find((o) => o.id === resourceId); // Only include values that differ from defaults const overrideThresholds: Record = {}; @@ -1012,12 +1053,47 @@ const dockerContainersGroupedByHost = createMemo>((pr setEditingThresholds({}); }; - const updateDelay = (key: 'guest' | 'node' | 'storage' | 'pbs', value: number) => { - const sanitized = Number.isFinite(value) ? Math.max(0, Math.round(value)) : 0; - const current = props.timeThresholds(); - if (current[key] === sanitized) return; - props.setTimeThresholds({ ...current, [key]: sanitized }); - props.setHasUnsavedChanges(true); + const updateMetricDelay = (typeKey: 'guest' | 'node' | 'storage' | 'pbs', metricKey: string, value: number | null) => { + const normalizedMetric = metricKey.trim().toLowerCase(); + if (!normalizedMetric) return; + + let changed = false; + props.setMetricTimeThresholds((prev) => { + const current = prev ? { ...prev } : {}; + const existing = prev?.[typeKey]; + const typeOverrides = existing ? { ...existing } : {}; + + if (value === null) { + if (typeOverrides[normalizedMetric] === undefined) { + return prev; + } + delete typeOverrides[normalizedMetric]; + changed = true; + } else { + const sanitized = Math.max(0, Math.round(value)); + if (typeOverrides[normalizedMetric] === sanitized) { + return prev; + } + typeOverrides[normalizedMetric] = sanitized; + changed = true; + } + + if (!changed) { + return prev; + } + + if (Object.keys(typeOverrides).length === 0) { + delete current[typeKey]; + } else { + current[typeKey] = typeOverrides; + } + + return current; + }); + + if (changed) { + props.setHasUnsavedChanges(true); + } }; const removeOverride = (resourceId: string) => { @@ -1052,6 +1128,7 @@ const dockerContainersGroupedByHost = createMemo>((pr // Get existing override if it exists const existingOverride = props.overrides().find((o) => o.id === resourceId); + const existingRaw = props.rawOverridesConfig()[resourceId]; // Determine the current disabled state - check the resource's current state, not the override const currentDisabledState = resource.disabled; @@ -1062,7 +1139,10 @@ const dockerContainersGroupedByHost = createMemo>((pr delete (cleanThresholds as Record).disabled; // If enabling (disabled = false) and no custom thresholds exist, remove the override entirely - if (!newDisabledState && (!existingOverride || Object.keys(cleanThresholds).length === 0)) { + if ( + !newDisabledState && + (!existingOverride || Object.keys(cleanThresholds).length === 0) + ) { // Remove the override completely props.setOverrides(props.overrides().filter((o) => o.id !== resourceId)); @@ -1108,9 +1188,15 @@ const dockerContainersGroupedByHost = createMemo>((pr if (newDisabledState) { hysteresisThresholds.disabled = true; + } else { + delete hysteresisThresholds.disabled; } - newRawConfig[resourceId] = hysteresisThresholds; + if (Object.keys(hysteresisThresholds).length === 0) { + delete newRawConfig[resourceId]; + } else { + newRawConfig[resourceId] = hysteresisThresholds; + } props.setRawOverridesConfig(newRawConfig); } @@ -1158,6 +1244,7 @@ const dockerContainersGroupedByHost = createMemo>((pr // Get existing override if it exists const existingOverride = props.overrides().find((o) => o.id === resourceId); + const existingRaw = props.rawOverridesConfig()[resourceId]; // Determine the current state - use the resource's computed state, not just the override const currentDisableConnectivity = resource.disableConnectivity; @@ -1215,9 +1302,15 @@ const dockerContainersGroupedByHost = createMemo>((pr if (newDisableConnectivity) { hysteresisThresholds.disableConnectivity = true; + } else { + delete hysteresisThresholds.disableConnectivity; } - newRawConfig[resourceId] = hysteresisThresholds; + if (Object.keys(hysteresisThresholds).length === 0) { + delete newRawConfig[resourceId]; + } else { + newRawConfig[resourceId] = hysteresisThresholds; + } props.setRawOverridesConfig(newRawConfig); } @@ -1242,6 +1335,7 @@ const dockerContainersGroupedByHost = createMemo>((pr const defaultSeverity = props.guestPoweredOffSeverity(); const existingOverride = props.overrides().find((o) => o.id === resourceId); + const existingRaw = props.rawOverridesConfig()[resourceId]; const cleanThresholds: Record = { ...(existingOverride?.thresholds || {}) }; delete (cleanThresholds as Record).disabled; delete (cleanThresholds as Record).disableConnectivity; @@ -1258,7 +1352,12 @@ const dockerContainersGroupedByHost = createMemo>((pr newDisableConnectivity !== defaultDisabled || (!newDisableConnectivity && newSeverity !== defaultSeverity); - if (!differsFromDefaults && !hasThresholds && !overrideDisabled && !existingOverride?.disableConnectivity) { + if ( + !differsFromDefaults && + !hasThresholds && + !overrideDisabled && + !existingOverride?.disableConnectivity + ) { // Remove override entirely if (existingOverride) { props.setOverrides(props.overrides().filter((o) => o.id !== resourceId)); @@ -1444,7 +1543,8 @@ const dockerContainersGroupedByHost = createMemo>((pr onToggleGlobalDisableOffline={() => props.setDisableAllNodesOffline(!props.disableAllNodesOffline())} showDelayColumn={true} globalDelaySeconds={props.timeThresholds().node} - onGlobalDelayChange={(value) => updateDelay('node', value)} + metricDelaySeconds={props.metricTimeThresholds().node ?? {}} + onMetricDelayChange={(metric, value) => updateMetricDelay('node', metric, value)} />
@@ -1492,7 +1592,8 @@ const dockerContainersGroupedByHost = createMemo>((pr onSetOfflineState={setOfflineState} showDelayColumn={true} globalDelaySeconds={props.timeThresholds().guest} - onGlobalDelayChange={(value) => updateDelay('guest', value)} + metricDelaySeconds={props.metricTimeThresholds().guest ?? {}} + onMetricDelayChange={(metric, value) => updateMetricDelay('guest', metric, value)} />
@@ -1531,7 +1632,8 @@ const dockerContainersGroupedByHost = createMemo>((pr onToggleGlobalDisable={() => props.setDisableAllStorage(!props.disableAllStorage())} showDelayColumn={true} globalDelaySeconds={props.timeThresholds().storage} - onGlobalDelayChange={(value) => updateDelay('storage', value)} + metricDelaySeconds={props.metricTimeThresholds().storage ?? {}} + onMetricDelayChange={(metric, value) => updateMetricDelay('storage', metric, value)} />
@@ -1572,7 +1674,8 @@ const dockerContainersGroupedByHost = createMemo>((pr onToggleGlobalDisableOffline={() => props.setDisableAllPBSOffline(!props.disableAllPBSOffline())} showDelayColumn={true} globalDelaySeconds={props.timeThresholds().pbs} - onGlobalDelayChange={(value) => updateDelay('pbs', value)} + metricDelaySeconds={props.metricTimeThresholds().pbs ?? {}} + onMetricDelayChange={(metric, value) => updateMetricDelay('pbs', metric, value)} /> @@ -1675,7 +1778,8 @@ const dockerContainersGroupedByHost = createMemo>((pr } showDelayColumn={true} globalDelaySeconds={props.timeThresholds().guest} - onGlobalDelayChange={(value) => updateDelay('guest', value)} + metricDelaySeconds={props.metricTimeThresholds().guest ?? {}} + onMetricDelayChange={(metric, value) => updateMetricDelay('guest', metric, value)} globalOfflineSeverity={props.guestPoweredOffSeverity()} onSetOfflineState={setOfflineState} /> diff --git a/frontend-modern/src/pages/Alerts.tsx b/frontend-modern/src/pages/Alerts.tsx index d1e66aa59..132615b7a 100644 --- a/frontend-modern/src/pages/Alerts.tsx +++ b/frontend-modern/src/pages/Alerts.tsx @@ -163,11 +163,40 @@ const createDefaultGrouping = (): GroupingConfig => ({ byGuest: false, }); +const normalizeMetricDelayMap = ( + input: Record> | undefined | null, +): Record> => { + if (!input) return {}; + const normalized: Record> = {}; + + Object.entries(input).forEach(([rawType, metrics]) => { + if (!metrics) return; + const typeKey = rawType.trim().toLowerCase(); + if (!typeKey) return; + + const entries: Record = {}; + Object.entries(metrics).forEach(([rawMetric, value]) => { + if (typeof value !== 'number' || Number.isNaN(value) || value < 0) return; + const metricKey = rawMetric.trim().toLowerCase(); + if (!metricKey) return; + entries[metricKey] = Math.round(value); + }); + + if (Object.keys(entries).length > 0) { + normalized[typeKey] = entries; + } + }); + + return normalized; +}; + const createDefaultEscalation = (): EscalationConfig => ({ enabled: false, levels: [], }); +const DEFAULT_DELAY_SECONDS = 5; + export function Alerts() { const { state, activeAlerts, updateAlert, removeAlerts } = useWebSocket(); const navigate = useNavigate(); @@ -462,23 +491,23 @@ export function Alerts() { const container = (state.containers || []).find((g) => g.id === key); const guest = vm || container; if (guest) { - overridesList.push({ - id: key, - name: guest.name, - type: 'guest', - resourceType: guest.type === 'qemu' ? 'VM' : 'CT', - vmid: guest.vmid, - node: guest.node, - instance: guest.instance, - disabled: thresholds.disabled || false, - poweredOffSeverity: - thresholds.poweredOffSeverity === 'critical' - ? 'critical' - : thresholds.poweredOffSeverity === 'warning' - ? 'warning' - : undefined, - thresholds: extractTriggerValues(thresholds), - }); + overridesList.push({ + id: key, + name: guest.name, + type: 'guest', + resourceType: guest.type === 'qemu' ? 'VM' : 'CT', + vmid: guest.vmid, + node: guest.node, + instance: guest.instance, + disabled: thresholds.disabled || false, + poweredOffSeverity: + thresholds.poweredOffSeverity === 'critical' + ? 'critical' + : thresholds.poweredOffSeverity === 'warning' + ? 'warning' + : undefined, + thresholds: extractTriggerValues(thresholds), + }); } } } @@ -507,7 +536,12 @@ export function Alerts() { (newOverride.type === 'guest' || newOverride.type === 'dockerContainer') && (newOverride.poweredOffSeverity ?? null) !== (existing.poweredOffSeverity ?? null); - return thresholdsChanged || connectivityChanged || disabledChanged || severityChanged; + return ( + thresholdsChanged || + connectivityChanged || + disabledChanged || + severityChanged + ); }); if (hasChanged) { @@ -539,8 +573,14 @@ export function Alerts() { temperature: 80, }); setStorageDefault(85); - setTimeThreshold(0); - setTimeThresholds({ guest: 10, node: 15, storage: 30, pbs: 30 }); + setTimeThreshold(DEFAULT_DELAY_SECONDS); + setTimeThresholds({ + guest: DEFAULT_DELAY_SECONDS, + node: DEFAULT_DELAY_SECONDS, + storage: DEFAULT_DELAY_SECONDS, + pbs: DEFAULT_DELAY_SECONDS, + }); + setMetricTimeThresholds({}); setScheduleQuietHours(createDefaultQuietHours()); setScheduleCooldown(createDefaultCooldown()); setScheduleGrouping(createDefaultGrouping()); @@ -613,19 +653,25 @@ export function Alerts() { } if (config.timeThresholds) { setTimeThresholds({ - guest: config.timeThresholds.guest ?? 10, - node: config.timeThresholds.node ?? 15, - storage: config.timeThresholds.storage ?? 30, - pbs: config.timeThresholds.pbs ?? 30, + guest: config.timeThresholds.guest ?? DEFAULT_DELAY_SECONDS, + node: config.timeThresholds.node ?? DEFAULT_DELAY_SECONDS, + storage: config.timeThresholds.storage ?? DEFAULT_DELAY_SECONDS, + pbs: config.timeThresholds.pbs ?? DEFAULT_DELAY_SECONDS, }); - } else if (config.timeThreshold !== undefined && config.timeThreshold > 0) { + } else { + const fallback = config.timeThreshold && config.timeThreshold > 0 ? config.timeThreshold : DEFAULT_DELAY_SECONDS; setTimeThresholds({ - guest: config.timeThreshold, - node: config.timeThreshold, - storage: config.timeThreshold, - pbs: config.timeThreshold, + guest: fallback, + node: fallback, + storage: fallback, + pbs: fallback, }); } + if (config.metricTimeThresholds) { + setMetricTimeThresholds(normalizeMetricDelayMap(config.metricTimeThresholds)); + } else { + setMetricTimeThresholds({}); + } // Load global disable flags setDisableAllNodes(config.disableAllNodes ?? false); @@ -856,14 +902,16 @@ export function Alerts() { memoryCriticalPct: 95, }); - const [storageDefault, setStorageDefault] = createSignal(85); - const [timeThreshold, setTimeThreshold] = createSignal(0); // Legacy - const [timeThresholds, setTimeThresholds] = createSignal({ - guest: 10, - node: 15, - storage: 30, - pbs: 30, - }); +const [storageDefault, setStorageDefault] = createSignal(85); +const [timeThreshold, setTimeThreshold] = createSignal(DEFAULT_DELAY_SECONDS); // Legacy +const [timeThresholds, setTimeThresholds] = createSignal({ + guest: DEFAULT_DELAY_SECONDS, + node: DEFAULT_DELAY_SECONDS, + storage: DEFAULT_DELAY_SECONDS, + pbs: DEFAULT_DELAY_SECONDS, +}); + const [metricTimeThresholds, setMetricTimeThresholds] = + createSignal>>({}); // Global disable flags per resource type const [disableAllNodes, setDisableAllNodes] = createSignal(false); @@ -999,6 +1047,7 @@ export function Alerts() { hysteresisMargin: 5.0, timeThreshold: timeThreshold() || 0, // Legacy timeThresholds: timeThresholds(), + metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()), // Use rawOverridesConfig which is already properly formatted with disabled flags overrides: rawOverridesConfig(), schedule: { @@ -1197,7 +1246,8 @@ export function Alerts() { storageDefault={storageDefault} setStorageDefault={setStorageDefault} timeThresholds={timeThresholds} - setTimeThresholds={setTimeThresholds} + metricTimeThresholds={metricTimeThresholds} + setMetricTimeThresholds={setMetricTimeThresholds} activeAlerts={activeAlerts} setHasUnsavedChanges={setHasUnsavedChanges} hasUnsavedChanges={hasUnsavedChanges} @@ -1669,6 +1719,7 @@ interface ThresholdsTabProps { dockerDefaults: () => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }; storageDefault: () => number; timeThresholds: () => { guest: number; node: number; storage: number; pbs: number }; + metricTimeThresholds: () => Record>; overrides: () => Override[]; rawOverridesConfig: () => Record; setGuestDefaults: ( @@ -1689,7 +1740,11 @@ interface ThresholdsTabProps { value: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number } | ((prev: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }) => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }), ) => void; setStorageDefault: (value: number) => void; - setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void; + setMetricTimeThresholds: ( + value: + | Record> + | ((prev: Record>) => Record>), + ) => void; setOverrides: (value: Override[]) => void; setRawOverridesConfig: (value: Record) => void; activeAlerts: Record; @@ -1746,7 +1801,8 @@ function ThresholdsTab(props: ThresholdsTabProps) { storageDefault={props.storageDefault} setStorageDefault={props.setStorageDefault} timeThresholds={props.timeThresholds} - setTimeThresholds={props.setTimeThresholds} + metricTimeThresholds={props.metricTimeThresholds} + setMetricTimeThresholds={props.setMetricTimeThresholds} setHasUnsavedChanges={props.setHasUnsavedChanges} activeAlerts={props.activeAlerts} removeAlerts={props.removeAlerts} @@ -2627,6 +2683,7 @@ function HistoryTab() { const [alertHistory, setAlertHistory] = createSignal([]); const [loading, setLoading] = createSignal(true); const [selectedBarIndex, setSelectedBarIndex] = createSignal(null); + const MS_PER_HOUR = 60 * 60 * 1000; // Ref for search input let searchInputRef: HTMLInputElement | undefined; @@ -2640,6 +2697,25 @@ function HistoryTab() { localStorage.setItem('alertHistorySeverityFilter', severityFilter()); }); + // Clear chart selection when high-level filters change + let lastTimeFilterValue: string | null = null; + createEffect(() => { + const current = timeFilter(); + if (lastTimeFilterValue !== null && current !== lastTimeFilterValue) { + setSelectedBarIndex(null); + } + lastTimeFilterValue = current; + }); + + let lastSeverityFilterValue: string | null = null; + createEffect(() => { + const current = severityFilter(); + if (lastSeverityFilterValue !== null && current !== lastSeverityFilterValue) { + setSelectedBarIndex(null); + } + lastSeverityFilterValue = current; + }); + // Load alert history on mount onMount(async () => { try { @@ -2703,8 +2779,54 @@ function HistoryTab() { return `${minutes}m`; }; - // Get resource type (VM, CT, Node, Storage) - const getResourceType = (resourceName: string) => { + const formatBucketRange = (startMs: number, endMs: number) => { + const start = new Date(startMs); + const end = new Date(endMs); + + const sameDay = + start.getFullYear() === end.getFullYear() && + start.getMonth() === end.getMonth() && + start.getDate() === end.getDate(); + + const startDay = start.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + year: start.getFullYear() !== end.getFullYear() ? 'numeric' : undefined, + }); + const endDay = end.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + year: 'numeric', + }); + + const timeFormatter: Intl.DateTimeFormatOptions = { + hour: 'numeric', + minute: '2-digit', + }; + + const startTimeStr = start.toLocaleTimeString('en-US', timeFormatter); + const endTimeStr = end.toLocaleTimeString('en-US', timeFormatter); + + if (sameDay) { + return `${startDay}, ${startTimeStr} – ${endTimeStr}`; + } + + return `${startDay}, ${startTimeStr} → ${endDay}, ${endTimeStr}`; + }; + + // Get resource type (VM, CT, Node, Storage, Docker, PBS, etc.) + const getResourceType = ( + resourceName: string, + metadata?: Record | undefined, + ) => { + const metadataType = + typeof metadata?.resourceType === 'string' + ? (metadata.resourceType as string) + : undefined; + if (metadataType && metadataType.trim().length > 0) { + return metadataType; + } + // Check VMs and containers const vm = state.vms?.find((v) => v.name === resourceName); if (vm) return 'VM'; @@ -2720,6 +2842,34 @@ function HistoryTab() { const storage = state.storage?.find((s) => s.name === resourceName || s.id === resourceName); if (storage) return 'Storage'; + // Docker hosts + const dockerHost = state.dockerHosts?.find( + (host) => + host.displayName === resourceName || + host.hostname === resourceName || + host.agentId === resourceName || + host.id === resourceName, + ); + if (dockerHost) return 'Docker Host'; + + // Docker containers (via known hosts) + const dockerContainer = state.dockerHosts + ?.flatMap((host) => host.containers || []) + .find((c) => c.name === resourceName || c.id === resourceName); + if (dockerContainer) return 'Docker Container'; + + // PBS instances + const pbsInstance = state.pbs?.find( + (pbs) => pbs.name === resourceName || pbs.host === resourceName || pbs.id === resourceName, + ); + if (pbsInstance) return 'PBS'; + + // Ceph clusters + const cephCluster = state.cephClusters?.find( + (cluster) => cluster.name === resourceName || cluster.id === resourceName, + ); + if (cephCluster) return 'Ceph'; + return 'Unknown'; }; @@ -2741,7 +2891,7 @@ function HistoryTab() { ...alert, status: 'active', duration: formatDuration(alert.startTime), - resourceType: getResourceType(alert.resourceName), + resourceType: getResourceType(alert.resourceName, alert.metadata), }); }); @@ -2759,63 +2909,67 @@ function HistoryTab() { ...alert, status: alert.acknowledged ? 'acknowledged' : 'resolved', duration: formatDuration(alert.startTime, alert.lastSeen), - resourceType: getResourceType(alert.resourceName), + resourceType: getResourceType(alert.resourceName, alert.metadata), }); }); return allAlerts; }); + // Apply severity & search filters (time filtering is layered separately) + const severityAndSearchFilteredAlerts = createMemo(() => { + let filtered = allAlertsData(); + + if (severityFilter() !== 'all') { + filtered = filtered.filter((a) => a.level === severityFilter()); + } + + if (searchTerm()) { + const term = searchTerm().toLowerCase(); + filtered = filtered.filter((alert) => { + const name = alert.resourceName?.toLowerCase() ?? ''; + const message = alert.message?.toLowerCase() ?? ''; + const type = alert.type?.toLowerCase() ?? ''; + const nodeName = alert.node?.toLowerCase() ?? ''; + return ( + name.includes(term) || message.includes(term) || type.includes(term) || nodeName.includes(term) + ); + }); + } + + return filtered; + }); + // Apply filters to get the final alert data const alertData = createMemo(() => { - let filtered = allAlertsData(); + let filtered = severityAndSearchFilteredAlerts(); // Selected bar filter (takes precedence over time filter) if (selectedBarIndex() !== null) { const trends = alertTrends(); const index = selectedBarIndex()!; const bucketStart = trends.bucketTimes[index]; - const bucketEnd = bucketStart + trends.bucketSize * 60 * 60 * 1000; + const bucketEnd = bucketStart + trends.bucketSize * MS_PER_HOUR; filtered = filtered.filter((alert) => { const alertTime = new Date(alert.startTime).getTime(); return alertTime >= bucketStart && alertTime < bucketEnd; }); - } else { - // Time filter - if (timeFilter() !== 'all') { - const now = Date.now(); - const cutoff = { - '24h': now - 24 * 60 * 60 * 1000, - '7d': now - 7 * 24 * 60 * 60 * 1000, - '30d': now - 30 * 24 * 60 * 60 * 1000, - }[timeFilter()]; + } else if (timeFilter() !== 'all') { + const now = Date.now(); + const cutoff = { + '24h': now - 24 * 60 * 60 * 1000, + '7d': now - 7 * 24 * 60 * 60 * 1000, + '30d': now - 30 * 24 * 60 * 60 * 1000, + }[timeFilter()]; - if (cutoff) { - filtered = filtered.filter((a) => new Date(a.startTime).getTime() > cutoff); - } + if (cutoff) { + filtered = filtered.filter((a) => new Date(a.startTime).getTime() > cutoff); } } - // Severity filter - if (severityFilter() !== 'all') { - filtered = filtered.filter((a) => a.level === severityFilter()); - } - - // Search filter - if (searchTerm()) { - const term = searchTerm().toLowerCase(); - filtered = filtered.filter( - (alert) => - alert.resourceName.toLowerCase().includes(term) || - alert.message.toLowerCase().includes(term) || - alert.type.toLowerCase().includes(term) || - alert.node.toLowerCase().includes(term), - ); - } - // Sort by start time (newest first) - return filtered.sort( + return [...filtered].sort( (a, b) => new Date(b.startTime).getTime() - new Date(a.startTime).getTime(), ); }); @@ -2919,60 +3073,114 @@ function HistoryTab() { // Calculate alert trends for mini-chart const alertTrends = createMemo(() => { const now = Date.now(); - const timeRange = - timeFilter() === '24h' - ? 24 - : timeFilter() === '7d' - ? 7 * 24 - : timeFilter() === '30d' - ? 30 * 24 - : 90 * 24; // hours - const bucketSize = - timeFilter() === '24h' ? 1 : timeFilter() === '7d' ? 6 : timeFilter() === '30d' ? 24 : 72; // hours per bucket - const numBuckets = Math.min(Math.floor(timeRange / bucketSize), 30); // Limit to 30 buckets max + const msPerHour = MS_PER_HOUR; + const filteredAlerts = severityAndSearchFilteredAlerts(); + const niceBucketSizes = [1, 2, 3, 6, 12, 24, 48, 72, 168, 336, 720, 1440]; // hours + const maxBuckets = 30; - // Calculate start time for the chart - const startTime = now - timeRange * 60 * 60 * 1000; + let bucketSizeHours: number; + let computedRangeHours: number; + let startTime: number; - // Initialize buckets - const buckets = new Array(numBuckets).fill(0); - // bucketTimes represents the START of each bucket - const bucketTimes = new Array(numBuckets) - .fill(0) - .map((_, i) => startTime + i * bucketSize * 60 * 60 * 1000); - - // Filter alerts based on current time filter - let alertsToCount = allAlertsData(); - if (timeFilter() !== 'all') { - const cutoff = { - '24h': now - 24 * 60 * 60 * 1000, - '7d': now - 7 * 24 * 60 * 60 * 1000, - '30d': now - 30 * 24 * 60 * 60 * 1000, - }[timeFilter()]; - - if (cutoff) { - alertsToCount = alertsToCount.filter((a) => new Date(a.startTime).getTime() > cutoff); + const filter = timeFilter(); + if (filter === '24h') { + bucketSizeHours = 1; + computedRangeHours = 24; + startTime = now - computedRangeHours * msPerHour; + } else if (filter === '7d') { + bucketSizeHours = 6; + computedRangeHours = 7 * 24; + startTime = now - computedRangeHours * msPerHour; + } else if (filter === '30d') { + bucketSizeHours = 24; + computedRangeHours = 30 * 24; + startTime = now - computedRangeHours * msPerHour; + } else { + if (!filteredAlerts.length) { + bucketSizeHours = 24; + computedRangeHours = 24; + startTime = now - computedRangeHours * msPerHour; + } else { + const earliest = filteredAlerts.reduce((min, alert) => { + const alertTime = new Date(alert.startTime).getTime(); + return Math.min(min, alertTime); + }, now); + const rawRangeHours = Math.max(1, Math.ceil((now - earliest) / msPerHour)); + const rawBucketSize = Math.max(1, Math.ceil(rawRangeHours / maxBuckets)); + bucketSizeHours = + niceBucketSizes.find((size) => size >= rawBucketSize) ?? rawBucketSize; + computedRangeHours = Math.max(rawRangeHours, bucketSizeHours); + const bucketsNeeded = Math.min( + Math.max(1, Math.ceil(computedRangeHours / bucketSizeHours)), + maxBuckets, + ); + startTime = now - bucketsNeeded * bucketSizeHours * msPerHour; } } - alertsToCount.forEach((alert) => { + const bucketCount = Math.min( + Math.max(1, Math.ceil(computedRangeHours / bucketSizeHours)), + maxBuckets, + ); + startTime = Math.min(startTime, now - bucketCount * bucketSizeHours * msPerHour); + + const buckets = new Array(bucketCount).fill(0); + const bucketTimes = new Array(bucketCount) + .fill(0) + .map((_, i) => startTime + i * bucketSizeHours * msPerHour); + + const windowStart = startTime; + const windowEnd = now; + + filteredAlerts.forEach((alert) => { const alertTime = new Date(alert.startTime).getTime(); - if (alertTime >= startTime && alertTime <= now) { - const bucketIndex = Math.floor((alertTime - startTime) / (bucketSize * 60 * 60 * 1000)); - if (bucketIndex >= 0 && bucketIndex < numBuckets) { - buckets[bucketIndex]++; - } + if (alertTime < windowStart || alertTime > windowEnd) { + return; + } + const rawIndex = Math.floor((alertTime - windowStart) / (bucketSizeHours * msPerHour)); + const bucketIndex = Math.min(bucketCount - 1, Math.max(0, rawIndex)); + if (bucketIndex >= 0 && bucketIndex < bucketCount) { + buckets[bucketIndex]++; } }); - // Find max for scaling const max = Math.max(...buckets, 1); return { buckets, max, - bucketSize, + bucketSize: bucketSizeHours, bucketTimes, + rangeStart: windowStart, + rangeHours: bucketCount * bucketSizeHours, + }; + }); + + const chartRangeLabel = createMemo(() => { + const filter = timeFilter(); + if (filter === '24h') return '24h ago'; + if (filter === '7d') return '7d ago'; + if (filter === '30d') return '30d ago'; + + const rangeHours = alertTrends().rangeHours ?? 0; + if (rangeHours <= 0) return '—'; + if (rangeHours >= 24) { + const days = Math.round(rangeHours / 24); + return `${days}d ago`; + } + return `${Math.round(rangeHours)}h ago`; + }); + + const selectedBucketDetails = createMemo(() => { + const index = selectedBarIndex(); + if (index === null) return null; + const trends = alertTrends(); + const bucketStart = trends.bucketTimes[index]; + const bucketEnd = bucketStart + trends.bucketSize * MS_PER_HOUR; + return { + rangeLabel: formatBucketRange(bucketStart, bucketEnd), + start: bucketStart, + end: bucketEnd, }; }); @@ -2980,7 +3188,7 @@ function HistoryTab() {
{/* Alert Trends Mini-Chart */} -
+
-
- - +
+ + {(selection) => ( +
+ + Filtered Range + + {selection().rangeLabel} +
+ )} -
- -
- {alertData().filter((a) => a.level === 'warning').length} warnings - - -
- {alertData().filter((a) => a.level === 'critical').length} critical - +
+ + + +
+ +
+ {alertData().filter((a) => a.level === 'warning').length} warnings + + +
+ {alertData().filter((a) => a.level === 'critical').length} critical + +
@@ -3049,11 +3269,10 @@ function HistoryTab() { } const rect = (e.currentTarget as HTMLElement).getBoundingClientRect(); const bucketHours = alertTrends().bucketSize; - const bucketLabel = (() => { - if (timeFilter() === '24h') return `${bucketHours} hour period`; - const bucketDays = bucketHours / 24; - return `${bucketDays} day period`; - })(); + const bucketLabel = + bucketHours % 24 === 0 + ? `${bucketHours / 24} day${bucketHours / 24 === 1 ? '' : 's'}` + : `${bucketHours} hour${bucketHours === 1 ? '' : 's'}`; const timestamp = new Date(alertTrends().bucketTimes[i]).toLocaleString('en-US', { month: 'short', day: 'numeric', @@ -3062,7 +3281,7 @@ function HistoryTab() { }); const content = [ `${val} alert${val !== 1 ? 's' : ''}`, - bucketLabel, + `${bucketLabel} period`, timestamp, ].join('\n'); showTooltip(content, rect.left + rect.width / 2, rect.top, { @@ -3080,13 +3299,7 @@ function HistoryTab() { {/* Time labels */}
- {timeFilter() === '24h' - ? '24h ago' - : timeFilter() === '7d' - ? '7d ago' - : timeFilter() === '30d' - ? '30d ago' - : '90d ago'} + {chartRangeLabel()} Now
diff --git a/frontend-modern/src/types/alerts.ts b/frontend-modern/src/types/alerts.ts index 21980222d..4d2914386 100644 --- a/frontend-modern/src/types/alerts.ts +++ b/frontend-modern/src/types/alerts.ts @@ -84,6 +84,7 @@ export interface AlertConfig { storage?: number; pbs?: number; }; + metricTimeThresholds?: Record>; aggregation?: { enabled: boolean; timeWindow: number; diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 2684343ac..1b8eb0c53 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -276,11 +276,12 @@ type AlertConfig struct { DisableAllPBSOffline bool `json:"disableAllPBSOffline"` // Disable PBS offline alerts globally DisableAllDockerHostsOffline bool `json:"disableAllDockerHostsOffline"` // Disable Docker host offline alerts globally // New configuration options - MinimumDelta float64 `json:"minimumDelta"` // Minimum % change to trigger new alert - SuppressionWindow int `json:"suppressionWindow"` // Minutes to suppress duplicate alerts - HysteresisMargin float64 `json:"hysteresisMargin"` // Default margin for legacy thresholds - TimeThreshold int `json:"timeThreshold"` // Legacy: Seconds that threshold must be exceeded before triggering - TimeThresholds map[string]int `json:"timeThresholds"` // Per-type delays: guest, node, storage, pbs + MinimumDelta float64 `json:"minimumDelta"` // Minimum % change to trigger new alert + SuppressionWindow int `json:"suppressionWindow"` // Minutes to suppress duplicate alerts + HysteresisMargin float64 `json:"hysteresisMargin"` // Default margin for legacy thresholds + TimeThreshold int `json:"timeThreshold"` // Legacy: Seconds that threshold must be exceeded before triggering + TimeThresholds map[string]int `json:"timeThresholds"` // Per-type delays: guest, node, storage, pbs + MetricTimeThresholds map[string]map[string]int `json:"metricTimeThresholds"` // Optional per-metric delays keyed by resource type } // Manager handles alert monitoring and state @@ -388,11 +389,12 @@ func NewManager() *Manager { MinimumDelta: 2.0, // 2% minimum change SuppressionWindow: 5, // 5 minutes HysteresisMargin: 5.0, // 5% default margin + TimeThreshold: 5, TimeThresholds: map[string]int{ - "guest": 10, // 10 second delay for guest CPU alerts - "node": 15, // 15 second delay for node alerts - "storage": 30, // 30 second delay for storage alerts - "pbs": 30, // 30 second delay for PBS alerts + "guest": 5, + "node": 5, + "storage": 5, + "pbs": 5, }, Overrides: make(map[string]ThresholdConfig), Schedule: ScheduleConfig{ @@ -539,6 +541,39 @@ func (m *Manager) UpdateConfig(config AlertConfig) { config.HysteresisMargin = 5.0 } + // Ensure temperature defaults exist for nodes so high temps alert out of the box + if config.NodeDefaults.Temperature == nil || config.NodeDefaults.Temperature.Trigger <= 0 { + config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75} + } else if config.NodeDefaults.Temperature.Clear <= 0 { + config.NodeDefaults.Temperature.Clear = config.NodeDefaults.Temperature.Trigger - 5 + if config.NodeDefaults.Temperature.Clear <= 0 { + config.NodeDefaults.Temperature.Clear = 75 + } + } + + // Normalize any metric-level delay overrides + config.MetricTimeThresholds = normalizeMetricTimeThresholds(config.MetricTimeThresholds) + + const defaultDelaySeconds = 5 + if config.TimeThreshold <= 0 { + config.TimeThreshold = defaultDelaySeconds + } + if config.TimeThresholds == nil { + config.TimeThresholds = make(map[string]int) + } + ensureDelay := func(key string) { + if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 { + config.TimeThresholds[key] = defaultDelaySeconds + } + } + ensureDelay("guest") + ensureDelay("node") + ensureDelay("storage") + ensureDelay("pbs") + if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 { + config.TimeThresholds["all"] = defaultDelaySeconds + } + config.GuestDefaults.PoweredOffSeverity = normalizePoweredOffSeverity(config.GuestDefaults.PoweredOffSeverity) config.NodeDefaults.PoweredOffSeverity = normalizePoweredOffSeverity(config.NodeDefaults.PoweredOffSeverity) @@ -563,6 +598,42 @@ func (m *Manager) UpdateConfig(config AlertConfig) { m.reevaluateActiveAlertsLocked() } +// normalizeMetricTimeThresholds cleans resource/metric keys and drops invalid delay overrides. +func normalizeMetricTimeThresholds(input map[string]map[string]int) map[string]map[string]int { + if len(input) == 0 { + return nil + } + + normalized := make(map[string]map[string]int) + for rawType, metrics := range input { + typeKey := strings.ToLower(strings.TrimSpace(rawType)) + if typeKey == "" || len(metrics) == 0 { + continue + } + for rawMetric, delay := range metrics { + metricKey := strings.ToLower(strings.TrimSpace(rawMetric)) + if metricKey == "" || delay < 0 { + continue + } + if _, exists := normalized[typeKey]; !exists { + normalized[typeKey] = make(map[string]int) + } + normalized[typeKey][metricKey] = delay + } + } + + if len(normalized) == 0 { + return nil + } + + return normalized +} + +// NormalizeMetricTimeThresholds exposes normalization for other packages (e.g., config persistence). +func NormalizeMetricTimeThresholds(input map[string]map[string]int) map[string]map[string]int { + return normalizeMetricTimeThresholds(input) +} + // applyGlobalOfflineSettingsLocked clears tracking and active alerts for globally disabled offline detectors. // Caller must hold m.mu. func (m *Manager) applyGlobalOfflineSettingsLocked() { @@ -2488,38 +2559,142 @@ func (m *Manager) clearAlert(alertID string) { Msg("Alert cleared") } -// getTimeThresholdForType returns the appropriate time threshold for the resource type -func (m *Manager) getTimeThresholdForType(resourceType string) int { - typeKey := strings.ToLower(strings.TrimSpace(resourceType)) +// getTimeThreshold determines the delay to apply for a metric/resource combination. +func (m *Manager) getTimeThreshold(_ string, resourceType, metricType string) int { + if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok { + return delay + } - // Use per-type thresholds if available - if m.config.TimeThresholds != nil { - switch typeKey { - case "guest", "qemu", "lxc", "vm", "ct", "container": - if delay, ok := m.config.TimeThresholds["guest"]; ok { - return delay - } - case "docker container", "dockercontainer", "docker": - if delay, ok := m.config.TimeThresholds["guest"]; ok { - return delay - } - case "node": - if delay, ok := m.config.TimeThresholds["node"]; ok { - return delay - } - case "storage": - if delay, ok := m.config.TimeThresholds["storage"]; ok { - return delay - } - case "pbs": - if delay, ok := m.config.TimeThresholds["pbs"]; ok { - return delay - } + base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType) + + if !hasTypeSpecific { + if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok { + return delay } } - // Fall back to legacy single threshold - return m.config.TimeThreshold + return base +} + +// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level. +func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) { + if len(m.config.MetricTimeThresholds) == 0 { + return 0, false + } + + metricKey := strings.ToLower(strings.TrimSpace(metricType)) + if metricKey == "" { + return 0, false + } + + for _, typeKey := range canonicalResourceTypeKeys(resourceType) { + perType, ok := m.config.MetricTimeThresholds[typeKey] + if !ok || len(perType) == 0 { + continue + } + + if delay, ok := perType[metricKey]; ok { + return delay, true + } + if delay, ok := perType["default"]; ok { + return delay, true + } + if delay, ok := perType["_default"]; ok { + return delay, true + } + if delay, ok := perType["*"]; ok { + return delay, true + } + } + + return 0, false +} + +// getBaseTimeThreshold returns the resource-type level delay. +func (m *Manager) getBaseTimeThreshold(resourceType string) (int, bool) { + if m.config.TimeThresholds != nil { + for _, key := range canonicalResourceTypeKeys(resourceType) { + if delay, ok := m.config.TimeThresholds[key]; ok { + return delay, true + } + } + if delay, ok := m.config.TimeThresholds["all"]; ok { + return delay, false + } + } + + return m.config.TimeThreshold, false +} + +func (m *Manager) getGlobalMetricTimeThreshold(metricType string) (int, bool) { + if len(m.config.MetricTimeThresholds) == 0 { + return 0, false + } + + perType, ok := m.config.MetricTimeThresholds["all"] + if !ok || len(perType) == 0 { + return 0, false + } + + metricKey := strings.ToLower(strings.TrimSpace(metricType)) + if metricKey == "" { + return 0, false + } + + if delay, ok := perType[metricKey]; ok { + return delay, true + } + if delay, ok := perType["default"]; ok { + return delay, true + } + if delay, ok := perType["_default"]; ok { + return delay, true + } + if delay, ok := perType["*"]; ok { + return delay, true + } + + return 0, false +} + +func canonicalResourceTypeKeys(resourceType string) []string { + typeKey := strings.ToLower(strings.TrimSpace(resourceType)) + + addUnique := func(slice []string, value string) []string { + if value == "" { + return slice + } + for _, existing := range slice { + if existing == value { + return slice + } + } + return append(slice, value) + } + + var keys []string + switch typeKey { + case "guest", "qemu", "vm", "ct", "container", "lxc": + keys = addUnique(keys, "guest") + case "docker", "docker container", "dockercontainer": + keys = addUnique(keys, "docker") + keys = addUnique(keys, "guest") + case "docker host", "dockerhost": + keys = addUnique(keys, "dockerhost") + keys = addUnique(keys, "docker") + keys = addUnique(keys, "node") + case "node": + keys = addUnique(keys, "node") + case "pbs", "pbs server", "pbsserver": + keys = addUnique(keys, "pbs") + keys = addUnique(keys, "node") + case "storage": + keys = addUnique(keys, "storage") + default: + keys = addUnique(keys, typeKey) + } + + return keys } // checkMetric checks a single metric against its threshold with hysteresis @@ -2563,8 +2738,8 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource if !exists { alertStartTime := time.Now() - // Determine the appropriate time threshold based on resource type - timeThreshold := m.getTimeThresholdForType(resourceType) + // Determine the appropriate time threshold based on resource/metric type + timeThreshold := m.getTimeThreshold(resourceID, resourceType, metricType) // Check if we have a time threshold configured if timeThreshold > 0 { diff --git a/internal/alerts/time_threshold_test.go b/internal/alerts/time_threshold_test.go index 8a7b02b57..32e79acb8 100644 --- a/internal/alerts/time_threshold_test.go +++ b/internal/alerts/time_threshold_test.go @@ -5,7 +5,7 @@ import ( "time" ) -func TestGetTimeThresholdForTypeMappings(t *testing.T) { +func TestGetTimeThresholdMappings(t *testing.T) { manager := NewManager() manager.mu.Lock() @@ -35,8 +35,53 @@ func TestGetTimeThresholdForTypeMappings(t *testing.T) { } for _, tc := range testCases { - if got := manager.getTimeThresholdForType(tc.resourceType); got != tc.expected { - t.Errorf("getTimeThresholdForType(%q) = %d, want %d", tc.resourceType, got, tc.expected) + if got := manager.getTimeThreshold("", tc.resourceType, "cpu"); got != tc.expected { + t.Errorf("getTimeThreshold(%q, \"cpu\") = %d, want %d", tc.resourceType, got, tc.expected) + } + } +} + +func TestGetTimeThresholdMetricOverrides(t *testing.T) { + manager := NewManager() + + manager.mu.Lock() + manager.config.TimeThreshold = 15 + manager.config.TimeThresholds = map[string]int{ + "guest": 30, + "node": 60, + "storage": 90, + } + manager.config.MetricTimeThresholds = map[string]map[string]int{ + "guest": { + "cpu": 5, + }, + "node": { + "temperature": 120, + }, + "all": { + "default": 20, + }, + } + manager.mu.Unlock() + + cases := []struct { + resourceID string + resourceType string + metricType string + expected int + }{ + {"vm-resource", "VM", "cpu", 5}, // guest metric override + {"vm-resource", "VM", "memory", 30}, // falls back to guest type delay + {"node-1", "Node", "temperature", 120}, // node metric override + {"node-1", "Node", "cpu", 60}, // node type delay + {"storage-1", "storage", "usage", 90}, // storage type delay + {"unknown", "unknown", "cpu", 20}, // global default metric override + {"unknown", "unknown", "disk", 20}, + } + + for _, tc := range cases { + if got := manager.getTimeThreshold(tc.resourceID, tc.resourceType, tc.metricType); got != tc.expected { + t.Errorf("getTimeThreshold(%q, %q, %q) = %d, want %d", tc.resourceID, tc.resourceType, tc.metricType, got, tc.expected) } } } diff --git a/internal/config/persistence.go b/internal/config/persistence.go index 8cd224e1a..c1a280bbb 100644 --- a/internal/config/persistence.go +++ b/internal/config/persistence.go @@ -103,6 +103,25 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error { if config.HysteresisMargin <= 0 { config.HysteresisMargin = 5.0 } + config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds) + if config.TimeThreshold <= 0 { + config.TimeThreshold = 5 + } + if config.TimeThresholds == nil { + config.TimeThresholds = make(map[string]int) + } + ensureDelay := func(key string) { + if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 { + config.TimeThresholds[key] = config.TimeThreshold + } + } + ensureDelay("guest") + ensureDelay("node") + ensureDelay("storage") + ensureDelay("pbs") + if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 { + config.TimeThresholds["all"] = config.TimeThreshold + } data, err := json.MarshalIndent(config, "", " ") if err != nil { @@ -138,11 +157,19 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) { Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}, }, NodeDefaults: alerts.ThresholdConfig{ - CPU: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}, - Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}, - Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}, + CPU: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}, + Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}, + Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}, + Temperature: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}, + }, + StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80}, + TimeThreshold: 5, + TimeThresholds: map[string]int{ + "guest": 5, + "node": 5, + "storage": 5, + "pbs": 5, }, - StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80}, MinimumDelta: 2.0, SuppressionWindow: 5, HysteresisMargin: 5.0, @@ -175,6 +202,28 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) { if config.HysteresisMargin <= 0 { config.HysteresisMargin = 5.0 } + if config.NodeDefaults.Temperature == nil || config.NodeDefaults.Temperature.Trigger <= 0 { + config.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} + } + if config.TimeThreshold <= 0 { + config.TimeThreshold = 5 + } + if config.TimeThresholds == nil { + config.TimeThresholds = make(map[string]int) + } + ensureDelay := func(key string) { + if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 { + config.TimeThresholds[key] = config.TimeThreshold + } + } + ensureDelay("guest") + ensureDelay("node") + ensureDelay("storage") + ensureDelay("pbs") + if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 { + config.TimeThresholds["all"] = config.TimeThreshold + } + config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds) // Migration: Set I/O metrics to Off (0) if they have the old default values // This helps existing users avoid noisy I/O alerts