feat: add per-metric alert delay configuration

Add sophisticated alert delay system with per-metric overrides:
- Global delay defaults per resource type (guest/node/storage/pbs)
- Per-metric delay overrides with UI controls
- Dynamic tooltips showing effective delay values
- Clear buttons to reset metric-specific overrides
- Temperature threshold defaults for nodes (80°C/75°C)
- Improved delay configuration persistence and normalization
This commit is contained in:
rcourtman
2025-10-09 16:14:39 +00:00
parent 42f20c5a77
commit 667590519a
7 changed files with 969 additions and 391 deletions
@@ -87,7 +87,6 @@ interface ResourceTableProps {
onSetOfflineState?: (resourceId: string, state: OfflineState) => void;
showDelayColumn?: boolean;
globalDelaySeconds?: number;
onGlobalDelayChange?: (value: number) => void;
editingId: () => string | null;
editingThresholds: () => Record<string, number | undefined>;
setEditingThresholds: (value: Record<string, number | undefined>) => void;
@@ -100,6 +99,8 @@ interface ResourceTableProps {
onToggleGlobalDisable?: () => void;
globalDisableOfflineFlag?: () => boolean;
onToggleGlobalDisableOffline?: () => void;
metricDelaySeconds?: Record<string, number>;
onMetricDelayChange?: (metricKey: string, value: number | null) => void;
groupHeaderMeta?: Record<string, GroupHeaderMeta>;
}
@@ -194,37 +195,21 @@ export function ResourceTable(props: ResourceTableProps) {
return 80;
};
const resourceDelaySeconds = (resource: Resource): number => {
if (typeof resource.delaySeconds === 'number' && Number.isFinite(resource.delaySeconds)) {
return resource.delaySeconds;
}
if (typeof props.globalDelaySeconds === 'number' && Number.isFinite(props.globalDelaySeconds)) {
return props.globalDelaySeconds;
}
return 0;
};
const formatDelayLabel = (delay: number): string => (delay <= 0 ? 'Instant' : `${delay}s`);
const isCustomDelay = (resource: Resource): boolean => {
if (typeof resource.delaySeconds !== 'number') {
return false;
const metricDelayOverride = (metric: string): number | undefined => {
const normalized = metric.trim().toLowerCase();
const value = props.metricDelaySeconds?.[normalized] ?? props.metricDelaySeconds?.[metric];
if (typeof value !== 'number' || !Number.isFinite(value)) {
return undefined;
}
if (!Number.isFinite(resource.delaySeconds)) {
return false;
}
const globalDelay =
typeof props.globalDelaySeconds === 'number' && Number.isFinite(props.globalDelaySeconds)
? props.globalDelaySeconds
: 0;
return resource.delaySeconds !== globalDelay;
return value;
};
const totalColumnCount = () =>
props.columns.length +
3 +
(props.showOfflineAlertsColumn ? 1 : 0) +
(props.showDelayColumn ? 1 : 0);
(props.showOfflineAlertsColumn ? 1 : 0);
const getColumnHeaderTooltip = (column: string): string | undefined => {
const normalized = column.trim().toLowerCase();
@@ -421,14 +406,6 @@ export function ResourceTable(props: ResourceTableProps) {
Offline Alerts
</th>
</Show>
<Show when={props.showDelayColumn}>
<th
class="px-3 py-2 text-center text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider"
title="Pulse waits for the configured alert delay before creating a notification. If the metric drops back below the threshold during this window, the alert is suppressed to reduce noise. Update the delay values using the Alert Delay column in each table."
>
Alert Delay (s)
</th>
</Show>
<th class="px-3 py-2 text-center text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
Actions
</th>
@@ -469,7 +446,7 @@ export function ResourceTable(props: ResourceTableProps) {
return (
<td class="p-1 px-2 text-center align-middle">
<div class="relative flex justify-center">
<div class="relative flex justify-center w-full">
<input
type="number"
min={bounds.min}
@@ -558,22 +535,86 @@ export function ResourceTable(props: ResourceTableProps) {
</Show>
</td>
</Show>
<Show when={props.showDelayColumn}>
<td class="p-1 px-2 text-center align-middle">
<span class="text-sm text-gray-400">-</span>
</td>
</tr>
</Show>
<Show when={props.showDelayColumn && typeof props.onMetricDelayChange === 'function'}>
<tr class={`bg-gray-50 dark:bg-gray-800/50 border-b border-gray-300 dark:border-gray-600 ${props.globalDisableFlag?.() ? 'opacity-40' : ''}`}>
<td class="p-1 px-2 text-center align-middle">
<span class="text-sm text-gray-400">-</span>
</td>
<td class="p-1 px-2 align-middle">
<span class="text-xs font-semibold uppercase tracking-wide text-gray-600 dark:text-gray-300">
Alert Delay (s)
</span>
</td>
<For each={props.columns}>
{(column) => {
const metric = normalizeMetricKey(column);
const typeDefaultDelay = props.globalDelaySeconds ?? 5;
const overrideDelay = metricDelayOverride(metric);
return (
<td class="p-1 px-2 text-center align-middle">
<div class="flex items-center justify-center gap-2">
<input
type="number"
min="0"
value={(() => {
return overrideDelay !== undefined ? overrideDelay : '';
})()}
placeholder={formatDelayLabel(typeDefaultDelay)}
class="w-20 rounded border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 px-2 py-0.5 text-xs text-gray-700 dark:text-gray-100 focus:border-blue-500 focus:ring-1 focus:ring-blue-500"
onInput={(e) => {
const raw = e.currentTarget.value;
if (raw === '') {
props.onMetricDelayChange?.(metric, null);
props.setHasUnsavedChanges?.(true);
} else {
const parsed = parseInt(raw, 10);
if (Number.isNaN(parsed)) {
return;
}
props.onMetricDelayChange?.(metric, Math.max(0, parsed));
props.setHasUnsavedChanges?.(true);
}
}}
/>
<Show when={metricDelayOverride(metric) !== undefined}>
<button
type="button"
class="rounded p-1 text-gray-500 transition hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200"
onClick={() => {
props.onMetricDelayChange?.(metric, null);
props.setHasUnsavedChanges?.(true);
}}
title="Clear delay override"
aria-label="Clear delay override"
>
<svg
class="h-3 w-3"
viewBox="0 0 20 20"
fill="currentColor"
aria-hidden="true"
>
<path
fill-rule="evenodd"
d="M10 18a8 8 0 100-16 8 8 0 000 16zm2.707-10.707a1 1 0 00-1.414-1.414L10 7.586 8.707 6.293A1 1 0 007.293 7.707L8.586 9l-1.293 1.293a1 1 0 101.414 1.414L10 10.414l1.293 1.293a1 1 0 001.414-1.414L11.414 9l1.293-1.293z"
clip-rule="evenodd"
/>
</svg>
</button>
</Show>
</div>
</td>
);
}}
</For>
<Show when={props.showOfflineAlertsColumn}>
<td class="p-1 px-2 text-center align-middle">
<div class="flex justify-center">
<input
type="number"
min="0"
value={props.globalDelaySeconds ?? 0}
onInput={(e) => {
const raw = parseInt(e.currentTarget.value, 10);
if (Number.isNaN(raw)) return;
props.onGlobalDelayChange?.(Math.max(0, raw));
props.setHasUnsavedChanges?.(true);
}}
class="w-20 rounded border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 text-sm text-gray-700 dark:text-gray-100 px-2 py-0.5 focus:border-blue-500 focus:ring-1 focus:ring-blue-500"
/>
</div>
<span class="text-sm text-gray-400">-</span>
</td>
</Show>
<td class="p-1 px-2 text-center align-middle">
@@ -765,74 +806,76 @@ export function ResourceTable(props: ResourceTableProps) {
</span>
}
>
<Show
when={isEditing()}
fallback={
<div
onClick={openMetricEditor}
class="cursor-pointer hover:bg-gray-100 dark:hover:bg-gray-800 rounded px-1 py-0.5 transition-colors"
title="Click to edit this metric"
>
<MetricValueWithHeat
resourceId={resource.id}
metric={metric}
value={displayValue(metric)}
isOverridden={isOverridden(metric)}
<Show
when={isEditing()}
fallback={
<div
onClick={(event) => {
openMetricEditor(event);
}}
class="cursor-pointer hover:bg-gray-100 dark:hover:bg-gray-800 rounded px-1 py-0.5 transition-colors"
title="Click to edit this metric"
>
<MetricValueWithHeat
resourceId={resource.id}
metric={metric}
value={displayValue(metric)}
isOverridden={isOverridden(metric)}
/>
</div>
}
>
<div class="flex items-center justify-center">
<input
type="number"
min={bounds.min}
max={bounds.max}
value={thresholds()?.[metric] ?? ''}
placeholder={isDisabled() ? 'Off' : ''}
ref={(el) => {
if (
isEditing() &&
activeMetricInput()?.resourceId === resource.id &&
activeMetricInput()?.metric === metric
) {
queueMicrotask(() => {
el.focus();
el.select();
});
}
}}
onInput={(e) => {
const raw = e.currentTarget.value;
if (raw === '') {
props.setEditingThresholds({
...props.editingThresholds(),
[metric]: undefined,
});
return;
}
const val = parseInt(raw, 10);
if (!Number.isNaN(val)) {
props.setEditingThresholds({
...props.editingThresholds(),
[metric]: val,
});
}
}}
onBlur={() => {
if (props.editingId() === resource.id) {
props.onSaveEdit(resource.id);
}
setActiveMetricInput(null);
}}
class={`w-16 px-2 py-0.5 text-sm text-center border rounded ${
isDisabled()
? 'bg-gray-100 dark:bg-gray-800 text-gray-400 dark:text-gray-600 border-gray-300 dark:border-gray-600'
: 'bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 border-gray-300 dark:border-gray-600'
}`}
/>
</div>
}
>
<div class="flex items-center justify-center">
<input
type="number"
min={bounds.min}
max={bounds.max}
value={thresholds()?.[metric] ?? ''}
placeholder={isDisabled() ? 'Off' : ''}
ref={(el) => {
if (
isEditing() &&
activeMetricInput()?.resourceId === resource.id &&
activeMetricInput()?.metric === metric
) {
queueMicrotask(() => {
el.focus();
el.select();
});
}
}}
onInput={(e) => {
const raw = e.currentTarget.value;
if (raw === '') {
props.setEditingThresholds({
...props.editingThresholds(),
[metric]: undefined,
});
return;
}
const val = parseInt(raw, 10);
if (!Number.isNaN(val)) {
props.setEditingThresholds({
...props.editingThresholds(),
[metric]: val,
});
}
}}
onBlur={() => {
if (props.editingId() === resource.id) {
props.onSaveEdit(resource.id);
}
setActiveMetricInput(null);
}}
class={`w-16 px-2 py-0.5 text-sm text-center border rounded ${
isDisabled()
? 'bg-gray-100 dark:bg-gray-800 text-gray-400 dark:text-gray-600 border-gray-300 dark:border-gray-600'
: 'bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 border-gray-300 dark:border-gray-600'
}`}
/>
</div>
</Show>
</Show>
</Show>
</Show>
</td>
);
}}
@@ -889,32 +932,6 @@ export function ResourceTable(props: ResourceTableProps) {
</td>
</Show>
<Show when={props.showDelayColumn}>
<td class="p-1 px-2 text-center align-middle">
{(() => {
const delay = resourceDelaySeconds(resource);
const label = formatDelayLabel(delay);
const custom = isCustomDelay(resource);
const title = custom
? 'Custom delay applied to this resource'
: 'Using global alert delay';
return (
<span
class={`text-sm ${
custom
? 'font-medium text-blue-600 dark:text-blue-300'
: 'text-gray-600 dark:text-gray-300'
}`}
title={title}
>
{label}
</span>
);
})()}
</td>
</Show>
{/* Actions column */}
<td class="p-1 px-2">
<div class="flex items-center justify-center gap-1">
@@ -1310,32 +1327,6 @@ export function ResourceTable(props: ResourceTableProps) {
</td>
</Show>
<Show when={props.showDelayColumn}>
<td class="p-1 px-2 text-center align-middle">
{(() => {
const delay = resourceDelaySeconds(resource);
const label = formatDelayLabel(delay);
const custom = isCustomDelay(resource);
const title = custom
? 'Custom delay applied to this resource'
: 'Using global alert delay';
return (
<span
class={`text-sm ${
custom
? 'font-medium text-blue-600 dark:text-blue-300'
: 'text-gray-600 dark:text-gray-300'
}`}
title={title}
>
{label}
</span>
);
})()}
</td>
</Show>
{/* Actions column */}
<td class="p-1 px-2">
<div class="flex items-center justify-center gap-1">
@@ -91,7 +91,12 @@ interface ThresholdsTableProps {
storageDefault: () => number;
setStorageDefault: (value: number) => void;
timeThresholds: () => { guest: number; node: number; storage: number; pbs: number };
setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void;
metricTimeThresholds: () => Record<string, Record<string, number>>;
setMetricTimeThresholds: (
value:
| Record<string, Record<string, number>>
| ((prev: Record<string, Record<string, number>>) => Record<string, Record<string, number>>),
) => void;
setHasUnsavedChanges: (value: boolean) => void;
activeAlerts?: Record<string, Alert>;
removeAlerts?: (predicate: (alert: Alert) => boolean) => void;
@@ -130,30 +135,37 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
// Set up keyboard shortcuts
onMount(() => {
const handleKeyDown = (e: KeyboardEvent) => {
// Skip if user is typing in an input or textarea (unless it's Escape)
const target = e.target as HTMLElement;
const isInInput =
target.tagName === 'INPUT' ||
target.tagName === 'TEXTAREA' ||
target.contentEditable === 'true';
const isEditableElement = (el: HTMLElement | null | undefined): boolean => {
if (!el) return false;
const tag = el.tagName;
return (
tag === 'INPUT' ||
tag === 'TEXTAREA' ||
tag === 'SELECT' ||
el.contentEditable === 'true'
);
};
const handleKeyDown = (e: KeyboardEvent) => {
const target = e.target as HTMLElement | null;
const activeElement = (document.activeElement as HTMLElement) ?? null;
const inEditable = isEditableElement(target);
// Escape clears search from anywhere
if (e.key === 'Escape') {
e.preventDefault();
setSearchTerm('');
if (searchInputRef && isInInput) {
if (searchTerm()) {
e.preventDefault();
setSearchTerm('');
}
if (searchInputRef && document.activeElement === searchInputRef) {
searchInputRef.blur();
}
return;
}
// Skip other shortcuts if already in an input
if (isInInput) {
if (e.defaultPrevented || inEditable || isEditableElement(activeElement) || editingId()) {
return;
}
// Any letter/number focuses search and starts typing
if (e.key.length === 1 && e.key.match(/[a-z0-9]/i)) {
e.preventDefault();
if (searchInputRef) {
@@ -293,6 +305,30 @@ const buildNodeHeaderMeta = (node: Node) => {
return { headerMeta, keys };
};
const overrideHasContent = (override: Override): boolean => {
const hasThresholds = Object.values(override.thresholds || {}).some(
(value) => value !== undefined,
);
const hasStateFlags = Boolean(
override.disabled || override.disableConnectivity || override.poweredOffSeverity,
);
return hasThresholds || hasStateFlags;
};
const buildOverrideFromResource = (resource: Resource): Override => ({
id: resource.id,
name: resource.name,
type: resource.type as OverrideType,
resourceType: resource.resourceType,
vmid: (resource as unknown as { vmid?: number }).vmid,
node: (resource as unknown as { node?: string }).node,
instance: (resource as unknown as { instance?: string }).instance,
disabled: resource.disabled,
disableConnectivity: resource.disableConnectivity,
poweredOffSeverity: resource.poweredOffSeverity,
thresholds: resource.thresholds ? { ...resource.thresholds } : {},
});
const nodesWithOverrides = createMemo<Resource[]>((prev = []) => {
// If we're currently editing, return the previous value to avoid re-renders
if (editingId()) {
@@ -316,6 +352,7 @@ const buildNodeHeaderMeta = (node: Node) => {
);
});
const originalDisplayName = node.displayName?.trim() || node.name;
const friendlyName = getFriendlyNodeName(originalDisplayName, node.clusterName);
const rawName = node.name;
@@ -341,8 +378,8 @@ const buildNodeHeaderMeta = (node: Node) => {
hasOverride: hasCustomThresholds || false,
disabled: false,
disableConnectivity: override?.disableConnectivity || false,
thresholds: override?.thresholds || {},
defaults: props.nodeDefaults,
thresholds: override?.thresholds || {},
defaults: props.nodeDefaults,
clusterName: node.isClusterMember ? node.clusterName?.trim() : undefined,
isClusterMember: node.isClusterMember ?? false,
instance: node.instance,
@@ -465,6 +502,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
);
});
const hasOverride =
hasCustomThresholds || override?.disabled || override?.disableConnectivity || overrideSeverity !== undefined || false;
@@ -621,6 +659,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
);
});
// A guest has an override if it has custom thresholds OR is disabled OR has connectivity disabled
const hasOverride =
hasCustomThresholds || override?.disabled || override?.disableConnectivity || overrideSeverity !== undefined || false;
@@ -720,6 +759,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
);
});
return {
id: pbsId,
name: pbs.name,
@@ -897,6 +937,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
const editedThresholds = editingThresholds();
const defaultThresholds = (resource.defaults ?? {}) as Record<string, number | undefined>;
const existingOverride = props.overrides().find((o) => o.id === resourceId);
// Only include values that differ from defaults
const overrideThresholds: Record<string, number> = {};
@@ -1012,12 +1053,47 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
setEditingThresholds({});
};
const updateDelay = (key: 'guest' | 'node' | 'storage' | 'pbs', value: number) => {
const sanitized = Number.isFinite(value) ? Math.max(0, Math.round(value)) : 0;
const current = props.timeThresholds();
if (current[key] === sanitized) return;
props.setTimeThresholds({ ...current, [key]: sanitized });
props.setHasUnsavedChanges(true);
const updateMetricDelay = (typeKey: 'guest' | 'node' | 'storage' | 'pbs', metricKey: string, value: number | null) => {
const normalizedMetric = metricKey.trim().toLowerCase();
if (!normalizedMetric) return;
let changed = false;
props.setMetricTimeThresholds((prev) => {
const current = prev ? { ...prev } : {};
const existing = prev?.[typeKey];
const typeOverrides = existing ? { ...existing } : {};
if (value === null) {
if (typeOverrides[normalizedMetric] === undefined) {
return prev;
}
delete typeOverrides[normalizedMetric];
changed = true;
} else {
const sanitized = Math.max(0, Math.round(value));
if (typeOverrides[normalizedMetric] === sanitized) {
return prev;
}
typeOverrides[normalizedMetric] = sanitized;
changed = true;
}
if (!changed) {
return prev;
}
if (Object.keys(typeOverrides).length === 0) {
delete current[typeKey];
} else {
current[typeKey] = typeOverrides;
}
return current;
});
if (changed) {
props.setHasUnsavedChanges(true);
}
};
const removeOverride = (resourceId: string) => {
@@ -1052,6 +1128,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
// Get existing override if it exists
const existingOverride = props.overrides().find((o) => o.id === resourceId);
const existingRaw = props.rawOverridesConfig()[resourceId];
// Determine the current disabled state - check the resource's current state, not the override
const currentDisabledState = resource.disabled;
@@ -1062,7 +1139,10 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
delete (cleanThresholds as Record<string, unknown>).disabled;
// If enabling (disabled = false) and no custom thresholds exist, remove the override entirely
if (!newDisabledState && (!existingOverride || Object.keys(cleanThresholds).length === 0)) {
if (
!newDisabledState &&
(!existingOverride || Object.keys(cleanThresholds).length === 0)
) {
// Remove the override completely
props.setOverrides(props.overrides().filter((o) => o.id !== resourceId));
@@ -1108,9 +1188,15 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
if (newDisabledState) {
hysteresisThresholds.disabled = true;
} else {
delete hysteresisThresholds.disabled;
}
newRawConfig[resourceId] = hysteresisThresholds;
if (Object.keys(hysteresisThresholds).length === 0) {
delete newRawConfig[resourceId];
} else {
newRawConfig[resourceId] = hysteresisThresholds;
}
props.setRawOverridesConfig(newRawConfig);
}
@@ -1158,6 +1244,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
// Get existing override if it exists
const existingOverride = props.overrides().find((o) => o.id === resourceId);
const existingRaw = props.rawOverridesConfig()[resourceId];
// Determine the current state - use the resource's computed state, not just the override
const currentDisableConnectivity = resource.disableConnectivity;
@@ -1215,9 +1302,15 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
if (newDisableConnectivity) {
hysteresisThresholds.disableConnectivity = true;
} else {
delete hysteresisThresholds.disableConnectivity;
}
newRawConfig[resourceId] = hysteresisThresholds;
if (Object.keys(hysteresisThresholds).length === 0) {
delete newRawConfig[resourceId];
} else {
newRawConfig[resourceId] = hysteresisThresholds;
}
props.setRawOverridesConfig(newRawConfig);
}
@@ -1242,6 +1335,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
const defaultSeverity = props.guestPoweredOffSeverity();
const existingOverride = props.overrides().find((o) => o.id === resourceId);
const existingRaw = props.rawOverridesConfig()[resourceId];
const cleanThresholds: Record<string, number> = { ...(existingOverride?.thresholds || {}) };
delete (cleanThresholds as Record<string, unknown>).disabled;
delete (cleanThresholds as Record<string, unknown>).disableConnectivity;
@@ -1258,7 +1352,12 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
newDisableConnectivity !== defaultDisabled ||
(!newDisableConnectivity && newSeverity !== defaultSeverity);
if (!differsFromDefaults && !hasThresholds && !overrideDisabled && !existingOverride?.disableConnectivity) {
if (
!differsFromDefaults &&
!hasThresholds &&
!overrideDisabled &&
!existingOverride?.disableConnectivity
) {
// Remove override entirely
if (existingOverride) {
props.setOverrides(props.overrides().filter((o) => o.id !== resourceId));
@@ -1444,7 +1543,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
onToggleGlobalDisableOffline={() => props.setDisableAllNodesOffline(!props.disableAllNodesOffline())}
showDelayColumn={true}
globalDelaySeconds={props.timeThresholds().node}
onGlobalDelayChange={(value) => updateDelay('node', value)}
metricDelaySeconds={props.metricTimeThresholds().node ?? {}}
onMetricDelayChange={(metric, value) => updateMetricDelay('node', metric, value)}
/>
</div>
</Show>
@@ -1492,7 +1592,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
onSetOfflineState={setOfflineState}
showDelayColumn={true}
globalDelaySeconds={props.timeThresholds().guest}
onGlobalDelayChange={(value) => updateDelay('guest', value)}
metricDelaySeconds={props.metricTimeThresholds().guest ?? {}}
onMetricDelayChange={(metric, value) => updateMetricDelay('guest', metric, value)}
/>
</div>
</Show>
@@ -1531,7 +1632,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
onToggleGlobalDisable={() => props.setDisableAllStorage(!props.disableAllStorage())}
showDelayColumn={true}
globalDelaySeconds={props.timeThresholds().storage}
onGlobalDelayChange={(value) => updateDelay('storage', value)}
metricDelaySeconds={props.metricTimeThresholds().storage ?? {}}
onMetricDelayChange={(metric, value) => updateMetricDelay('storage', metric, value)}
/>
</div>
</Show>
@@ -1572,7 +1674,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
onToggleGlobalDisableOffline={() => props.setDisableAllPBSOffline(!props.disableAllPBSOffline())}
showDelayColumn={true}
globalDelaySeconds={props.timeThresholds().pbs}
onGlobalDelayChange={(value) => updateDelay('pbs', value)}
metricDelaySeconds={props.metricTimeThresholds().pbs ?? {}}
onMetricDelayChange={(metric, value) => updateMetricDelay('pbs', metric, value)}
/>
</div>
</Show>
@@ -1675,7 +1778,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
}
showDelayColumn={true}
globalDelaySeconds={props.timeThresholds().guest}
onGlobalDelayChange={(value) => updateDelay('guest', value)}
metricDelaySeconds={props.metricTimeThresholds().guest ?? {}}
onMetricDelayChange={(metric, value) => updateMetricDelay('guest', metric, value)}
globalOfflineSeverity={props.guestPoweredOffSeverity()}
onSetOfflineState={setOfflineState}
/>
+360 -147
View File
@@ -163,11 +163,40 @@ const createDefaultGrouping = (): GroupingConfig => ({
byGuest: false,
});
const normalizeMetricDelayMap = (
input: Record<string, Record<string, number>> | undefined | null,
): Record<string, Record<string, number>> => {
if (!input) return {};
const normalized: Record<string, Record<string, number>> = {};
Object.entries(input).forEach(([rawType, metrics]) => {
if (!metrics) return;
const typeKey = rawType.trim().toLowerCase();
if (!typeKey) return;
const entries: Record<string, number> = {};
Object.entries(metrics).forEach(([rawMetric, value]) => {
if (typeof value !== 'number' || Number.isNaN(value) || value < 0) return;
const metricKey = rawMetric.trim().toLowerCase();
if (!metricKey) return;
entries[metricKey] = Math.round(value);
});
if (Object.keys(entries).length > 0) {
normalized[typeKey] = entries;
}
});
return normalized;
};
const createDefaultEscalation = (): EscalationConfig => ({
enabled: false,
levels: [],
});
const DEFAULT_DELAY_SECONDS = 5;
export function Alerts() {
const { state, activeAlerts, updateAlert, removeAlerts } = useWebSocket();
const navigate = useNavigate();
@@ -462,23 +491,23 @@ export function Alerts() {
const container = (state.containers || []).find((g) => g.id === key);
const guest = vm || container;
if (guest) {
overridesList.push({
id: key,
name: guest.name,
type: 'guest',
resourceType: guest.type === 'qemu' ? 'VM' : 'CT',
vmid: guest.vmid,
node: guest.node,
instance: guest.instance,
disabled: thresholds.disabled || false,
poweredOffSeverity:
thresholds.poweredOffSeverity === 'critical'
? 'critical'
: thresholds.poweredOffSeverity === 'warning'
? 'warning'
: undefined,
thresholds: extractTriggerValues(thresholds),
});
overridesList.push({
id: key,
name: guest.name,
type: 'guest',
resourceType: guest.type === 'qemu' ? 'VM' : 'CT',
vmid: guest.vmid,
node: guest.node,
instance: guest.instance,
disabled: thresholds.disabled || false,
poweredOffSeverity:
thresholds.poweredOffSeverity === 'critical'
? 'critical'
: thresholds.poweredOffSeverity === 'warning'
? 'warning'
: undefined,
thresholds: extractTriggerValues(thresholds),
});
}
}
}
@@ -507,7 +536,12 @@ export function Alerts() {
(newOverride.type === 'guest' || newOverride.type === 'dockerContainer') &&
(newOverride.poweredOffSeverity ?? null) !==
(existing.poweredOffSeverity ?? null);
return thresholdsChanged || connectivityChanged || disabledChanged || severityChanged;
return (
thresholdsChanged ||
connectivityChanged ||
disabledChanged ||
severityChanged
);
});
if (hasChanged) {
@@ -539,8 +573,14 @@ export function Alerts() {
temperature: 80,
});
setStorageDefault(85);
setTimeThreshold(0);
setTimeThresholds({ guest: 10, node: 15, storage: 30, pbs: 30 });
setTimeThreshold(DEFAULT_DELAY_SECONDS);
setTimeThresholds({
guest: DEFAULT_DELAY_SECONDS,
node: DEFAULT_DELAY_SECONDS,
storage: DEFAULT_DELAY_SECONDS,
pbs: DEFAULT_DELAY_SECONDS,
});
setMetricTimeThresholds({});
setScheduleQuietHours(createDefaultQuietHours());
setScheduleCooldown(createDefaultCooldown());
setScheduleGrouping(createDefaultGrouping());
@@ -613,19 +653,25 @@ export function Alerts() {
}
if (config.timeThresholds) {
setTimeThresholds({
guest: config.timeThresholds.guest ?? 10,
node: config.timeThresholds.node ?? 15,
storage: config.timeThresholds.storage ?? 30,
pbs: config.timeThresholds.pbs ?? 30,
guest: config.timeThresholds.guest ?? DEFAULT_DELAY_SECONDS,
node: config.timeThresholds.node ?? DEFAULT_DELAY_SECONDS,
storage: config.timeThresholds.storage ?? DEFAULT_DELAY_SECONDS,
pbs: config.timeThresholds.pbs ?? DEFAULT_DELAY_SECONDS,
});
} else if (config.timeThreshold !== undefined && config.timeThreshold > 0) {
} else {
const fallback = config.timeThreshold && config.timeThreshold > 0 ? config.timeThreshold : DEFAULT_DELAY_SECONDS;
setTimeThresholds({
guest: config.timeThreshold,
node: config.timeThreshold,
storage: config.timeThreshold,
pbs: config.timeThreshold,
guest: fallback,
node: fallback,
storage: fallback,
pbs: fallback,
});
}
if (config.metricTimeThresholds) {
setMetricTimeThresholds(normalizeMetricDelayMap(config.metricTimeThresholds));
} else {
setMetricTimeThresholds({});
}
// Load global disable flags
setDisableAllNodes(config.disableAllNodes ?? false);
@@ -856,14 +902,16 @@ export function Alerts() {
memoryCriticalPct: 95,
});
const [storageDefault, setStorageDefault] = createSignal(85);
const [timeThreshold, setTimeThreshold] = createSignal(0); // Legacy
const [timeThresholds, setTimeThresholds] = createSignal({
guest: 10,
node: 15,
storage: 30,
pbs: 30,
});
const [storageDefault, setStorageDefault] = createSignal(85);
const [timeThreshold, setTimeThreshold] = createSignal(DEFAULT_DELAY_SECONDS); // Legacy
const [timeThresholds, setTimeThresholds] = createSignal({
guest: DEFAULT_DELAY_SECONDS,
node: DEFAULT_DELAY_SECONDS,
storage: DEFAULT_DELAY_SECONDS,
pbs: DEFAULT_DELAY_SECONDS,
});
const [metricTimeThresholds, setMetricTimeThresholds] =
createSignal<Record<string, Record<string, number>>>({});
// Global disable flags per resource type
const [disableAllNodes, setDisableAllNodes] = createSignal(false);
@@ -999,6 +1047,7 @@ export function Alerts() {
hysteresisMargin: 5.0,
timeThreshold: timeThreshold() || 0, // Legacy
timeThresholds: timeThresholds(),
metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()),
// Use rawOverridesConfig which is already properly formatted with disabled flags
overrides: rawOverridesConfig(),
schedule: {
@@ -1197,7 +1246,8 @@ export function Alerts() {
storageDefault={storageDefault}
setStorageDefault={setStorageDefault}
timeThresholds={timeThresholds}
setTimeThresholds={setTimeThresholds}
metricTimeThresholds={metricTimeThresholds}
setMetricTimeThresholds={setMetricTimeThresholds}
activeAlerts={activeAlerts}
setHasUnsavedChanges={setHasUnsavedChanges}
hasUnsavedChanges={hasUnsavedChanges}
@@ -1669,6 +1719,7 @@ interface ThresholdsTabProps {
dockerDefaults: () => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number };
storageDefault: () => number;
timeThresholds: () => { guest: number; node: number; storage: number; pbs: number };
metricTimeThresholds: () => Record<string, Record<string, number>>;
overrides: () => Override[];
rawOverridesConfig: () => Record<string, RawOverrideConfig>;
setGuestDefaults: (
@@ -1689,7 +1740,11 @@ interface ThresholdsTabProps {
value: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number } | ((prev: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }) => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }),
) => void;
setStorageDefault: (value: number) => void;
setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void;
setMetricTimeThresholds: (
value:
| Record<string, Record<string, number>>
| ((prev: Record<string, Record<string, number>>) => Record<string, Record<string, number>>),
) => void;
setOverrides: (value: Override[]) => void;
setRawOverridesConfig: (value: Record<string, RawOverrideConfig>) => void;
activeAlerts: Record<string, Alert>;
@@ -1746,7 +1801,8 @@ function ThresholdsTab(props: ThresholdsTabProps) {
storageDefault={props.storageDefault}
setStorageDefault={props.setStorageDefault}
timeThresholds={props.timeThresholds}
setTimeThresholds={props.setTimeThresholds}
metricTimeThresholds={props.metricTimeThresholds}
setMetricTimeThresholds={props.setMetricTimeThresholds}
setHasUnsavedChanges={props.setHasUnsavedChanges}
activeAlerts={props.activeAlerts}
removeAlerts={props.removeAlerts}
@@ -2627,6 +2683,7 @@ function HistoryTab() {
const [alertHistory, setAlertHistory] = createSignal<Alert[]>([]);
const [loading, setLoading] = createSignal(true);
const [selectedBarIndex, setSelectedBarIndex] = createSignal<number | null>(null);
const MS_PER_HOUR = 60 * 60 * 1000;
// Ref for search input
let searchInputRef: HTMLInputElement | undefined;
@@ -2640,6 +2697,25 @@ function HistoryTab() {
localStorage.setItem('alertHistorySeverityFilter', severityFilter());
});
// Clear chart selection when high-level filters change
let lastTimeFilterValue: string | null = null;
createEffect(() => {
const current = timeFilter();
if (lastTimeFilterValue !== null && current !== lastTimeFilterValue) {
setSelectedBarIndex(null);
}
lastTimeFilterValue = current;
});
let lastSeverityFilterValue: string | null = null;
createEffect(() => {
const current = severityFilter();
if (lastSeverityFilterValue !== null && current !== lastSeverityFilterValue) {
setSelectedBarIndex(null);
}
lastSeverityFilterValue = current;
});
// Load alert history on mount
onMount(async () => {
try {
@@ -2703,8 +2779,54 @@ function HistoryTab() {
return `${minutes}m`;
};
// Get resource type (VM, CT, Node, Storage)
const getResourceType = (resourceName: string) => {
const formatBucketRange = (startMs: number, endMs: number) => {
const start = new Date(startMs);
const end = new Date(endMs);
const sameDay =
start.getFullYear() === end.getFullYear() &&
start.getMonth() === end.getMonth() &&
start.getDate() === end.getDate();
const startDay = start.toLocaleDateString('en-US', {
month: 'short',
day: 'numeric',
year: start.getFullYear() !== end.getFullYear() ? 'numeric' : undefined,
});
const endDay = end.toLocaleDateString('en-US', {
month: 'short',
day: 'numeric',
year: 'numeric',
});
const timeFormatter: Intl.DateTimeFormatOptions = {
hour: 'numeric',
minute: '2-digit',
};
const startTimeStr = start.toLocaleTimeString('en-US', timeFormatter);
const endTimeStr = end.toLocaleTimeString('en-US', timeFormatter);
if (sameDay) {
return `${startDay}, ${startTimeStr} ${endTimeStr}`;
}
return `${startDay}, ${startTimeStr}${endDay}, ${endTimeStr}`;
};
// Get resource type (VM, CT, Node, Storage, Docker, PBS, etc.)
const getResourceType = (
resourceName: string,
metadata?: Record<string, unknown> | undefined,
) => {
const metadataType =
typeof metadata?.resourceType === 'string'
? (metadata.resourceType as string)
: undefined;
if (metadataType && metadataType.trim().length > 0) {
return metadataType;
}
// Check VMs and containers
const vm = state.vms?.find((v) => v.name === resourceName);
if (vm) return 'VM';
@@ -2720,6 +2842,34 @@ function HistoryTab() {
const storage = state.storage?.find((s) => s.name === resourceName || s.id === resourceName);
if (storage) return 'Storage';
// Docker hosts
const dockerHost = state.dockerHosts?.find(
(host) =>
host.displayName === resourceName ||
host.hostname === resourceName ||
host.agentId === resourceName ||
host.id === resourceName,
);
if (dockerHost) return 'Docker Host';
// Docker containers (via known hosts)
const dockerContainer = state.dockerHosts
?.flatMap((host) => host.containers || [])
.find((c) => c.name === resourceName || c.id === resourceName);
if (dockerContainer) return 'Docker Container';
// PBS instances
const pbsInstance = state.pbs?.find(
(pbs) => pbs.name === resourceName || pbs.host === resourceName || pbs.id === resourceName,
);
if (pbsInstance) return 'PBS';
// Ceph clusters
const cephCluster = state.cephClusters?.find(
(cluster) => cluster.name === resourceName || cluster.id === resourceName,
);
if (cephCluster) return 'Ceph';
return 'Unknown';
};
@@ -2741,7 +2891,7 @@ function HistoryTab() {
...alert,
status: 'active',
duration: formatDuration(alert.startTime),
resourceType: getResourceType(alert.resourceName),
resourceType: getResourceType(alert.resourceName, alert.metadata),
});
});
@@ -2759,63 +2909,67 @@ function HistoryTab() {
...alert,
status: alert.acknowledged ? 'acknowledged' : 'resolved',
duration: formatDuration(alert.startTime, alert.lastSeen),
resourceType: getResourceType(alert.resourceName),
resourceType: getResourceType(alert.resourceName, alert.metadata),
});
});
return allAlerts;
});
// Apply severity & search filters (time filtering is layered separately)
const severityAndSearchFilteredAlerts = createMemo(() => {
let filtered = allAlertsData();
if (severityFilter() !== 'all') {
filtered = filtered.filter((a) => a.level === severityFilter());
}
if (searchTerm()) {
const term = searchTerm().toLowerCase();
filtered = filtered.filter((alert) => {
const name = alert.resourceName?.toLowerCase() ?? '';
const message = alert.message?.toLowerCase() ?? '';
const type = alert.type?.toLowerCase() ?? '';
const nodeName = alert.node?.toLowerCase() ?? '';
return (
name.includes(term) || message.includes(term) || type.includes(term) || nodeName.includes(term)
);
});
}
return filtered;
});
// Apply filters to get the final alert data
const alertData = createMemo(() => {
let filtered = allAlertsData();
let filtered = severityAndSearchFilteredAlerts();
// Selected bar filter (takes precedence over time filter)
if (selectedBarIndex() !== null) {
const trends = alertTrends();
const index = selectedBarIndex()!;
const bucketStart = trends.bucketTimes[index];
const bucketEnd = bucketStart + trends.bucketSize * 60 * 60 * 1000;
const bucketEnd = bucketStart + trends.bucketSize * MS_PER_HOUR;
filtered = filtered.filter((alert) => {
const alertTime = new Date(alert.startTime).getTime();
return alertTime >= bucketStart && alertTime < bucketEnd;
});
} else {
// Time filter
if (timeFilter() !== 'all') {
const now = Date.now();
const cutoff = {
'24h': now - 24 * 60 * 60 * 1000,
'7d': now - 7 * 24 * 60 * 60 * 1000,
'30d': now - 30 * 24 * 60 * 60 * 1000,
}[timeFilter()];
} else if (timeFilter() !== 'all') {
const now = Date.now();
const cutoff = {
'24h': now - 24 * 60 * 60 * 1000,
'7d': now - 7 * 24 * 60 * 60 * 1000,
'30d': now - 30 * 24 * 60 * 60 * 1000,
}[timeFilter()];
if (cutoff) {
filtered = filtered.filter((a) => new Date(a.startTime).getTime() > cutoff);
}
if (cutoff) {
filtered = filtered.filter((a) => new Date(a.startTime).getTime() > cutoff);
}
}
// Severity filter
if (severityFilter() !== 'all') {
filtered = filtered.filter((a) => a.level === severityFilter());
}
// Search filter
if (searchTerm()) {
const term = searchTerm().toLowerCase();
filtered = filtered.filter(
(alert) =>
alert.resourceName.toLowerCase().includes(term) ||
alert.message.toLowerCase().includes(term) ||
alert.type.toLowerCase().includes(term) ||
alert.node.toLowerCase().includes(term),
);
}
// Sort by start time (newest first)
return filtered.sort(
return [...filtered].sort(
(a, b) => new Date(b.startTime).getTime() - new Date(a.startTime).getTime(),
);
});
@@ -2919,60 +3073,114 @@ function HistoryTab() {
// Calculate alert trends for mini-chart
const alertTrends = createMemo(() => {
const now = Date.now();
const timeRange =
timeFilter() === '24h'
? 24
: timeFilter() === '7d'
? 7 * 24
: timeFilter() === '30d'
? 30 * 24
: 90 * 24; // hours
const bucketSize =
timeFilter() === '24h' ? 1 : timeFilter() === '7d' ? 6 : timeFilter() === '30d' ? 24 : 72; // hours per bucket
const numBuckets = Math.min(Math.floor(timeRange / bucketSize), 30); // Limit to 30 buckets max
const msPerHour = MS_PER_HOUR;
const filteredAlerts = severityAndSearchFilteredAlerts();
const niceBucketSizes = [1, 2, 3, 6, 12, 24, 48, 72, 168, 336, 720, 1440]; // hours
const maxBuckets = 30;
// Calculate start time for the chart
const startTime = now - timeRange * 60 * 60 * 1000;
let bucketSizeHours: number;
let computedRangeHours: number;
let startTime: number;
// Initialize buckets
const buckets = new Array(numBuckets).fill(0);
// bucketTimes represents the START of each bucket
const bucketTimes = new Array(numBuckets)
.fill(0)
.map((_, i) => startTime + i * bucketSize * 60 * 60 * 1000);
// Filter alerts based on current time filter
let alertsToCount = allAlertsData();
if (timeFilter() !== 'all') {
const cutoff = {
'24h': now - 24 * 60 * 60 * 1000,
'7d': now - 7 * 24 * 60 * 60 * 1000,
'30d': now - 30 * 24 * 60 * 60 * 1000,
}[timeFilter()];
if (cutoff) {
alertsToCount = alertsToCount.filter((a) => new Date(a.startTime).getTime() > cutoff);
const filter = timeFilter();
if (filter === '24h') {
bucketSizeHours = 1;
computedRangeHours = 24;
startTime = now - computedRangeHours * msPerHour;
} else if (filter === '7d') {
bucketSizeHours = 6;
computedRangeHours = 7 * 24;
startTime = now - computedRangeHours * msPerHour;
} else if (filter === '30d') {
bucketSizeHours = 24;
computedRangeHours = 30 * 24;
startTime = now - computedRangeHours * msPerHour;
} else {
if (!filteredAlerts.length) {
bucketSizeHours = 24;
computedRangeHours = 24;
startTime = now - computedRangeHours * msPerHour;
} else {
const earliest = filteredAlerts.reduce((min, alert) => {
const alertTime = new Date(alert.startTime).getTime();
return Math.min(min, alertTime);
}, now);
const rawRangeHours = Math.max(1, Math.ceil((now - earliest) / msPerHour));
const rawBucketSize = Math.max(1, Math.ceil(rawRangeHours / maxBuckets));
bucketSizeHours =
niceBucketSizes.find((size) => size >= rawBucketSize) ?? rawBucketSize;
computedRangeHours = Math.max(rawRangeHours, bucketSizeHours);
const bucketsNeeded = Math.min(
Math.max(1, Math.ceil(computedRangeHours / bucketSizeHours)),
maxBuckets,
);
startTime = now - bucketsNeeded * bucketSizeHours * msPerHour;
}
}
alertsToCount.forEach((alert) => {
const bucketCount = Math.min(
Math.max(1, Math.ceil(computedRangeHours / bucketSizeHours)),
maxBuckets,
);
startTime = Math.min(startTime, now - bucketCount * bucketSizeHours * msPerHour);
const buckets = new Array(bucketCount).fill(0);
const bucketTimes = new Array(bucketCount)
.fill(0)
.map((_, i) => startTime + i * bucketSizeHours * msPerHour);
const windowStart = startTime;
const windowEnd = now;
filteredAlerts.forEach((alert) => {
const alertTime = new Date(alert.startTime).getTime();
if (alertTime >= startTime && alertTime <= now) {
const bucketIndex = Math.floor((alertTime - startTime) / (bucketSize * 60 * 60 * 1000));
if (bucketIndex >= 0 && bucketIndex < numBuckets) {
buckets[bucketIndex]++;
}
if (alertTime < windowStart || alertTime > windowEnd) {
return;
}
const rawIndex = Math.floor((alertTime - windowStart) / (bucketSizeHours * msPerHour));
const bucketIndex = Math.min(bucketCount - 1, Math.max(0, rawIndex));
if (bucketIndex >= 0 && bucketIndex < bucketCount) {
buckets[bucketIndex]++;
}
});
// Find max for scaling
const max = Math.max(...buckets, 1);
return {
buckets,
max,
bucketSize,
bucketSize: bucketSizeHours,
bucketTimes,
rangeStart: windowStart,
rangeHours: bucketCount * bucketSizeHours,
};
});
const chartRangeLabel = createMemo(() => {
const filter = timeFilter();
if (filter === '24h') return '24h ago';
if (filter === '7d') return '7d ago';
if (filter === '30d') return '30d ago';
const rangeHours = alertTrends().rangeHours ?? 0;
if (rangeHours <= 0) return '—';
if (rangeHours >= 24) {
const days = Math.round(rangeHours / 24);
return `${days}d ago`;
}
return `${Math.round(rangeHours)}h ago`;
});
const selectedBucketDetails = createMemo(() => {
const index = selectedBarIndex();
if (index === null) return null;
const trends = alertTrends();
const bucketStart = trends.bucketTimes[index];
const bucketEnd = bucketStart + trends.bucketSize * MS_PER_HOUR;
return {
rangeLabel: formatBucketRange(bucketStart, bucketEnd),
start: bucketStart,
end: bucketEnd,
};
});
@@ -2980,7 +3188,7 @@ function HistoryTab() {
<div class="space-y-4">
{/* Alert Trends Mini-Chart */}
<Card padding="md">
<div class="mb-3 flex items-start justify-between gap-3">
<div class="mb-3 flex flex-col gap-2 sm:flex-row sm:items-start sm:justify-between sm:gap-3">
<SectionHeader
title="Alert frequency"
description={
@@ -2991,25 +3199,37 @@ function HistoryTab() {
size="sm"
class="flex-1"
/>
<div class="flex items-center gap-2">
<Show when={selectedBarIndex() !== null}>
<button
type="button"
onClick={() => setSelectedBarIndex(null)}
class="px-2 py-0.5 text-xs bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded hover:bg-blue-200 dark:hover:bg-blue-800/50 transition-colors"
>
Clear filter
</button>
<div class="flex flex-col items-start gap-2 sm:items-end">
<Show when={selectedBucketDetails()}>
{(selection) => (
<div class="inline-flex items-center gap-2 rounded-full border border-blue-200 dark:border-blue-700 bg-blue-50 dark:bg-blue-900/30 px-3 py-1 text-xs text-blue-700 dark:text-blue-200">
<span class="font-medium uppercase tracking-wide text-[10px] text-blue-600 dark:text-blue-300">
Filtered Range
</span>
<span class="font-mono text-[11px]">{selection().rangeLabel}</span>
</div>
)}
</Show>
<div class="flex items-center gap-2 text-xs text-gray-500 dark:text-gray-400">
<span class="flex items-center gap-1">
<div class="w-2 h-2 bg-yellow-500 rounded-full"></div>
{alertData().filter((a) => a.level === 'warning').length} warnings
</span>
<span class="flex items-center gap-1">
<div class="w-2 h-2 bg-red-500 rounded-full"></div>
{alertData().filter((a) => a.level === 'critical').length} critical
</span>
<div class="flex flex-wrap items-center justify-end gap-2">
<Show when={selectedBarIndex() !== null}>
<button
type="button"
onClick={() => setSelectedBarIndex(null)}
class="px-2 py-0.5 text-xs bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded hover:bg-blue-200 dark:hover:bg-blue-800/50 transition-colors"
>
Clear filter
</button>
</Show>
<div class="flex items-center gap-2 text-xs text-gray-500 dark:text-gray-400">
<span class="flex items-center gap-1">
<div class="h-2 w-2 rounded-full bg-yellow-500"></div>
{alertData().filter((a) => a.level === 'warning').length} warnings
</span>
<span class="flex items-center gap-1">
<div class="h-2 w-2 rounded-full bg-red-500"></div>
{alertData().filter((a) => a.level === 'critical').length} critical
</span>
</div>
</div>
</div>
</div>
@@ -3049,11 +3269,10 @@ function HistoryTab() {
}
const rect = (e.currentTarget as HTMLElement).getBoundingClientRect();
const bucketHours = alertTrends().bucketSize;
const bucketLabel = (() => {
if (timeFilter() === '24h') return `${bucketHours} hour period`;
const bucketDays = bucketHours / 24;
return `${bucketDays} day period`;
})();
const bucketLabel =
bucketHours % 24 === 0
? `${bucketHours / 24} day${bucketHours / 24 === 1 ? '' : 's'}`
: `${bucketHours} hour${bucketHours === 1 ? '' : 's'}`;
const timestamp = new Date(alertTrends().bucketTimes[i]).toLocaleString('en-US', {
month: 'short',
day: 'numeric',
@@ -3062,7 +3281,7 @@ function HistoryTab() {
});
const content = [
`${val} alert${val !== 1 ? 's' : ''}`,
bucketLabel,
`${bucketLabel} period`,
timestamp,
].join('\n');
showTooltip(content, rect.left + rect.width / 2, rect.top, {
@@ -3080,13 +3299,7 @@ function HistoryTab() {
{/* Time labels */}
<div class="flex justify-between mt-1 text-[10px] text-gray-400 dark:text-gray-500">
<span>
{timeFilter() === '24h'
? '24h ago'
: timeFilter() === '7d'
? '7d ago'
: timeFilter() === '30d'
? '30d ago'
: '90d ago'}
{chartRangeLabel()}
</span>
<span>Now</span>
</div>
+1
View File
@@ -84,6 +84,7 @@ export interface AlertConfig {
storage?: number;
pbs?: number;
};
metricTimeThresholds?: Record<string, Record<string, number>>;
aggregation?: {
enabled: boolean;
timeWindow: number;
+214 -39
View File
@@ -276,11 +276,12 @@ type AlertConfig struct {
DisableAllPBSOffline bool `json:"disableAllPBSOffline"` // Disable PBS offline alerts globally
DisableAllDockerHostsOffline bool `json:"disableAllDockerHostsOffline"` // Disable Docker host offline alerts globally
// New configuration options
MinimumDelta float64 `json:"minimumDelta"` // Minimum % change to trigger new alert
SuppressionWindow int `json:"suppressionWindow"` // Minutes to suppress duplicate alerts
HysteresisMargin float64 `json:"hysteresisMargin"` // Default margin for legacy thresholds
TimeThreshold int `json:"timeThreshold"` // Legacy: Seconds that threshold must be exceeded before triggering
TimeThresholds map[string]int `json:"timeThresholds"` // Per-type delays: guest, node, storage, pbs
MinimumDelta float64 `json:"minimumDelta"` // Minimum % change to trigger new alert
SuppressionWindow int `json:"suppressionWindow"` // Minutes to suppress duplicate alerts
HysteresisMargin float64 `json:"hysteresisMargin"` // Default margin for legacy thresholds
TimeThreshold int `json:"timeThreshold"` // Legacy: Seconds that threshold must be exceeded before triggering
TimeThresholds map[string]int `json:"timeThresholds"` // Per-type delays: guest, node, storage, pbs
MetricTimeThresholds map[string]map[string]int `json:"metricTimeThresholds"` // Optional per-metric delays keyed by resource type
}
// Manager handles alert monitoring and state
@@ -388,11 +389,12 @@ func NewManager() *Manager {
MinimumDelta: 2.0, // 2% minimum change
SuppressionWindow: 5, // 5 minutes
HysteresisMargin: 5.0, // 5% default margin
TimeThreshold: 5,
TimeThresholds: map[string]int{
"guest": 10, // 10 second delay for guest CPU alerts
"node": 15, // 15 second delay for node alerts
"storage": 30, // 30 second delay for storage alerts
"pbs": 30, // 30 second delay for PBS alerts
"guest": 5,
"node": 5,
"storage": 5,
"pbs": 5,
},
Overrides: make(map[string]ThresholdConfig),
Schedule: ScheduleConfig{
@@ -539,6 +541,39 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
config.HysteresisMargin = 5.0
}
// Ensure temperature defaults exist for nodes so high temps alert out of the box
if config.NodeDefaults.Temperature == nil || config.NodeDefaults.Temperature.Trigger <= 0 {
config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75}
} else if config.NodeDefaults.Temperature.Clear <= 0 {
config.NodeDefaults.Temperature.Clear = config.NodeDefaults.Temperature.Trigger - 5
if config.NodeDefaults.Temperature.Clear <= 0 {
config.NodeDefaults.Temperature.Clear = 75
}
}
// Normalize any metric-level delay overrides
config.MetricTimeThresholds = normalizeMetricTimeThresholds(config.MetricTimeThresholds)
const defaultDelaySeconds = 5
if config.TimeThreshold <= 0 {
config.TimeThreshold = defaultDelaySeconds
}
if config.TimeThresholds == nil {
config.TimeThresholds = make(map[string]int)
}
ensureDelay := func(key string) {
if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 {
config.TimeThresholds[key] = defaultDelaySeconds
}
}
ensureDelay("guest")
ensureDelay("node")
ensureDelay("storage")
ensureDelay("pbs")
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
config.TimeThresholds["all"] = defaultDelaySeconds
}
config.GuestDefaults.PoweredOffSeverity = normalizePoweredOffSeverity(config.GuestDefaults.PoweredOffSeverity)
config.NodeDefaults.PoweredOffSeverity = normalizePoweredOffSeverity(config.NodeDefaults.PoweredOffSeverity)
@@ -563,6 +598,42 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
m.reevaluateActiveAlertsLocked()
}
// normalizeMetricTimeThresholds cleans resource/metric keys and drops invalid delay overrides.
func normalizeMetricTimeThresholds(input map[string]map[string]int) map[string]map[string]int {
if len(input) == 0 {
return nil
}
normalized := make(map[string]map[string]int)
for rawType, metrics := range input {
typeKey := strings.ToLower(strings.TrimSpace(rawType))
if typeKey == "" || len(metrics) == 0 {
continue
}
for rawMetric, delay := range metrics {
metricKey := strings.ToLower(strings.TrimSpace(rawMetric))
if metricKey == "" || delay < 0 {
continue
}
if _, exists := normalized[typeKey]; !exists {
normalized[typeKey] = make(map[string]int)
}
normalized[typeKey][metricKey] = delay
}
}
if len(normalized) == 0 {
return nil
}
return normalized
}
// NormalizeMetricTimeThresholds exposes normalization for other packages (e.g., config persistence).
func NormalizeMetricTimeThresholds(input map[string]map[string]int) map[string]map[string]int {
return normalizeMetricTimeThresholds(input)
}
// applyGlobalOfflineSettingsLocked clears tracking and active alerts for globally disabled offline detectors.
// Caller must hold m.mu.
func (m *Manager) applyGlobalOfflineSettingsLocked() {
@@ -2488,38 +2559,142 @@ func (m *Manager) clearAlert(alertID string) {
Msg("Alert cleared")
}
// getTimeThresholdForType returns the appropriate time threshold for the resource type
func (m *Manager) getTimeThresholdForType(resourceType string) int {
typeKey := strings.ToLower(strings.TrimSpace(resourceType))
// getTimeThreshold determines the delay to apply for a metric/resource combination.
func (m *Manager) getTimeThreshold(_ string, resourceType, metricType string) int {
if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok {
return delay
}
// Use per-type thresholds if available
if m.config.TimeThresholds != nil {
switch typeKey {
case "guest", "qemu", "lxc", "vm", "ct", "container":
if delay, ok := m.config.TimeThresholds["guest"]; ok {
return delay
}
case "docker container", "dockercontainer", "docker":
if delay, ok := m.config.TimeThresholds["guest"]; ok {
return delay
}
case "node":
if delay, ok := m.config.TimeThresholds["node"]; ok {
return delay
}
case "storage":
if delay, ok := m.config.TimeThresholds["storage"]; ok {
return delay
}
case "pbs":
if delay, ok := m.config.TimeThresholds["pbs"]; ok {
return delay
}
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
if !hasTypeSpecific {
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
return delay
}
}
// Fall back to legacy single threshold
return m.config.TimeThreshold
return base
}
// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level.
func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) {
if len(m.config.MetricTimeThresholds) == 0 {
return 0, false
}
metricKey := strings.ToLower(strings.TrimSpace(metricType))
if metricKey == "" {
return 0, false
}
for _, typeKey := range canonicalResourceTypeKeys(resourceType) {
perType, ok := m.config.MetricTimeThresholds[typeKey]
if !ok || len(perType) == 0 {
continue
}
if delay, ok := perType[metricKey]; ok {
return delay, true
}
if delay, ok := perType["default"]; ok {
return delay, true
}
if delay, ok := perType["_default"]; ok {
return delay, true
}
if delay, ok := perType["*"]; ok {
return delay, true
}
}
return 0, false
}
// getBaseTimeThreshold returns the resource-type level delay.
func (m *Manager) getBaseTimeThreshold(resourceType string) (int, bool) {
if m.config.TimeThresholds != nil {
for _, key := range canonicalResourceTypeKeys(resourceType) {
if delay, ok := m.config.TimeThresholds[key]; ok {
return delay, true
}
}
if delay, ok := m.config.TimeThresholds["all"]; ok {
return delay, false
}
}
return m.config.TimeThreshold, false
}
func (m *Manager) getGlobalMetricTimeThreshold(metricType string) (int, bool) {
if len(m.config.MetricTimeThresholds) == 0 {
return 0, false
}
perType, ok := m.config.MetricTimeThresholds["all"]
if !ok || len(perType) == 0 {
return 0, false
}
metricKey := strings.ToLower(strings.TrimSpace(metricType))
if metricKey == "" {
return 0, false
}
if delay, ok := perType[metricKey]; ok {
return delay, true
}
if delay, ok := perType["default"]; ok {
return delay, true
}
if delay, ok := perType["_default"]; ok {
return delay, true
}
if delay, ok := perType["*"]; ok {
return delay, true
}
return 0, false
}
func canonicalResourceTypeKeys(resourceType string) []string {
typeKey := strings.ToLower(strings.TrimSpace(resourceType))
addUnique := func(slice []string, value string) []string {
if value == "" {
return slice
}
for _, existing := range slice {
if existing == value {
return slice
}
}
return append(slice, value)
}
var keys []string
switch typeKey {
case "guest", "qemu", "vm", "ct", "container", "lxc":
keys = addUnique(keys, "guest")
case "docker", "docker container", "dockercontainer":
keys = addUnique(keys, "docker")
keys = addUnique(keys, "guest")
case "docker host", "dockerhost":
keys = addUnique(keys, "dockerhost")
keys = addUnique(keys, "docker")
keys = addUnique(keys, "node")
case "node":
keys = addUnique(keys, "node")
case "pbs", "pbs server", "pbsserver":
keys = addUnique(keys, "pbs")
keys = addUnique(keys, "node")
case "storage":
keys = addUnique(keys, "storage")
default:
keys = addUnique(keys, typeKey)
}
return keys
}
// checkMetric checks a single metric against its threshold with hysteresis
@@ -2563,8 +2738,8 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
if !exists {
alertStartTime := time.Now()
// Determine the appropriate time threshold based on resource type
timeThreshold := m.getTimeThresholdForType(resourceType)
// Determine the appropriate time threshold based on resource/metric type
timeThreshold := m.getTimeThreshold(resourceID, resourceType, metricType)
// Check if we have a time threshold configured
if timeThreshold > 0 {
+48 -3
View File
@@ -5,7 +5,7 @@ import (
"time"
)
func TestGetTimeThresholdForTypeMappings(t *testing.T) {
func TestGetTimeThresholdMappings(t *testing.T) {
manager := NewManager()
manager.mu.Lock()
@@ -35,8 +35,53 @@ func TestGetTimeThresholdForTypeMappings(t *testing.T) {
}
for _, tc := range testCases {
if got := manager.getTimeThresholdForType(tc.resourceType); got != tc.expected {
t.Errorf("getTimeThresholdForType(%q) = %d, want %d", tc.resourceType, got, tc.expected)
if got := manager.getTimeThreshold("", tc.resourceType, "cpu"); got != tc.expected {
t.Errorf("getTimeThreshold(%q, \"cpu\") = %d, want %d", tc.resourceType, got, tc.expected)
}
}
}
func TestGetTimeThresholdMetricOverrides(t *testing.T) {
manager := NewManager()
manager.mu.Lock()
manager.config.TimeThreshold = 15
manager.config.TimeThresholds = map[string]int{
"guest": 30,
"node": 60,
"storage": 90,
}
manager.config.MetricTimeThresholds = map[string]map[string]int{
"guest": {
"cpu": 5,
},
"node": {
"temperature": 120,
},
"all": {
"default": 20,
},
}
manager.mu.Unlock()
cases := []struct {
resourceID string
resourceType string
metricType string
expected int
}{
{"vm-resource", "VM", "cpu", 5}, // guest metric override
{"vm-resource", "VM", "memory", 30}, // falls back to guest type delay
{"node-1", "Node", "temperature", 120}, // node metric override
{"node-1", "Node", "cpu", 60}, // node type delay
{"storage-1", "storage", "usage", 90}, // storage type delay
{"unknown", "unknown", "cpu", 20}, // global default metric override
{"unknown", "unknown", "disk", 20},
}
for _, tc := range cases {
if got := manager.getTimeThreshold(tc.resourceID, tc.resourceType, tc.metricType); got != tc.expected {
t.Errorf("getTimeThreshold(%q, %q, %q) = %d, want %d", tc.resourceID, tc.resourceType, tc.metricType, got, tc.expected)
}
}
}
+53 -4
View File
@@ -103,6 +103,25 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error {
if config.HysteresisMargin <= 0 {
config.HysteresisMargin = 5.0
}
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
if config.TimeThreshold <= 0 {
config.TimeThreshold = 5
}
if config.TimeThresholds == nil {
config.TimeThresholds = make(map[string]int)
}
ensureDelay := func(key string) {
if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 {
config.TimeThresholds[key] = config.TimeThreshold
}
}
ensureDelay("guest")
ensureDelay("node")
ensureDelay("storage")
ensureDelay("pbs")
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
config.TimeThresholds["all"] = config.TimeThreshold
}
data, err := json.MarshalIndent(config, "", " ")
if err != nil {
@@ -138,11 +157,19 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85},
},
NodeDefaults: alerts.ThresholdConfig{
CPU: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85},
CPU: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85},
Temperature: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
},
StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
TimeThreshold: 5,
TimeThresholds: map[string]int{
"guest": 5,
"node": 5,
"storage": 5,
"pbs": 5,
},
StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
MinimumDelta: 2.0,
SuppressionWindow: 5,
HysteresisMargin: 5.0,
@@ -175,6 +202,28 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
if config.HysteresisMargin <= 0 {
config.HysteresisMargin = 5.0
}
if config.NodeDefaults.Temperature == nil || config.NodeDefaults.Temperature.Trigger <= 0 {
config.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
}
if config.TimeThreshold <= 0 {
config.TimeThreshold = 5
}
if config.TimeThresholds == nil {
config.TimeThresholds = make(map[string]int)
}
ensureDelay := func(key string) {
if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 {
config.TimeThresholds[key] = config.TimeThreshold
}
}
ensureDelay("guest")
ensureDelay("node")
ensureDelay("storage")
ensureDelay("pbs")
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
config.TimeThresholds["all"] = config.TimeThreshold
}
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
// Migration: Set I/O metrics to Off (0) if they have the old default values
// This helps existing users avoid noisy I/O alerts