mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-22 03:04:03 +00:00
feat: add per-metric alert delay configuration
Add sophisticated alert delay system with per-metric overrides: - Global delay defaults per resource type (guest/node/storage/pbs) - Per-metric delay overrides with UI controls - Dynamic tooltips showing effective delay values - Clear buttons to reset metric-specific overrides - Temperature threshold defaults for nodes (80°C/75°C) - Improved delay configuration persistence and normalization
This commit is contained in:
@@ -87,7 +87,6 @@ interface ResourceTableProps {
|
||||
onSetOfflineState?: (resourceId: string, state: OfflineState) => void;
|
||||
showDelayColumn?: boolean;
|
||||
globalDelaySeconds?: number;
|
||||
onGlobalDelayChange?: (value: number) => void;
|
||||
editingId: () => string | null;
|
||||
editingThresholds: () => Record<string, number | undefined>;
|
||||
setEditingThresholds: (value: Record<string, number | undefined>) => void;
|
||||
@@ -100,6 +99,8 @@ interface ResourceTableProps {
|
||||
onToggleGlobalDisable?: () => void;
|
||||
globalDisableOfflineFlag?: () => boolean;
|
||||
onToggleGlobalDisableOffline?: () => void;
|
||||
metricDelaySeconds?: Record<string, number>;
|
||||
onMetricDelayChange?: (metricKey: string, value: number | null) => void;
|
||||
groupHeaderMeta?: Record<string, GroupHeaderMeta>;
|
||||
}
|
||||
|
||||
@@ -194,37 +195,21 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||
return 80;
|
||||
};
|
||||
|
||||
const resourceDelaySeconds = (resource: Resource): number => {
|
||||
if (typeof resource.delaySeconds === 'number' && Number.isFinite(resource.delaySeconds)) {
|
||||
return resource.delaySeconds;
|
||||
}
|
||||
if (typeof props.globalDelaySeconds === 'number' && Number.isFinite(props.globalDelaySeconds)) {
|
||||
return props.globalDelaySeconds;
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
|
||||
const formatDelayLabel = (delay: number): string => (delay <= 0 ? 'Instant' : `${delay}s`);
|
||||
|
||||
const isCustomDelay = (resource: Resource): boolean => {
|
||||
if (typeof resource.delaySeconds !== 'number') {
|
||||
return false;
|
||||
const metricDelayOverride = (metric: string): number | undefined => {
|
||||
const normalized = metric.trim().toLowerCase();
|
||||
const value = props.metricDelaySeconds?.[normalized] ?? props.metricDelaySeconds?.[metric];
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
||||
return undefined;
|
||||
}
|
||||
if (!Number.isFinite(resource.delaySeconds)) {
|
||||
return false;
|
||||
}
|
||||
const globalDelay =
|
||||
typeof props.globalDelaySeconds === 'number' && Number.isFinite(props.globalDelaySeconds)
|
||||
? props.globalDelaySeconds
|
||||
: 0;
|
||||
return resource.delaySeconds !== globalDelay;
|
||||
return value;
|
||||
};
|
||||
|
||||
const totalColumnCount = () =>
|
||||
props.columns.length +
|
||||
3 +
|
||||
(props.showOfflineAlertsColumn ? 1 : 0) +
|
||||
(props.showDelayColumn ? 1 : 0);
|
||||
(props.showOfflineAlertsColumn ? 1 : 0);
|
||||
|
||||
const getColumnHeaderTooltip = (column: string): string | undefined => {
|
||||
const normalized = column.trim().toLowerCase();
|
||||
@@ -421,14 +406,6 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||
Offline Alerts
|
||||
</th>
|
||||
</Show>
|
||||
<Show when={props.showDelayColumn}>
|
||||
<th
|
||||
class="px-3 py-2 text-center text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider"
|
||||
title="Pulse waits for the configured alert delay before creating a notification. If the metric drops back below the threshold during this window, the alert is suppressed to reduce noise. Update the delay values using the Alert Delay column in each table."
|
||||
>
|
||||
Alert Delay (s)
|
||||
</th>
|
||||
</Show>
|
||||
<th class="px-3 py-2 text-center text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider">
|
||||
Actions
|
||||
</th>
|
||||
@@ -469,7 +446,7 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||
|
||||
return (
|
||||
<td class="p-1 px-2 text-center align-middle">
|
||||
<div class="relative flex justify-center">
|
||||
<div class="relative flex justify-center w-full">
|
||||
<input
|
||||
type="number"
|
||||
min={bounds.min}
|
||||
@@ -558,22 +535,86 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||
</Show>
|
||||
</td>
|
||||
</Show>
|
||||
<Show when={props.showDelayColumn}>
|
||||
<td class="p-1 px-2 text-center align-middle">
|
||||
<span class="text-sm text-gray-400">-</span>
|
||||
</td>
|
||||
</tr>
|
||||
</Show>
|
||||
<Show when={props.showDelayColumn && typeof props.onMetricDelayChange === 'function'}>
|
||||
<tr class={`bg-gray-50 dark:bg-gray-800/50 border-b border-gray-300 dark:border-gray-600 ${props.globalDisableFlag?.() ? 'opacity-40' : ''}`}>
|
||||
<td class="p-1 px-2 text-center align-middle">
|
||||
<span class="text-sm text-gray-400">-</span>
|
||||
</td>
|
||||
<td class="p-1 px-2 align-middle">
|
||||
<span class="text-xs font-semibold uppercase tracking-wide text-gray-600 dark:text-gray-300">
|
||||
Alert Delay (s)
|
||||
</span>
|
||||
</td>
|
||||
<For each={props.columns}>
|
||||
{(column) => {
|
||||
const metric = normalizeMetricKey(column);
|
||||
const typeDefaultDelay = props.globalDelaySeconds ?? 5;
|
||||
const overrideDelay = metricDelayOverride(metric);
|
||||
|
||||
return (
|
||||
<td class="p-1 px-2 text-center align-middle">
|
||||
<div class="flex items-center justify-center gap-2">
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
value={(() => {
|
||||
return overrideDelay !== undefined ? overrideDelay : '';
|
||||
})()}
|
||||
placeholder={formatDelayLabel(typeDefaultDelay)}
|
||||
class="w-20 rounded border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 px-2 py-0.5 text-xs text-gray-700 dark:text-gray-100 focus:border-blue-500 focus:ring-1 focus:ring-blue-500"
|
||||
onInput={(e) => {
|
||||
const raw = e.currentTarget.value;
|
||||
if (raw === '') {
|
||||
props.onMetricDelayChange?.(metric, null);
|
||||
props.setHasUnsavedChanges?.(true);
|
||||
} else {
|
||||
const parsed = parseInt(raw, 10);
|
||||
if (Number.isNaN(parsed)) {
|
||||
return;
|
||||
}
|
||||
props.onMetricDelayChange?.(metric, Math.max(0, parsed));
|
||||
props.setHasUnsavedChanges?.(true);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<Show when={metricDelayOverride(metric) !== undefined}>
|
||||
<button
|
||||
type="button"
|
||||
class="rounded p-1 text-gray-500 transition hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200"
|
||||
onClick={() => {
|
||||
props.onMetricDelayChange?.(metric, null);
|
||||
props.setHasUnsavedChanges?.(true);
|
||||
}}
|
||||
title="Clear delay override"
|
||||
aria-label="Clear delay override"
|
||||
>
|
||||
<svg
|
||||
class="h-3 w-3"
|
||||
viewBox="0 0 20 20"
|
||||
fill="currentColor"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
fill-rule="evenodd"
|
||||
d="M10 18a8 8 0 100-16 8 8 0 000 16zm2.707-10.707a1 1 0 00-1.414-1.414L10 7.586 8.707 6.293A1 1 0 007.293 7.707L8.586 9l-1.293 1.293a1 1 0 101.414 1.414L10 10.414l1.293 1.293a1 1 0 001.414-1.414L11.414 9l1.293-1.293z"
|
||||
clip-rule="evenodd"
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
</Show>
|
||||
</div>
|
||||
</td>
|
||||
);
|
||||
}}
|
||||
</For>
|
||||
<Show when={props.showOfflineAlertsColumn}>
|
||||
<td class="p-1 px-2 text-center align-middle">
|
||||
<div class="flex justify-center">
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
value={props.globalDelaySeconds ?? 0}
|
||||
onInput={(e) => {
|
||||
const raw = parseInt(e.currentTarget.value, 10);
|
||||
if (Number.isNaN(raw)) return;
|
||||
props.onGlobalDelayChange?.(Math.max(0, raw));
|
||||
props.setHasUnsavedChanges?.(true);
|
||||
}}
|
||||
class="w-20 rounded border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-800 text-sm text-gray-700 dark:text-gray-100 px-2 py-0.5 focus:border-blue-500 focus:ring-1 focus:ring-blue-500"
|
||||
/>
|
||||
</div>
|
||||
<span class="text-sm text-gray-400">-</span>
|
||||
</td>
|
||||
</Show>
|
||||
<td class="p-1 px-2 text-center align-middle">
|
||||
@@ -765,74 +806,76 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||
</span>
|
||||
}
|
||||
>
|
||||
<Show
|
||||
when={isEditing()}
|
||||
fallback={
|
||||
<div
|
||||
onClick={openMetricEditor}
|
||||
class="cursor-pointer hover:bg-gray-100 dark:hover:bg-gray-800 rounded px-1 py-0.5 transition-colors"
|
||||
title="Click to edit this metric"
|
||||
>
|
||||
<MetricValueWithHeat
|
||||
resourceId={resource.id}
|
||||
metric={metric}
|
||||
value={displayValue(metric)}
|
||||
isOverridden={isOverridden(metric)}
|
||||
<Show
|
||||
when={isEditing()}
|
||||
fallback={
|
||||
<div
|
||||
onClick={(event) => {
|
||||
openMetricEditor(event);
|
||||
}}
|
||||
class="cursor-pointer hover:bg-gray-100 dark:hover:bg-gray-800 rounded px-1 py-0.5 transition-colors"
|
||||
title="Click to edit this metric"
|
||||
>
|
||||
<MetricValueWithHeat
|
||||
resourceId={resource.id}
|
||||
metric={metric}
|
||||
value={displayValue(metric)}
|
||||
isOverridden={isOverridden(metric)}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
>
|
||||
<div class="flex items-center justify-center">
|
||||
<input
|
||||
type="number"
|
||||
min={bounds.min}
|
||||
max={bounds.max}
|
||||
value={thresholds()?.[metric] ?? ''}
|
||||
placeholder={isDisabled() ? 'Off' : ''}
|
||||
ref={(el) => {
|
||||
if (
|
||||
isEditing() &&
|
||||
activeMetricInput()?.resourceId === resource.id &&
|
||||
activeMetricInput()?.metric === metric
|
||||
) {
|
||||
queueMicrotask(() => {
|
||||
el.focus();
|
||||
el.select();
|
||||
});
|
||||
}
|
||||
}}
|
||||
onInput={(e) => {
|
||||
const raw = e.currentTarget.value;
|
||||
if (raw === '') {
|
||||
props.setEditingThresholds({
|
||||
...props.editingThresholds(),
|
||||
[metric]: undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const val = parseInt(raw, 10);
|
||||
if (!Number.isNaN(val)) {
|
||||
props.setEditingThresholds({
|
||||
...props.editingThresholds(),
|
||||
[metric]: val,
|
||||
});
|
||||
}
|
||||
}}
|
||||
onBlur={() => {
|
||||
if (props.editingId() === resource.id) {
|
||||
props.onSaveEdit(resource.id);
|
||||
}
|
||||
setActiveMetricInput(null);
|
||||
}}
|
||||
class={`w-16 px-2 py-0.5 text-sm text-center border rounded ${
|
||||
isDisabled()
|
||||
? 'bg-gray-100 dark:bg-gray-800 text-gray-400 dark:text-gray-600 border-gray-300 dark:border-gray-600'
|
||||
: 'bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 border-gray-300 dark:border-gray-600'
|
||||
}`}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
>
|
||||
<div class="flex items-center justify-center">
|
||||
<input
|
||||
type="number"
|
||||
min={bounds.min}
|
||||
max={bounds.max}
|
||||
value={thresholds()?.[metric] ?? ''}
|
||||
placeholder={isDisabled() ? 'Off' : ''}
|
||||
ref={(el) => {
|
||||
if (
|
||||
isEditing() &&
|
||||
activeMetricInput()?.resourceId === resource.id &&
|
||||
activeMetricInput()?.metric === metric
|
||||
) {
|
||||
queueMicrotask(() => {
|
||||
el.focus();
|
||||
el.select();
|
||||
});
|
||||
}
|
||||
}}
|
||||
onInput={(e) => {
|
||||
const raw = e.currentTarget.value;
|
||||
if (raw === '') {
|
||||
props.setEditingThresholds({
|
||||
...props.editingThresholds(),
|
||||
[metric]: undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const val = parseInt(raw, 10);
|
||||
if (!Number.isNaN(val)) {
|
||||
props.setEditingThresholds({
|
||||
...props.editingThresholds(),
|
||||
[metric]: val,
|
||||
});
|
||||
}
|
||||
}}
|
||||
onBlur={() => {
|
||||
if (props.editingId() === resource.id) {
|
||||
props.onSaveEdit(resource.id);
|
||||
}
|
||||
setActiveMetricInput(null);
|
||||
}}
|
||||
class={`w-16 px-2 py-0.5 text-sm text-center border rounded ${
|
||||
isDisabled()
|
||||
? 'bg-gray-100 dark:bg-gray-800 text-gray-400 dark:text-gray-600 border-gray-300 dark:border-gray-600'
|
||||
: 'bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 border-gray-300 dark:border-gray-600'
|
||||
}`}
|
||||
/>
|
||||
</div>
|
||||
</Show>
|
||||
</Show>
|
||||
</Show>
|
||||
</Show>
|
||||
</td>
|
||||
);
|
||||
}}
|
||||
@@ -889,32 +932,6 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||
</td>
|
||||
</Show>
|
||||
|
||||
<Show when={props.showDelayColumn}>
|
||||
<td class="p-1 px-2 text-center align-middle">
|
||||
{(() => {
|
||||
const delay = resourceDelaySeconds(resource);
|
||||
const label = formatDelayLabel(delay);
|
||||
const custom = isCustomDelay(resource);
|
||||
const title = custom
|
||||
? 'Custom delay applied to this resource'
|
||||
: 'Using global alert delay';
|
||||
|
||||
return (
|
||||
<span
|
||||
class={`text-sm ${
|
||||
custom
|
||||
? 'font-medium text-blue-600 dark:text-blue-300'
|
||||
: 'text-gray-600 dark:text-gray-300'
|
||||
}`}
|
||||
title={title}
|
||||
>
|
||||
{label}
|
||||
</span>
|
||||
);
|
||||
})()}
|
||||
</td>
|
||||
</Show>
|
||||
|
||||
{/* Actions column */}
|
||||
<td class="p-1 px-2">
|
||||
<div class="flex items-center justify-center gap-1">
|
||||
@@ -1310,32 +1327,6 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||
</td>
|
||||
</Show>
|
||||
|
||||
<Show when={props.showDelayColumn}>
|
||||
<td class="p-1 px-2 text-center align-middle">
|
||||
{(() => {
|
||||
const delay = resourceDelaySeconds(resource);
|
||||
const label = formatDelayLabel(delay);
|
||||
const custom = isCustomDelay(resource);
|
||||
const title = custom
|
||||
? 'Custom delay applied to this resource'
|
||||
: 'Using global alert delay';
|
||||
|
||||
return (
|
||||
<span
|
||||
class={`text-sm ${
|
||||
custom
|
||||
? 'font-medium text-blue-600 dark:text-blue-300'
|
||||
: 'text-gray-600 dark:text-gray-300'
|
||||
}`}
|
||||
title={title}
|
||||
>
|
||||
{label}
|
||||
</span>
|
||||
);
|
||||
})()}
|
||||
</td>
|
||||
</Show>
|
||||
|
||||
{/* Actions column */}
|
||||
<td class="p-1 px-2">
|
||||
<div class="flex items-center justify-center gap-1">
|
||||
|
||||
@@ -91,7 +91,12 @@ interface ThresholdsTableProps {
|
||||
storageDefault: () => number;
|
||||
setStorageDefault: (value: number) => void;
|
||||
timeThresholds: () => { guest: number; node: number; storage: number; pbs: number };
|
||||
setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void;
|
||||
metricTimeThresholds: () => Record<string, Record<string, number>>;
|
||||
setMetricTimeThresholds: (
|
||||
value:
|
||||
| Record<string, Record<string, number>>
|
||||
| ((prev: Record<string, Record<string, number>>) => Record<string, Record<string, number>>),
|
||||
) => void;
|
||||
setHasUnsavedChanges: (value: boolean) => void;
|
||||
activeAlerts?: Record<string, Alert>;
|
||||
removeAlerts?: (predicate: (alert: Alert) => boolean) => void;
|
||||
@@ -130,30 +135,37 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
||||
|
||||
// Set up keyboard shortcuts
|
||||
onMount(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
// Skip if user is typing in an input or textarea (unless it's Escape)
|
||||
const target = e.target as HTMLElement;
|
||||
const isInInput =
|
||||
target.tagName === 'INPUT' ||
|
||||
target.tagName === 'TEXTAREA' ||
|
||||
target.contentEditable === 'true';
|
||||
const isEditableElement = (el: HTMLElement | null | undefined): boolean => {
|
||||
if (!el) return false;
|
||||
const tag = el.tagName;
|
||||
return (
|
||||
tag === 'INPUT' ||
|
||||
tag === 'TEXTAREA' ||
|
||||
tag === 'SELECT' ||
|
||||
el.contentEditable === 'true'
|
||||
);
|
||||
};
|
||||
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
const target = e.target as HTMLElement | null;
|
||||
const activeElement = (document.activeElement as HTMLElement) ?? null;
|
||||
const inEditable = isEditableElement(target);
|
||||
|
||||
// Escape clears search from anywhere
|
||||
if (e.key === 'Escape') {
|
||||
e.preventDefault();
|
||||
setSearchTerm('');
|
||||
if (searchInputRef && isInInput) {
|
||||
if (searchTerm()) {
|
||||
e.preventDefault();
|
||||
setSearchTerm('');
|
||||
}
|
||||
if (searchInputRef && document.activeElement === searchInputRef) {
|
||||
searchInputRef.blur();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip other shortcuts if already in an input
|
||||
if (isInInput) {
|
||||
if (e.defaultPrevented || inEditable || isEditableElement(activeElement) || editingId()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Any letter/number focuses search and starts typing
|
||||
if (e.key.length === 1 && e.key.match(/[a-z0-9]/i)) {
|
||||
e.preventDefault();
|
||||
if (searchInputRef) {
|
||||
@@ -293,6 +305,30 @@ const buildNodeHeaderMeta = (node: Node) => {
|
||||
return { headerMeta, keys };
|
||||
};
|
||||
|
||||
const overrideHasContent = (override: Override): boolean => {
|
||||
const hasThresholds = Object.values(override.thresholds || {}).some(
|
||||
(value) => value !== undefined,
|
||||
);
|
||||
const hasStateFlags = Boolean(
|
||||
override.disabled || override.disableConnectivity || override.poweredOffSeverity,
|
||||
);
|
||||
return hasThresholds || hasStateFlags;
|
||||
};
|
||||
|
||||
const buildOverrideFromResource = (resource: Resource): Override => ({
|
||||
id: resource.id,
|
||||
name: resource.name,
|
||||
type: resource.type as OverrideType,
|
||||
resourceType: resource.resourceType,
|
||||
vmid: (resource as unknown as { vmid?: number }).vmid,
|
||||
node: (resource as unknown as { node?: string }).node,
|
||||
instance: (resource as unknown as { instance?: string }).instance,
|
||||
disabled: resource.disabled,
|
||||
disableConnectivity: resource.disableConnectivity,
|
||||
poweredOffSeverity: resource.poweredOffSeverity,
|
||||
thresholds: resource.thresholds ? { ...resource.thresholds } : {},
|
||||
});
|
||||
|
||||
const nodesWithOverrides = createMemo<Resource[]>((prev = []) => {
|
||||
// If we're currently editing, return the previous value to avoid re-renders
|
||||
if (editingId()) {
|
||||
@@ -316,6 +352,7 @@ const buildNodeHeaderMeta = (node: Node) => {
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
const originalDisplayName = node.displayName?.trim() || node.name;
|
||||
const friendlyName = getFriendlyNodeName(originalDisplayName, node.clusterName);
|
||||
const rawName = node.name;
|
||||
@@ -341,8 +378,8 @@ const buildNodeHeaderMeta = (node: Node) => {
|
||||
hasOverride: hasCustomThresholds || false,
|
||||
disabled: false,
|
||||
disableConnectivity: override?.disableConnectivity || false,
|
||||
thresholds: override?.thresholds || {},
|
||||
defaults: props.nodeDefaults,
|
||||
thresholds: override?.thresholds || {},
|
||||
defaults: props.nodeDefaults,
|
||||
clusterName: node.isClusterMember ? node.clusterName?.trim() : undefined,
|
||||
isClusterMember: node.isClusterMember ?? false,
|
||||
instance: node.instance,
|
||||
@@ -465,6 +502,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
const hasOverride =
|
||||
hasCustomThresholds || override?.disabled || override?.disableConnectivity || overrideSeverity !== undefined || false;
|
||||
|
||||
@@ -621,6 +659,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
// A guest has an override if it has custom thresholds OR is disabled OR has connectivity disabled
|
||||
const hasOverride =
|
||||
hasCustomThresholds || override?.disabled || override?.disableConnectivity || overrideSeverity !== undefined || false;
|
||||
@@ -720,6 +759,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
return {
|
||||
id: pbsId,
|
||||
name: pbs.name,
|
||||
@@ -897,6 +937,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
|
||||
const editedThresholds = editingThresholds();
|
||||
const defaultThresholds = (resource.defaults ?? {}) as Record<string, number | undefined>;
|
||||
const existingOverride = props.overrides().find((o) => o.id === resourceId);
|
||||
|
||||
// Only include values that differ from defaults
|
||||
const overrideThresholds: Record<string, number> = {};
|
||||
@@ -1012,12 +1053,47 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
setEditingThresholds({});
|
||||
};
|
||||
|
||||
const updateDelay = (key: 'guest' | 'node' | 'storage' | 'pbs', value: number) => {
|
||||
const sanitized = Number.isFinite(value) ? Math.max(0, Math.round(value)) : 0;
|
||||
const current = props.timeThresholds();
|
||||
if (current[key] === sanitized) return;
|
||||
props.setTimeThresholds({ ...current, [key]: sanitized });
|
||||
props.setHasUnsavedChanges(true);
|
||||
const updateMetricDelay = (typeKey: 'guest' | 'node' | 'storage' | 'pbs', metricKey: string, value: number | null) => {
|
||||
const normalizedMetric = metricKey.trim().toLowerCase();
|
||||
if (!normalizedMetric) return;
|
||||
|
||||
let changed = false;
|
||||
props.setMetricTimeThresholds((prev) => {
|
||||
const current = prev ? { ...prev } : {};
|
||||
const existing = prev?.[typeKey];
|
||||
const typeOverrides = existing ? { ...existing } : {};
|
||||
|
||||
if (value === null) {
|
||||
if (typeOverrides[normalizedMetric] === undefined) {
|
||||
return prev;
|
||||
}
|
||||
delete typeOverrides[normalizedMetric];
|
||||
changed = true;
|
||||
} else {
|
||||
const sanitized = Math.max(0, Math.round(value));
|
||||
if (typeOverrides[normalizedMetric] === sanitized) {
|
||||
return prev;
|
||||
}
|
||||
typeOverrides[normalizedMetric] = sanitized;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if (!changed) {
|
||||
return prev;
|
||||
}
|
||||
|
||||
if (Object.keys(typeOverrides).length === 0) {
|
||||
delete current[typeKey];
|
||||
} else {
|
||||
current[typeKey] = typeOverrides;
|
||||
}
|
||||
|
||||
return current;
|
||||
});
|
||||
|
||||
if (changed) {
|
||||
props.setHasUnsavedChanges(true);
|
||||
}
|
||||
};
|
||||
|
||||
const removeOverride = (resourceId: string) => {
|
||||
@@ -1052,6 +1128,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
|
||||
// Get existing override if it exists
|
||||
const existingOverride = props.overrides().find((o) => o.id === resourceId);
|
||||
const existingRaw = props.rawOverridesConfig()[resourceId];
|
||||
|
||||
// Determine the current disabled state - check the resource's current state, not the override
|
||||
const currentDisabledState = resource.disabled;
|
||||
@@ -1062,7 +1139,10 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
delete (cleanThresholds as Record<string, unknown>).disabled;
|
||||
|
||||
// If enabling (disabled = false) and no custom thresholds exist, remove the override entirely
|
||||
if (!newDisabledState && (!existingOverride || Object.keys(cleanThresholds).length === 0)) {
|
||||
if (
|
||||
!newDisabledState &&
|
||||
(!existingOverride || Object.keys(cleanThresholds).length === 0)
|
||||
) {
|
||||
// Remove the override completely
|
||||
props.setOverrides(props.overrides().filter((o) => o.id !== resourceId));
|
||||
|
||||
@@ -1108,9 +1188,15 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
|
||||
if (newDisabledState) {
|
||||
hysteresisThresholds.disabled = true;
|
||||
} else {
|
||||
delete hysteresisThresholds.disabled;
|
||||
}
|
||||
|
||||
newRawConfig[resourceId] = hysteresisThresholds;
|
||||
if (Object.keys(hysteresisThresholds).length === 0) {
|
||||
delete newRawConfig[resourceId];
|
||||
} else {
|
||||
newRawConfig[resourceId] = hysteresisThresholds;
|
||||
}
|
||||
props.setRawOverridesConfig(newRawConfig);
|
||||
}
|
||||
|
||||
@@ -1158,6 +1244,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
|
||||
// Get existing override if it exists
|
||||
const existingOverride = props.overrides().find((o) => o.id === resourceId);
|
||||
const existingRaw = props.rawOverridesConfig()[resourceId];
|
||||
|
||||
// Determine the current state - use the resource's computed state, not just the override
|
||||
const currentDisableConnectivity = resource.disableConnectivity;
|
||||
@@ -1215,9 +1302,15 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
|
||||
if (newDisableConnectivity) {
|
||||
hysteresisThresholds.disableConnectivity = true;
|
||||
} else {
|
||||
delete hysteresisThresholds.disableConnectivity;
|
||||
}
|
||||
|
||||
newRawConfig[resourceId] = hysteresisThresholds;
|
||||
if (Object.keys(hysteresisThresholds).length === 0) {
|
||||
delete newRawConfig[resourceId];
|
||||
} else {
|
||||
newRawConfig[resourceId] = hysteresisThresholds;
|
||||
}
|
||||
props.setRawOverridesConfig(newRawConfig);
|
||||
}
|
||||
|
||||
@@ -1242,6 +1335,7 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
const defaultSeverity = props.guestPoweredOffSeverity();
|
||||
|
||||
const existingOverride = props.overrides().find((o) => o.id === resourceId);
|
||||
const existingRaw = props.rawOverridesConfig()[resourceId];
|
||||
const cleanThresholds: Record<string, number> = { ...(existingOverride?.thresholds || {}) };
|
||||
delete (cleanThresholds as Record<string, unknown>).disabled;
|
||||
delete (cleanThresholds as Record<string, unknown>).disableConnectivity;
|
||||
@@ -1258,7 +1352,12 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
newDisableConnectivity !== defaultDisabled ||
|
||||
(!newDisableConnectivity && newSeverity !== defaultSeverity);
|
||||
|
||||
if (!differsFromDefaults && !hasThresholds && !overrideDisabled && !existingOverride?.disableConnectivity) {
|
||||
if (
|
||||
!differsFromDefaults &&
|
||||
!hasThresholds &&
|
||||
!overrideDisabled &&
|
||||
!existingOverride?.disableConnectivity
|
||||
) {
|
||||
// Remove override entirely
|
||||
if (existingOverride) {
|
||||
props.setOverrides(props.overrides().filter((o) => o.id !== resourceId));
|
||||
@@ -1444,7 +1543,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
onToggleGlobalDisableOffline={() => props.setDisableAllNodesOffline(!props.disableAllNodesOffline())}
|
||||
showDelayColumn={true}
|
||||
globalDelaySeconds={props.timeThresholds().node}
|
||||
onGlobalDelayChange={(value) => updateDelay('node', value)}
|
||||
metricDelaySeconds={props.metricTimeThresholds().node ?? {}}
|
||||
onMetricDelayChange={(metric, value) => updateMetricDelay('node', metric, value)}
|
||||
/>
|
||||
</div>
|
||||
</Show>
|
||||
@@ -1492,7 +1592,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
onSetOfflineState={setOfflineState}
|
||||
showDelayColumn={true}
|
||||
globalDelaySeconds={props.timeThresholds().guest}
|
||||
onGlobalDelayChange={(value) => updateDelay('guest', value)}
|
||||
metricDelaySeconds={props.metricTimeThresholds().guest ?? {}}
|
||||
onMetricDelayChange={(metric, value) => updateMetricDelay('guest', metric, value)}
|
||||
/>
|
||||
</div>
|
||||
</Show>
|
||||
@@ -1531,7 +1632,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
onToggleGlobalDisable={() => props.setDisableAllStorage(!props.disableAllStorage())}
|
||||
showDelayColumn={true}
|
||||
globalDelaySeconds={props.timeThresholds().storage}
|
||||
onGlobalDelayChange={(value) => updateDelay('storage', value)}
|
||||
metricDelaySeconds={props.metricTimeThresholds().storage ?? {}}
|
||||
onMetricDelayChange={(metric, value) => updateMetricDelay('storage', metric, value)}
|
||||
/>
|
||||
</div>
|
||||
</Show>
|
||||
@@ -1572,7 +1674,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
onToggleGlobalDisableOffline={() => props.setDisableAllPBSOffline(!props.disableAllPBSOffline())}
|
||||
showDelayColumn={true}
|
||||
globalDelaySeconds={props.timeThresholds().pbs}
|
||||
onGlobalDelayChange={(value) => updateDelay('pbs', value)}
|
||||
metricDelaySeconds={props.metricTimeThresholds().pbs ?? {}}
|
||||
onMetricDelayChange={(metric, value) => updateMetricDelay('pbs', metric, value)}
|
||||
/>
|
||||
</div>
|
||||
</Show>
|
||||
@@ -1675,7 +1778,8 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
}
|
||||
showDelayColumn={true}
|
||||
globalDelaySeconds={props.timeThresholds().guest}
|
||||
onGlobalDelayChange={(value) => updateDelay('guest', value)}
|
||||
metricDelaySeconds={props.metricTimeThresholds().guest ?? {}}
|
||||
onMetricDelayChange={(metric, value) => updateMetricDelay('guest', metric, value)}
|
||||
globalOfflineSeverity={props.guestPoweredOffSeverity()}
|
||||
onSetOfflineState={setOfflineState}
|
||||
/>
|
||||
|
||||
@@ -163,11 +163,40 @@ const createDefaultGrouping = (): GroupingConfig => ({
|
||||
byGuest: false,
|
||||
});
|
||||
|
||||
const normalizeMetricDelayMap = (
|
||||
input: Record<string, Record<string, number>> | undefined | null,
|
||||
): Record<string, Record<string, number>> => {
|
||||
if (!input) return {};
|
||||
const normalized: Record<string, Record<string, number>> = {};
|
||||
|
||||
Object.entries(input).forEach(([rawType, metrics]) => {
|
||||
if (!metrics) return;
|
||||
const typeKey = rawType.trim().toLowerCase();
|
||||
if (!typeKey) return;
|
||||
|
||||
const entries: Record<string, number> = {};
|
||||
Object.entries(metrics).forEach(([rawMetric, value]) => {
|
||||
if (typeof value !== 'number' || Number.isNaN(value) || value < 0) return;
|
||||
const metricKey = rawMetric.trim().toLowerCase();
|
||||
if (!metricKey) return;
|
||||
entries[metricKey] = Math.round(value);
|
||||
});
|
||||
|
||||
if (Object.keys(entries).length > 0) {
|
||||
normalized[typeKey] = entries;
|
||||
}
|
||||
});
|
||||
|
||||
return normalized;
|
||||
};
|
||||
|
||||
const createDefaultEscalation = (): EscalationConfig => ({
|
||||
enabled: false,
|
||||
levels: [],
|
||||
});
|
||||
|
||||
const DEFAULT_DELAY_SECONDS = 5;
|
||||
|
||||
export function Alerts() {
|
||||
const { state, activeAlerts, updateAlert, removeAlerts } = useWebSocket();
|
||||
const navigate = useNavigate();
|
||||
@@ -462,23 +491,23 @@ export function Alerts() {
|
||||
const container = (state.containers || []).find((g) => g.id === key);
|
||||
const guest = vm || container;
|
||||
if (guest) {
|
||||
overridesList.push({
|
||||
id: key,
|
||||
name: guest.name,
|
||||
type: 'guest',
|
||||
resourceType: guest.type === 'qemu' ? 'VM' : 'CT',
|
||||
vmid: guest.vmid,
|
||||
node: guest.node,
|
||||
instance: guest.instance,
|
||||
disabled: thresholds.disabled || false,
|
||||
poweredOffSeverity:
|
||||
thresholds.poweredOffSeverity === 'critical'
|
||||
? 'critical'
|
||||
: thresholds.poweredOffSeverity === 'warning'
|
||||
? 'warning'
|
||||
: undefined,
|
||||
thresholds: extractTriggerValues(thresholds),
|
||||
});
|
||||
overridesList.push({
|
||||
id: key,
|
||||
name: guest.name,
|
||||
type: 'guest',
|
||||
resourceType: guest.type === 'qemu' ? 'VM' : 'CT',
|
||||
vmid: guest.vmid,
|
||||
node: guest.node,
|
||||
instance: guest.instance,
|
||||
disabled: thresholds.disabled || false,
|
||||
poweredOffSeverity:
|
||||
thresholds.poweredOffSeverity === 'critical'
|
||||
? 'critical'
|
||||
: thresholds.poweredOffSeverity === 'warning'
|
||||
? 'warning'
|
||||
: undefined,
|
||||
thresholds: extractTriggerValues(thresholds),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -507,7 +536,12 @@ export function Alerts() {
|
||||
(newOverride.type === 'guest' || newOverride.type === 'dockerContainer') &&
|
||||
(newOverride.poweredOffSeverity ?? null) !==
|
||||
(existing.poweredOffSeverity ?? null);
|
||||
return thresholdsChanged || connectivityChanged || disabledChanged || severityChanged;
|
||||
return (
|
||||
thresholdsChanged ||
|
||||
connectivityChanged ||
|
||||
disabledChanged ||
|
||||
severityChanged
|
||||
);
|
||||
});
|
||||
|
||||
if (hasChanged) {
|
||||
@@ -539,8 +573,14 @@ export function Alerts() {
|
||||
temperature: 80,
|
||||
});
|
||||
setStorageDefault(85);
|
||||
setTimeThreshold(0);
|
||||
setTimeThresholds({ guest: 10, node: 15, storage: 30, pbs: 30 });
|
||||
setTimeThreshold(DEFAULT_DELAY_SECONDS);
|
||||
setTimeThresholds({
|
||||
guest: DEFAULT_DELAY_SECONDS,
|
||||
node: DEFAULT_DELAY_SECONDS,
|
||||
storage: DEFAULT_DELAY_SECONDS,
|
||||
pbs: DEFAULT_DELAY_SECONDS,
|
||||
});
|
||||
setMetricTimeThresholds({});
|
||||
setScheduleQuietHours(createDefaultQuietHours());
|
||||
setScheduleCooldown(createDefaultCooldown());
|
||||
setScheduleGrouping(createDefaultGrouping());
|
||||
@@ -613,19 +653,25 @@ export function Alerts() {
|
||||
}
|
||||
if (config.timeThresholds) {
|
||||
setTimeThresholds({
|
||||
guest: config.timeThresholds.guest ?? 10,
|
||||
node: config.timeThresholds.node ?? 15,
|
||||
storage: config.timeThresholds.storage ?? 30,
|
||||
pbs: config.timeThresholds.pbs ?? 30,
|
||||
guest: config.timeThresholds.guest ?? DEFAULT_DELAY_SECONDS,
|
||||
node: config.timeThresholds.node ?? DEFAULT_DELAY_SECONDS,
|
||||
storage: config.timeThresholds.storage ?? DEFAULT_DELAY_SECONDS,
|
||||
pbs: config.timeThresholds.pbs ?? DEFAULT_DELAY_SECONDS,
|
||||
});
|
||||
} else if (config.timeThreshold !== undefined && config.timeThreshold > 0) {
|
||||
} else {
|
||||
const fallback = config.timeThreshold && config.timeThreshold > 0 ? config.timeThreshold : DEFAULT_DELAY_SECONDS;
|
||||
setTimeThresholds({
|
||||
guest: config.timeThreshold,
|
||||
node: config.timeThreshold,
|
||||
storage: config.timeThreshold,
|
||||
pbs: config.timeThreshold,
|
||||
guest: fallback,
|
||||
node: fallback,
|
||||
storage: fallback,
|
||||
pbs: fallback,
|
||||
});
|
||||
}
|
||||
if (config.metricTimeThresholds) {
|
||||
setMetricTimeThresholds(normalizeMetricDelayMap(config.metricTimeThresholds));
|
||||
} else {
|
||||
setMetricTimeThresholds({});
|
||||
}
|
||||
|
||||
// Load global disable flags
|
||||
setDisableAllNodes(config.disableAllNodes ?? false);
|
||||
@@ -856,14 +902,16 @@ export function Alerts() {
|
||||
memoryCriticalPct: 95,
|
||||
});
|
||||
|
||||
const [storageDefault, setStorageDefault] = createSignal(85);
|
||||
const [timeThreshold, setTimeThreshold] = createSignal(0); // Legacy
|
||||
const [timeThresholds, setTimeThresholds] = createSignal({
|
||||
guest: 10,
|
||||
node: 15,
|
||||
storage: 30,
|
||||
pbs: 30,
|
||||
});
|
||||
const [storageDefault, setStorageDefault] = createSignal(85);
|
||||
const [timeThreshold, setTimeThreshold] = createSignal(DEFAULT_DELAY_SECONDS); // Legacy
|
||||
const [timeThresholds, setTimeThresholds] = createSignal({
|
||||
guest: DEFAULT_DELAY_SECONDS,
|
||||
node: DEFAULT_DELAY_SECONDS,
|
||||
storage: DEFAULT_DELAY_SECONDS,
|
||||
pbs: DEFAULT_DELAY_SECONDS,
|
||||
});
|
||||
const [metricTimeThresholds, setMetricTimeThresholds] =
|
||||
createSignal<Record<string, Record<string, number>>>({});
|
||||
|
||||
// Global disable flags per resource type
|
||||
const [disableAllNodes, setDisableAllNodes] = createSignal(false);
|
||||
@@ -999,6 +1047,7 @@ export function Alerts() {
|
||||
hysteresisMargin: 5.0,
|
||||
timeThreshold: timeThreshold() || 0, // Legacy
|
||||
timeThresholds: timeThresholds(),
|
||||
metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()),
|
||||
// Use rawOverridesConfig which is already properly formatted with disabled flags
|
||||
overrides: rawOverridesConfig(),
|
||||
schedule: {
|
||||
@@ -1197,7 +1246,8 @@ export function Alerts() {
|
||||
storageDefault={storageDefault}
|
||||
setStorageDefault={setStorageDefault}
|
||||
timeThresholds={timeThresholds}
|
||||
setTimeThresholds={setTimeThresholds}
|
||||
metricTimeThresholds={metricTimeThresholds}
|
||||
setMetricTimeThresholds={setMetricTimeThresholds}
|
||||
activeAlerts={activeAlerts}
|
||||
setHasUnsavedChanges={setHasUnsavedChanges}
|
||||
hasUnsavedChanges={hasUnsavedChanges}
|
||||
@@ -1669,6 +1719,7 @@ interface ThresholdsTabProps {
|
||||
dockerDefaults: () => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number };
|
||||
storageDefault: () => number;
|
||||
timeThresholds: () => { guest: number; node: number; storage: number; pbs: number };
|
||||
metricTimeThresholds: () => Record<string, Record<string, number>>;
|
||||
overrides: () => Override[];
|
||||
rawOverridesConfig: () => Record<string, RawOverrideConfig>;
|
||||
setGuestDefaults: (
|
||||
@@ -1689,7 +1740,11 @@ interface ThresholdsTabProps {
|
||||
value: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number } | ((prev: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }) => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }),
|
||||
) => void;
|
||||
setStorageDefault: (value: number) => void;
|
||||
setTimeThresholds: (value: { guest: number; node: number; storage: number; pbs: number }) => void;
|
||||
setMetricTimeThresholds: (
|
||||
value:
|
||||
| Record<string, Record<string, number>>
|
||||
| ((prev: Record<string, Record<string, number>>) => Record<string, Record<string, number>>),
|
||||
) => void;
|
||||
setOverrides: (value: Override[]) => void;
|
||||
setRawOverridesConfig: (value: Record<string, RawOverrideConfig>) => void;
|
||||
activeAlerts: Record<string, Alert>;
|
||||
@@ -1746,7 +1801,8 @@ function ThresholdsTab(props: ThresholdsTabProps) {
|
||||
storageDefault={props.storageDefault}
|
||||
setStorageDefault={props.setStorageDefault}
|
||||
timeThresholds={props.timeThresholds}
|
||||
setTimeThresholds={props.setTimeThresholds}
|
||||
metricTimeThresholds={props.metricTimeThresholds}
|
||||
setMetricTimeThresholds={props.setMetricTimeThresholds}
|
||||
setHasUnsavedChanges={props.setHasUnsavedChanges}
|
||||
activeAlerts={props.activeAlerts}
|
||||
removeAlerts={props.removeAlerts}
|
||||
@@ -2627,6 +2683,7 @@ function HistoryTab() {
|
||||
const [alertHistory, setAlertHistory] = createSignal<Alert[]>([]);
|
||||
const [loading, setLoading] = createSignal(true);
|
||||
const [selectedBarIndex, setSelectedBarIndex] = createSignal<number | null>(null);
|
||||
const MS_PER_HOUR = 60 * 60 * 1000;
|
||||
|
||||
// Ref for search input
|
||||
let searchInputRef: HTMLInputElement | undefined;
|
||||
@@ -2640,6 +2697,25 @@ function HistoryTab() {
|
||||
localStorage.setItem('alertHistorySeverityFilter', severityFilter());
|
||||
});
|
||||
|
||||
// Clear chart selection when high-level filters change
|
||||
let lastTimeFilterValue: string | null = null;
|
||||
createEffect(() => {
|
||||
const current = timeFilter();
|
||||
if (lastTimeFilterValue !== null && current !== lastTimeFilterValue) {
|
||||
setSelectedBarIndex(null);
|
||||
}
|
||||
lastTimeFilterValue = current;
|
||||
});
|
||||
|
||||
let lastSeverityFilterValue: string | null = null;
|
||||
createEffect(() => {
|
||||
const current = severityFilter();
|
||||
if (lastSeverityFilterValue !== null && current !== lastSeverityFilterValue) {
|
||||
setSelectedBarIndex(null);
|
||||
}
|
||||
lastSeverityFilterValue = current;
|
||||
});
|
||||
|
||||
// Load alert history on mount
|
||||
onMount(async () => {
|
||||
try {
|
||||
@@ -2703,8 +2779,54 @@ function HistoryTab() {
|
||||
return `${minutes}m`;
|
||||
};
|
||||
|
||||
// Get resource type (VM, CT, Node, Storage)
|
||||
const getResourceType = (resourceName: string) => {
|
||||
const formatBucketRange = (startMs: number, endMs: number) => {
|
||||
const start = new Date(startMs);
|
||||
const end = new Date(endMs);
|
||||
|
||||
const sameDay =
|
||||
start.getFullYear() === end.getFullYear() &&
|
||||
start.getMonth() === end.getMonth() &&
|
||||
start.getDate() === end.getDate();
|
||||
|
||||
const startDay = start.toLocaleDateString('en-US', {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
year: start.getFullYear() !== end.getFullYear() ? 'numeric' : undefined,
|
||||
});
|
||||
const endDay = end.toLocaleDateString('en-US', {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
year: 'numeric',
|
||||
});
|
||||
|
||||
const timeFormatter: Intl.DateTimeFormatOptions = {
|
||||
hour: 'numeric',
|
||||
minute: '2-digit',
|
||||
};
|
||||
|
||||
const startTimeStr = start.toLocaleTimeString('en-US', timeFormatter);
|
||||
const endTimeStr = end.toLocaleTimeString('en-US', timeFormatter);
|
||||
|
||||
if (sameDay) {
|
||||
return `${startDay}, ${startTimeStr} – ${endTimeStr}`;
|
||||
}
|
||||
|
||||
return `${startDay}, ${startTimeStr} → ${endDay}, ${endTimeStr}`;
|
||||
};
|
||||
|
||||
// Get resource type (VM, CT, Node, Storage, Docker, PBS, etc.)
|
||||
const getResourceType = (
|
||||
resourceName: string,
|
||||
metadata?: Record<string, unknown> | undefined,
|
||||
) => {
|
||||
const metadataType =
|
||||
typeof metadata?.resourceType === 'string'
|
||||
? (metadata.resourceType as string)
|
||||
: undefined;
|
||||
if (metadataType && metadataType.trim().length > 0) {
|
||||
return metadataType;
|
||||
}
|
||||
|
||||
// Check VMs and containers
|
||||
const vm = state.vms?.find((v) => v.name === resourceName);
|
||||
if (vm) return 'VM';
|
||||
@@ -2720,6 +2842,34 @@ function HistoryTab() {
|
||||
const storage = state.storage?.find((s) => s.name === resourceName || s.id === resourceName);
|
||||
if (storage) return 'Storage';
|
||||
|
||||
// Docker hosts
|
||||
const dockerHost = state.dockerHosts?.find(
|
||||
(host) =>
|
||||
host.displayName === resourceName ||
|
||||
host.hostname === resourceName ||
|
||||
host.agentId === resourceName ||
|
||||
host.id === resourceName,
|
||||
);
|
||||
if (dockerHost) return 'Docker Host';
|
||||
|
||||
// Docker containers (via known hosts)
|
||||
const dockerContainer = state.dockerHosts
|
||||
?.flatMap((host) => host.containers || [])
|
||||
.find((c) => c.name === resourceName || c.id === resourceName);
|
||||
if (dockerContainer) return 'Docker Container';
|
||||
|
||||
// PBS instances
|
||||
const pbsInstance = state.pbs?.find(
|
||||
(pbs) => pbs.name === resourceName || pbs.host === resourceName || pbs.id === resourceName,
|
||||
);
|
||||
if (pbsInstance) return 'PBS';
|
||||
|
||||
// Ceph clusters
|
||||
const cephCluster = state.cephClusters?.find(
|
||||
(cluster) => cluster.name === resourceName || cluster.id === resourceName,
|
||||
);
|
||||
if (cephCluster) return 'Ceph';
|
||||
|
||||
return 'Unknown';
|
||||
};
|
||||
|
||||
@@ -2741,7 +2891,7 @@ function HistoryTab() {
|
||||
...alert,
|
||||
status: 'active',
|
||||
duration: formatDuration(alert.startTime),
|
||||
resourceType: getResourceType(alert.resourceName),
|
||||
resourceType: getResourceType(alert.resourceName, alert.metadata),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2759,63 +2909,67 @@ function HistoryTab() {
|
||||
...alert,
|
||||
status: alert.acknowledged ? 'acknowledged' : 'resolved',
|
||||
duration: formatDuration(alert.startTime, alert.lastSeen),
|
||||
resourceType: getResourceType(alert.resourceName),
|
||||
resourceType: getResourceType(alert.resourceName, alert.metadata),
|
||||
});
|
||||
});
|
||||
|
||||
return allAlerts;
|
||||
});
|
||||
|
||||
// Apply severity & search filters (time filtering is layered separately)
|
||||
const severityAndSearchFilteredAlerts = createMemo(() => {
|
||||
let filtered = allAlertsData();
|
||||
|
||||
if (severityFilter() !== 'all') {
|
||||
filtered = filtered.filter((a) => a.level === severityFilter());
|
||||
}
|
||||
|
||||
if (searchTerm()) {
|
||||
const term = searchTerm().toLowerCase();
|
||||
filtered = filtered.filter((alert) => {
|
||||
const name = alert.resourceName?.toLowerCase() ?? '';
|
||||
const message = alert.message?.toLowerCase() ?? '';
|
||||
const type = alert.type?.toLowerCase() ?? '';
|
||||
const nodeName = alert.node?.toLowerCase() ?? '';
|
||||
return (
|
||||
name.includes(term) || message.includes(term) || type.includes(term) || nodeName.includes(term)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
return filtered;
|
||||
});
|
||||
|
||||
// Apply filters to get the final alert data
|
||||
const alertData = createMemo(() => {
|
||||
let filtered = allAlertsData();
|
||||
let filtered = severityAndSearchFilteredAlerts();
|
||||
|
||||
// Selected bar filter (takes precedence over time filter)
|
||||
if (selectedBarIndex() !== null) {
|
||||
const trends = alertTrends();
|
||||
const index = selectedBarIndex()!;
|
||||
const bucketStart = trends.bucketTimes[index];
|
||||
const bucketEnd = bucketStart + trends.bucketSize * 60 * 60 * 1000;
|
||||
const bucketEnd = bucketStart + trends.bucketSize * MS_PER_HOUR;
|
||||
|
||||
filtered = filtered.filter((alert) => {
|
||||
const alertTime = new Date(alert.startTime).getTime();
|
||||
return alertTime >= bucketStart && alertTime < bucketEnd;
|
||||
});
|
||||
} else {
|
||||
// Time filter
|
||||
if (timeFilter() !== 'all') {
|
||||
const now = Date.now();
|
||||
const cutoff = {
|
||||
'24h': now - 24 * 60 * 60 * 1000,
|
||||
'7d': now - 7 * 24 * 60 * 60 * 1000,
|
||||
'30d': now - 30 * 24 * 60 * 60 * 1000,
|
||||
}[timeFilter()];
|
||||
} else if (timeFilter() !== 'all') {
|
||||
const now = Date.now();
|
||||
const cutoff = {
|
||||
'24h': now - 24 * 60 * 60 * 1000,
|
||||
'7d': now - 7 * 24 * 60 * 60 * 1000,
|
||||
'30d': now - 30 * 24 * 60 * 60 * 1000,
|
||||
}[timeFilter()];
|
||||
|
||||
if (cutoff) {
|
||||
filtered = filtered.filter((a) => new Date(a.startTime).getTime() > cutoff);
|
||||
}
|
||||
if (cutoff) {
|
||||
filtered = filtered.filter((a) => new Date(a.startTime).getTime() > cutoff);
|
||||
}
|
||||
}
|
||||
|
||||
// Severity filter
|
||||
if (severityFilter() !== 'all') {
|
||||
filtered = filtered.filter((a) => a.level === severityFilter());
|
||||
}
|
||||
|
||||
// Search filter
|
||||
if (searchTerm()) {
|
||||
const term = searchTerm().toLowerCase();
|
||||
filtered = filtered.filter(
|
||||
(alert) =>
|
||||
alert.resourceName.toLowerCase().includes(term) ||
|
||||
alert.message.toLowerCase().includes(term) ||
|
||||
alert.type.toLowerCase().includes(term) ||
|
||||
alert.node.toLowerCase().includes(term),
|
||||
);
|
||||
}
|
||||
|
||||
// Sort by start time (newest first)
|
||||
return filtered.sort(
|
||||
return [...filtered].sort(
|
||||
(a, b) => new Date(b.startTime).getTime() - new Date(a.startTime).getTime(),
|
||||
);
|
||||
});
|
||||
@@ -2919,60 +3073,114 @@ function HistoryTab() {
|
||||
// Calculate alert trends for mini-chart
|
||||
const alertTrends = createMemo(() => {
|
||||
const now = Date.now();
|
||||
const timeRange =
|
||||
timeFilter() === '24h'
|
||||
? 24
|
||||
: timeFilter() === '7d'
|
||||
? 7 * 24
|
||||
: timeFilter() === '30d'
|
||||
? 30 * 24
|
||||
: 90 * 24; // hours
|
||||
const bucketSize =
|
||||
timeFilter() === '24h' ? 1 : timeFilter() === '7d' ? 6 : timeFilter() === '30d' ? 24 : 72; // hours per bucket
|
||||
const numBuckets = Math.min(Math.floor(timeRange / bucketSize), 30); // Limit to 30 buckets max
|
||||
const msPerHour = MS_PER_HOUR;
|
||||
const filteredAlerts = severityAndSearchFilteredAlerts();
|
||||
const niceBucketSizes = [1, 2, 3, 6, 12, 24, 48, 72, 168, 336, 720, 1440]; // hours
|
||||
const maxBuckets = 30;
|
||||
|
||||
// Calculate start time for the chart
|
||||
const startTime = now - timeRange * 60 * 60 * 1000;
|
||||
let bucketSizeHours: number;
|
||||
let computedRangeHours: number;
|
||||
let startTime: number;
|
||||
|
||||
// Initialize buckets
|
||||
const buckets = new Array(numBuckets).fill(0);
|
||||
// bucketTimes represents the START of each bucket
|
||||
const bucketTimes = new Array(numBuckets)
|
||||
.fill(0)
|
||||
.map((_, i) => startTime + i * bucketSize * 60 * 60 * 1000);
|
||||
|
||||
// Filter alerts based on current time filter
|
||||
let alertsToCount = allAlertsData();
|
||||
if (timeFilter() !== 'all') {
|
||||
const cutoff = {
|
||||
'24h': now - 24 * 60 * 60 * 1000,
|
||||
'7d': now - 7 * 24 * 60 * 60 * 1000,
|
||||
'30d': now - 30 * 24 * 60 * 60 * 1000,
|
||||
}[timeFilter()];
|
||||
|
||||
if (cutoff) {
|
||||
alertsToCount = alertsToCount.filter((a) => new Date(a.startTime).getTime() > cutoff);
|
||||
const filter = timeFilter();
|
||||
if (filter === '24h') {
|
||||
bucketSizeHours = 1;
|
||||
computedRangeHours = 24;
|
||||
startTime = now - computedRangeHours * msPerHour;
|
||||
} else if (filter === '7d') {
|
||||
bucketSizeHours = 6;
|
||||
computedRangeHours = 7 * 24;
|
||||
startTime = now - computedRangeHours * msPerHour;
|
||||
} else if (filter === '30d') {
|
||||
bucketSizeHours = 24;
|
||||
computedRangeHours = 30 * 24;
|
||||
startTime = now - computedRangeHours * msPerHour;
|
||||
} else {
|
||||
if (!filteredAlerts.length) {
|
||||
bucketSizeHours = 24;
|
||||
computedRangeHours = 24;
|
||||
startTime = now - computedRangeHours * msPerHour;
|
||||
} else {
|
||||
const earliest = filteredAlerts.reduce((min, alert) => {
|
||||
const alertTime = new Date(alert.startTime).getTime();
|
||||
return Math.min(min, alertTime);
|
||||
}, now);
|
||||
const rawRangeHours = Math.max(1, Math.ceil((now - earliest) / msPerHour));
|
||||
const rawBucketSize = Math.max(1, Math.ceil(rawRangeHours / maxBuckets));
|
||||
bucketSizeHours =
|
||||
niceBucketSizes.find((size) => size >= rawBucketSize) ?? rawBucketSize;
|
||||
computedRangeHours = Math.max(rawRangeHours, bucketSizeHours);
|
||||
const bucketsNeeded = Math.min(
|
||||
Math.max(1, Math.ceil(computedRangeHours / bucketSizeHours)),
|
||||
maxBuckets,
|
||||
);
|
||||
startTime = now - bucketsNeeded * bucketSizeHours * msPerHour;
|
||||
}
|
||||
}
|
||||
|
||||
alertsToCount.forEach((alert) => {
|
||||
const bucketCount = Math.min(
|
||||
Math.max(1, Math.ceil(computedRangeHours / bucketSizeHours)),
|
||||
maxBuckets,
|
||||
);
|
||||
startTime = Math.min(startTime, now - bucketCount * bucketSizeHours * msPerHour);
|
||||
|
||||
const buckets = new Array(bucketCount).fill(0);
|
||||
const bucketTimes = new Array(bucketCount)
|
||||
.fill(0)
|
||||
.map((_, i) => startTime + i * bucketSizeHours * msPerHour);
|
||||
|
||||
const windowStart = startTime;
|
||||
const windowEnd = now;
|
||||
|
||||
filteredAlerts.forEach((alert) => {
|
||||
const alertTime = new Date(alert.startTime).getTime();
|
||||
if (alertTime >= startTime && alertTime <= now) {
|
||||
const bucketIndex = Math.floor((alertTime - startTime) / (bucketSize * 60 * 60 * 1000));
|
||||
if (bucketIndex >= 0 && bucketIndex < numBuckets) {
|
||||
buckets[bucketIndex]++;
|
||||
}
|
||||
if (alertTime < windowStart || alertTime > windowEnd) {
|
||||
return;
|
||||
}
|
||||
const rawIndex = Math.floor((alertTime - windowStart) / (bucketSizeHours * msPerHour));
|
||||
const bucketIndex = Math.min(bucketCount - 1, Math.max(0, rawIndex));
|
||||
if (bucketIndex >= 0 && bucketIndex < bucketCount) {
|
||||
buckets[bucketIndex]++;
|
||||
}
|
||||
});
|
||||
|
||||
// Find max for scaling
|
||||
const max = Math.max(...buckets, 1);
|
||||
|
||||
return {
|
||||
buckets,
|
||||
max,
|
||||
bucketSize,
|
||||
bucketSize: bucketSizeHours,
|
||||
bucketTimes,
|
||||
rangeStart: windowStart,
|
||||
rangeHours: bucketCount * bucketSizeHours,
|
||||
};
|
||||
});
|
||||
|
||||
const chartRangeLabel = createMemo(() => {
|
||||
const filter = timeFilter();
|
||||
if (filter === '24h') return '24h ago';
|
||||
if (filter === '7d') return '7d ago';
|
||||
if (filter === '30d') return '30d ago';
|
||||
|
||||
const rangeHours = alertTrends().rangeHours ?? 0;
|
||||
if (rangeHours <= 0) return '—';
|
||||
if (rangeHours >= 24) {
|
||||
const days = Math.round(rangeHours / 24);
|
||||
return `${days}d ago`;
|
||||
}
|
||||
return `${Math.round(rangeHours)}h ago`;
|
||||
});
|
||||
|
||||
const selectedBucketDetails = createMemo(() => {
|
||||
const index = selectedBarIndex();
|
||||
if (index === null) return null;
|
||||
const trends = alertTrends();
|
||||
const bucketStart = trends.bucketTimes[index];
|
||||
const bucketEnd = bucketStart + trends.bucketSize * MS_PER_HOUR;
|
||||
return {
|
||||
rangeLabel: formatBucketRange(bucketStart, bucketEnd),
|
||||
start: bucketStart,
|
||||
end: bucketEnd,
|
||||
};
|
||||
});
|
||||
|
||||
@@ -2980,7 +3188,7 @@ function HistoryTab() {
|
||||
<div class="space-y-4">
|
||||
{/* Alert Trends Mini-Chart */}
|
||||
<Card padding="md">
|
||||
<div class="mb-3 flex items-start justify-between gap-3">
|
||||
<div class="mb-3 flex flex-col gap-2 sm:flex-row sm:items-start sm:justify-between sm:gap-3">
|
||||
<SectionHeader
|
||||
title="Alert frequency"
|
||||
description={
|
||||
@@ -2991,25 +3199,37 @@ function HistoryTab() {
|
||||
size="sm"
|
||||
class="flex-1"
|
||||
/>
|
||||
<div class="flex items-center gap-2">
|
||||
<Show when={selectedBarIndex() !== null}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setSelectedBarIndex(null)}
|
||||
class="px-2 py-0.5 text-xs bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded hover:bg-blue-200 dark:hover:bg-blue-800/50 transition-colors"
|
||||
>
|
||||
Clear filter
|
||||
</button>
|
||||
<div class="flex flex-col items-start gap-2 sm:items-end">
|
||||
<Show when={selectedBucketDetails()}>
|
||||
{(selection) => (
|
||||
<div class="inline-flex items-center gap-2 rounded-full border border-blue-200 dark:border-blue-700 bg-blue-50 dark:bg-blue-900/30 px-3 py-1 text-xs text-blue-700 dark:text-blue-200">
|
||||
<span class="font-medium uppercase tracking-wide text-[10px] text-blue-600 dark:text-blue-300">
|
||||
Filtered Range
|
||||
</span>
|
||||
<span class="font-mono text-[11px]">{selection().rangeLabel}</span>
|
||||
</div>
|
||||
)}
|
||||
</Show>
|
||||
<div class="flex items-center gap-2 text-xs text-gray-500 dark:text-gray-400">
|
||||
<span class="flex items-center gap-1">
|
||||
<div class="w-2 h-2 bg-yellow-500 rounded-full"></div>
|
||||
{alertData().filter((a) => a.level === 'warning').length} warnings
|
||||
</span>
|
||||
<span class="flex items-center gap-1">
|
||||
<div class="w-2 h-2 bg-red-500 rounded-full"></div>
|
||||
{alertData().filter((a) => a.level === 'critical').length} critical
|
||||
</span>
|
||||
<div class="flex flex-wrap items-center justify-end gap-2">
|
||||
<Show when={selectedBarIndex() !== null}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setSelectedBarIndex(null)}
|
||||
class="px-2 py-0.5 text-xs bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300 rounded hover:bg-blue-200 dark:hover:bg-blue-800/50 transition-colors"
|
||||
>
|
||||
Clear filter
|
||||
</button>
|
||||
</Show>
|
||||
<div class="flex items-center gap-2 text-xs text-gray-500 dark:text-gray-400">
|
||||
<span class="flex items-center gap-1">
|
||||
<div class="h-2 w-2 rounded-full bg-yellow-500"></div>
|
||||
{alertData().filter((a) => a.level === 'warning').length} warnings
|
||||
</span>
|
||||
<span class="flex items-center gap-1">
|
||||
<div class="h-2 w-2 rounded-full bg-red-500"></div>
|
||||
{alertData().filter((a) => a.level === 'critical').length} critical
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -3049,11 +3269,10 @@ function HistoryTab() {
|
||||
}
|
||||
const rect = (e.currentTarget as HTMLElement).getBoundingClientRect();
|
||||
const bucketHours = alertTrends().bucketSize;
|
||||
const bucketLabel = (() => {
|
||||
if (timeFilter() === '24h') return `${bucketHours} hour period`;
|
||||
const bucketDays = bucketHours / 24;
|
||||
return `${bucketDays} day period`;
|
||||
})();
|
||||
const bucketLabel =
|
||||
bucketHours % 24 === 0
|
||||
? `${bucketHours / 24} day${bucketHours / 24 === 1 ? '' : 's'}`
|
||||
: `${bucketHours} hour${bucketHours === 1 ? '' : 's'}`;
|
||||
const timestamp = new Date(alertTrends().bucketTimes[i]).toLocaleString('en-US', {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
@@ -3062,7 +3281,7 @@ function HistoryTab() {
|
||||
});
|
||||
const content = [
|
||||
`${val} alert${val !== 1 ? 's' : ''}`,
|
||||
bucketLabel,
|
||||
`${bucketLabel} period`,
|
||||
timestamp,
|
||||
].join('\n');
|
||||
showTooltip(content, rect.left + rect.width / 2, rect.top, {
|
||||
@@ -3080,13 +3299,7 @@ function HistoryTab() {
|
||||
{/* Time labels */}
|
||||
<div class="flex justify-between mt-1 text-[10px] text-gray-400 dark:text-gray-500">
|
||||
<span>
|
||||
{timeFilter() === '24h'
|
||||
? '24h ago'
|
||||
: timeFilter() === '7d'
|
||||
? '7d ago'
|
||||
: timeFilter() === '30d'
|
||||
? '30d ago'
|
||||
: '90d ago'}
|
||||
{chartRangeLabel()}
|
||||
</span>
|
||||
<span>Now</span>
|
||||
</div>
|
||||
|
||||
@@ -84,6 +84,7 @@ export interface AlertConfig {
|
||||
storage?: number;
|
||||
pbs?: number;
|
||||
};
|
||||
metricTimeThresholds?: Record<string, Record<string, number>>;
|
||||
aggregation?: {
|
||||
enabled: boolean;
|
||||
timeWindow: number;
|
||||
|
||||
+214
-39
@@ -276,11 +276,12 @@ type AlertConfig struct {
|
||||
DisableAllPBSOffline bool `json:"disableAllPBSOffline"` // Disable PBS offline alerts globally
|
||||
DisableAllDockerHostsOffline bool `json:"disableAllDockerHostsOffline"` // Disable Docker host offline alerts globally
|
||||
// New configuration options
|
||||
MinimumDelta float64 `json:"minimumDelta"` // Minimum % change to trigger new alert
|
||||
SuppressionWindow int `json:"suppressionWindow"` // Minutes to suppress duplicate alerts
|
||||
HysteresisMargin float64 `json:"hysteresisMargin"` // Default margin for legacy thresholds
|
||||
TimeThreshold int `json:"timeThreshold"` // Legacy: Seconds that threshold must be exceeded before triggering
|
||||
TimeThresholds map[string]int `json:"timeThresholds"` // Per-type delays: guest, node, storage, pbs
|
||||
MinimumDelta float64 `json:"minimumDelta"` // Minimum % change to trigger new alert
|
||||
SuppressionWindow int `json:"suppressionWindow"` // Minutes to suppress duplicate alerts
|
||||
HysteresisMargin float64 `json:"hysteresisMargin"` // Default margin for legacy thresholds
|
||||
TimeThreshold int `json:"timeThreshold"` // Legacy: Seconds that threshold must be exceeded before triggering
|
||||
TimeThresholds map[string]int `json:"timeThresholds"` // Per-type delays: guest, node, storage, pbs
|
||||
MetricTimeThresholds map[string]map[string]int `json:"metricTimeThresholds"` // Optional per-metric delays keyed by resource type
|
||||
}
|
||||
|
||||
// Manager handles alert monitoring and state
|
||||
@@ -388,11 +389,12 @@ func NewManager() *Manager {
|
||||
MinimumDelta: 2.0, // 2% minimum change
|
||||
SuppressionWindow: 5, // 5 minutes
|
||||
HysteresisMargin: 5.0, // 5% default margin
|
||||
TimeThreshold: 5,
|
||||
TimeThresholds: map[string]int{
|
||||
"guest": 10, // 10 second delay for guest CPU alerts
|
||||
"node": 15, // 15 second delay for node alerts
|
||||
"storage": 30, // 30 second delay for storage alerts
|
||||
"pbs": 30, // 30 second delay for PBS alerts
|
||||
"guest": 5,
|
||||
"node": 5,
|
||||
"storage": 5,
|
||||
"pbs": 5,
|
||||
},
|
||||
Overrides: make(map[string]ThresholdConfig),
|
||||
Schedule: ScheduleConfig{
|
||||
@@ -539,6 +541,39 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
|
||||
config.HysteresisMargin = 5.0
|
||||
}
|
||||
|
||||
// Ensure temperature defaults exist for nodes so high temps alert out of the box
|
||||
if config.NodeDefaults.Temperature == nil || config.NodeDefaults.Temperature.Trigger <= 0 {
|
||||
config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75}
|
||||
} else if config.NodeDefaults.Temperature.Clear <= 0 {
|
||||
config.NodeDefaults.Temperature.Clear = config.NodeDefaults.Temperature.Trigger - 5
|
||||
if config.NodeDefaults.Temperature.Clear <= 0 {
|
||||
config.NodeDefaults.Temperature.Clear = 75
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize any metric-level delay overrides
|
||||
config.MetricTimeThresholds = normalizeMetricTimeThresholds(config.MetricTimeThresholds)
|
||||
|
||||
const defaultDelaySeconds = 5
|
||||
if config.TimeThreshold <= 0 {
|
||||
config.TimeThreshold = defaultDelaySeconds
|
||||
}
|
||||
if config.TimeThresholds == nil {
|
||||
config.TimeThresholds = make(map[string]int)
|
||||
}
|
||||
ensureDelay := func(key string) {
|
||||
if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 {
|
||||
config.TimeThresholds[key] = defaultDelaySeconds
|
||||
}
|
||||
}
|
||||
ensureDelay("guest")
|
||||
ensureDelay("node")
|
||||
ensureDelay("storage")
|
||||
ensureDelay("pbs")
|
||||
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
|
||||
config.TimeThresholds["all"] = defaultDelaySeconds
|
||||
}
|
||||
|
||||
config.GuestDefaults.PoweredOffSeverity = normalizePoweredOffSeverity(config.GuestDefaults.PoweredOffSeverity)
|
||||
config.NodeDefaults.PoweredOffSeverity = normalizePoweredOffSeverity(config.NodeDefaults.PoweredOffSeverity)
|
||||
|
||||
@@ -563,6 +598,42 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
|
||||
m.reevaluateActiveAlertsLocked()
|
||||
}
|
||||
|
||||
// normalizeMetricTimeThresholds cleans resource/metric keys and drops invalid delay overrides.
|
||||
func normalizeMetricTimeThresholds(input map[string]map[string]int) map[string]map[string]int {
|
||||
if len(input) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
normalized := make(map[string]map[string]int)
|
||||
for rawType, metrics := range input {
|
||||
typeKey := strings.ToLower(strings.TrimSpace(rawType))
|
||||
if typeKey == "" || len(metrics) == 0 {
|
||||
continue
|
||||
}
|
||||
for rawMetric, delay := range metrics {
|
||||
metricKey := strings.ToLower(strings.TrimSpace(rawMetric))
|
||||
if metricKey == "" || delay < 0 {
|
||||
continue
|
||||
}
|
||||
if _, exists := normalized[typeKey]; !exists {
|
||||
normalized[typeKey] = make(map[string]int)
|
||||
}
|
||||
normalized[typeKey][metricKey] = delay
|
||||
}
|
||||
}
|
||||
|
||||
if len(normalized) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
// NormalizeMetricTimeThresholds exposes normalization for other packages (e.g., config persistence).
|
||||
func NormalizeMetricTimeThresholds(input map[string]map[string]int) map[string]map[string]int {
|
||||
return normalizeMetricTimeThresholds(input)
|
||||
}
|
||||
|
||||
// applyGlobalOfflineSettingsLocked clears tracking and active alerts for globally disabled offline detectors.
|
||||
// Caller must hold m.mu.
|
||||
func (m *Manager) applyGlobalOfflineSettingsLocked() {
|
||||
@@ -2488,38 +2559,142 @@ func (m *Manager) clearAlert(alertID string) {
|
||||
Msg("Alert cleared")
|
||||
}
|
||||
|
||||
// getTimeThresholdForType returns the appropriate time threshold for the resource type
|
||||
func (m *Manager) getTimeThresholdForType(resourceType string) int {
|
||||
typeKey := strings.ToLower(strings.TrimSpace(resourceType))
|
||||
// getTimeThreshold determines the delay to apply for a metric/resource combination.
|
||||
func (m *Manager) getTimeThreshold(_ string, resourceType, metricType string) int {
|
||||
if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok {
|
||||
return delay
|
||||
}
|
||||
|
||||
// Use per-type thresholds if available
|
||||
if m.config.TimeThresholds != nil {
|
||||
switch typeKey {
|
||||
case "guest", "qemu", "lxc", "vm", "ct", "container":
|
||||
if delay, ok := m.config.TimeThresholds["guest"]; ok {
|
||||
return delay
|
||||
}
|
||||
case "docker container", "dockercontainer", "docker":
|
||||
if delay, ok := m.config.TimeThresholds["guest"]; ok {
|
||||
return delay
|
||||
}
|
||||
case "node":
|
||||
if delay, ok := m.config.TimeThresholds["node"]; ok {
|
||||
return delay
|
||||
}
|
||||
case "storage":
|
||||
if delay, ok := m.config.TimeThresholds["storage"]; ok {
|
||||
return delay
|
||||
}
|
||||
case "pbs":
|
||||
if delay, ok := m.config.TimeThresholds["pbs"]; ok {
|
||||
return delay
|
||||
}
|
||||
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
|
||||
|
||||
if !hasTypeSpecific {
|
||||
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
|
||||
return delay
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to legacy single threshold
|
||||
return m.config.TimeThreshold
|
||||
return base
|
||||
}
|
||||
|
||||
// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level.
|
||||
func (m *Manager) getMetricTimeThreshold(resourceType, metricType string) (int, bool) {
|
||||
if len(m.config.MetricTimeThresholds) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
metricKey := strings.ToLower(strings.TrimSpace(metricType))
|
||||
if metricKey == "" {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
for _, typeKey := range canonicalResourceTypeKeys(resourceType) {
|
||||
perType, ok := m.config.MetricTimeThresholds[typeKey]
|
||||
if !ok || len(perType) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if delay, ok := perType[metricKey]; ok {
|
||||
return delay, true
|
||||
}
|
||||
if delay, ok := perType["default"]; ok {
|
||||
return delay, true
|
||||
}
|
||||
if delay, ok := perType["_default"]; ok {
|
||||
return delay, true
|
||||
}
|
||||
if delay, ok := perType["*"]; ok {
|
||||
return delay, true
|
||||
}
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// getBaseTimeThreshold returns the resource-type level delay.
|
||||
func (m *Manager) getBaseTimeThreshold(resourceType string) (int, bool) {
|
||||
if m.config.TimeThresholds != nil {
|
||||
for _, key := range canonicalResourceTypeKeys(resourceType) {
|
||||
if delay, ok := m.config.TimeThresholds[key]; ok {
|
||||
return delay, true
|
||||
}
|
||||
}
|
||||
if delay, ok := m.config.TimeThresholds["all"]; ok {
|
||||
return delay, false
|
||||
}
|
||||
}
|
||||
|
||||
return m.config.TimeThreshold, false
|
||||
}
|
||||
|
||||
func (m *Manager) getGlobalMetricTimeThreshold(metricType string) (int, bool) {
|
||||
if len(m.config.MetricTimeThresholds) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
perType, ok := m.config.MetricTimeThresholds["all"]
|
||||
if !ok || len(perType) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
metricKey := strings.ToLower(strings.TrimSpace(metricType))
|
||||
if metricKey == "" {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
if delay, ok := perType[metricKey]; ok {
|
||||
return delay, true
|
||||
}
|
||||
if delay, ok := perType["default"]; ok {
|
||||
return delay, true
|
||||
}
|
||||
if delay, ok := perType["_default"]; ok {
|
||||
return delay, true
|
||||
}
|
||||
if delay, ok := perType["*"]; ok {
|
||||
return delay, true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func canonicalResourceTypeKeys(resourceType string) []string {
|
||||
typeKey := strings.ToLower(strings.TrimSpace(resourceType))
|
||||
|
||||
addUnique := func(slice []string, value string) []string {
|
||||
if value == "" {
|
||||
return slice
|
||||
}
|
||||
for _, existing := range slice {
|
||||
if existing == value {
|
||||
return slice
|
||||
}
|
||||
}
|
||||
return append(slice, value)
|
||||
}
|
||||
|
||||
var keys []string
|
||||
switch typeKey {
|
||||
case "guest", "qemu", "vm", "ct", "container", "lxc":
|
||||
keys = addUnique(keys, "guest")
|
||||
case "docker", "docker container", "dockercontainer":
|
||||
keys = addUnique(keys, "docker")
|
||||
keys = addUnique(keys, "guest")
|
||||
case "docker host", "dockerhost":
|
||||
keys = addUnique(keys, "dockerhost")
|
||||
keys = addUnique(keys, "docker")
|
||||
keys = addUnique(keys, "node")
|
||||
case "node":
|
||||
keys = addUnique(keys, "node")
|
||||
case "pbs", "pbs server", "pbsserver":
|
||||
keys = addUnique(keys, "pbs")
|
||||
keys = addUnique(keys, "node")
|
||||
case "storage":
|
||||
keys = addUnique(keys, "storage")
|
||||
default:
|
||||
keys = addUnique(keys, typeKey)
|
||||
}
|
||||
|
||||
return keys
|
||||
}
|
||||
|
||||
// checkMetric checks a single metric against its threshold with hysteresis
|
||||
@@ -2563,8 +2738,8 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||
if !exists {
|
||||
alertStartTime := time.Now()
|
||||
|
||||
// Determine the appropriate time threshold based on resource type
|
||||
timeThreshold := m.getTimeThresholdForType(resourceType)
|
||||
// Determine the appropriate time threshold based on resource/metric type
|
||||
timeThreshold := m.getTimeThreshold(resourceID, resourceType, metricType)
|
||||
|
||||
// Check if we have a time threshold configured
|
||||
if timeThreshold > 0 {
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGetTimeThresholdForTypeMappings(t *testing.T) {
|
||||
func TestGetTimeThresholdMappings(t *testing.T) {
|
||||
manager := NewManager()
|
||||
|
||||
manager.mu.Lock()
|
||||
@@ -35,8 +35,53 @@ func TestGetTimeThresholdForTypeMappings(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
if got := manager.getTimeThresholdForType(tc.resourceType); got != tc.expected {
|
||||
t.Errorf("getTimeThresholdForType(%q) = %d, want %d", tc.resourceType, got, tc.expected)
|
||||
if got := manager.getTimeThreshold("", tc.resourceType, "cpu"); got != tc.expected {
|
||||
t.Errorf("getTimeThreshold(%q, \"cpu\") = %d, want %d", tc.resourceType, got, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTimeThresholdMetricOverrides(t *testing.T) {
|
||||
manager := NewManager()
|
||||
|
||||
manager.mu.Lock()
|
||||
manager.config.TimeThreshold = 15
|
||||
manager.config.TimeThresholds = map[string]int{
|
||||
"guest": 30,
|
||||
"node": 60,
|
||||
"storage": 90,
|
||||
}
|
||||
manager.config.MetricTimeThresholds = map[string]map[string]int{
|
||||
"guest": {
|
||||
"cpu": 5,
|
||||
},
|
||||
"node": {
|
||||
"temperature": 120,
|
||||
},
|
||||
"all": {
|
||||
"default": 20,
|
||||
},
|
||||
}
|
||||
manager.mu.Unlock()
|
||||
|
||||
cases := []struct {
|
||||
resourceID string
|
||||
resourceType string
|
||||
metricType string
|
||||
expected int
|
||||
}{
|
||||
{"vm-resource", "VM", "cpu", 5}, // guest metric override
|
||||
{"vm-resource", "VM", "memory", 30}, // falls back to guest type delay
|
||||
{"node-1", "Node", "temperature", 120}, // node metric override
|
||||
{"node-1", "Node", "cpu", 60}, // node type delay
|
||||
{"storage-1", "storage", "usage", 90}, // storage type delay
|
||||
{"unknown", "unknown", "cpu", 20}, // global default metric override
|
||||
{"unknown", "unknown", "disk", 20},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
if got := manager.getTimeThreshold(tc.resourceID, tc.resourceType, tc.metricType); got != tc.expected {
|
||||
t.Errorf("getTimeThreshold(%q, %q, %q) = %d, want %d", tc.resourceID, tc.resourceType, tc.metricType, got, tc.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,6 +103,25 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error {
|
||||
if config.HysteresisMargin <= 0 {
|
||||
config.HysteresisMargin = 5.0
|
||||
}
|
||||
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
|
||||
if config.TimeThreshold <= 0 {
|
||||
config.TimeThreshold = 5
|
||||
}
|
||||
if config.TimeThresholds == nil {
|
||||
config.TimeThresholds = make(map[string]int)
|
||||
}
|
||||
ensureDelay := func(key string) {
|
||||
if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 {
|
||||
config.TimeThresholds[key] = config.TimeThreshold
|
||||
}
|
||||
}
|
||||
ensureDelay("guest")
|
||||
ensureDelay("node")
|
||||
ensureDelay("storage")
|
||||
ensureDelay("pbs")
|
||||
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
|
||||
config.TimeThresholds["all"] = config.TimeThreshold
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(config, "", " ")
|
||||
if err != nil {
|
||||
@@ -138,11 +157,19 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
|
||||
Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85},
|
||||
},
|
||||
NodeDefaults: alerts.ThresholdConfig{
|
||||
CPU: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85},
|
||||
CPU: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
Memory: &alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
Disk: &alerts.HysteresisThreshold{Trigger: 90, Clear: 85},
|
||||
Temperature: &alerts.HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
},
|
||||
StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
TimeThreshold: 5,
|
||||
TimeThresholds: map[string]int{
|
||||
"guest": 5,
|
||||
"node": 5,
|
||||
"storage": 5,
|
||||
"pbs": 5,
|
||||
},
|
||||
StorageDefault: alerts.HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
MinimumDelta: 2.0,
|
||||
SuppressionWindow: 5,
|
||||
HysteresisMargin: 5.0,
|
||||
@@ -175,6 +202,28 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
|
||||
if config.HysteresisMargin <= 0 {
|
||||
config.HysteresisMargin = 5.0
|
||||
}
|
||||
if config.NodeDefaults.Temperature == nil || config.NodeDefaults.Temperature.Trigger <= 0 {
|
||||
config.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
|
||||
}
|
||||
if config.TimeThreshold <= 0 {
|
||||
config.TimeThreshold = 5
|
||||
}
|
||||
if config.TimeThresholds == nil {
|
||||
config.TimeThresholds = make(map[string]int)
|
||||
}
|
||||
ensureDelay := func(key string) {
|
||||
if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 {
|
||||
config.TimeThresholds[key] = config.TimeThreshold
|
||||
}
|
||||
}
|
||||
ensureDelay("guest")
|
||||
ensureDelay("node")
|
||||
ensureDelay("storage")
|
||||
ensureDelay("pbs")
|
||||
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
|
||||
config.TimeThresholds["all"] = config.TimeThreshold
|
||||
}
|
||||
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
|
||||
|
||||
// Migration: Set I/O metrics to Off (0) if they have the old default values
|
||||
// This helps existing users avoid noisy I/O alerts
|
||||
|
||||
Reference in New Issue
Block a user