mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
feat(alerts): configure SMART disk thresholds
Expose host disk SMART health, counter, endurance, spare, and CRC-growth rules in alert settings while preserving existing defaults and supporting explicit disable values. Change-source: pulse-maintainer
This commit is contained in:
@@ -118,6 +118,22 @@ provider-backed disk surface rather than creating a duplicate host alert.
|
||||
`internal/alerts/alerts_test.go` pin the baseline, growth, reset, recovery, and
|
||||
retention boundaries.
|
||||
|
||||
Host SMART alert policy is configured through the resolved agent threshold
|
||||
chain, not embedded inside the disk evaluator. Missing SMART rule fields seed
|
||||
the backward-compatible policy: failed health is enabled; reallocated,
|
||||
pending, uncorrectable, media-error, and CRC-growth counters trigger at one;
|
||||
remaining-life warning/critical thresholds are 10/5 percent; and NVMe spare
|
||||
warning/critical thresholds are 20/10 percent. An explicit zero disables that
|
||||
individual rule and must survive normalization, cloning, persistence, and
|
||||
per-machine override resolution. Counter rules trigger when current evidence
|
||||
reaches the configured value, while the CRC rule applies the configured
|
||||
minimum increase to two successive reports. Endurance and spare values remain
|
||||
evidence-aware, and invalid percentage inputs clamp to 0..100 with critical no
|
||||
higher than its warning boundary. The Machines threshold surface must
|
||||
round-trip every rule without changing these defaults. Linked Proxmox host
|
||||
agents continue to defer risk alert ownership even when their resolved SMART
|
||||
policy differs.
|
||||
|
||||
Threshold sections are keyed by override identity, not by resource type. The
|
||||
Virtualization Hosts section reads and writes overrides on the bare resource id,
|
||||
while the Machines section resolves through the agent-derived identity
|
||||
|
||||
@@ -191,6 +191,18 @@ page, platform navigation, the resource drawer and the alerts threshold sections
|
||||
all read it. Selecting by bare `type === 'agent'` collapses that distinction and
|
||||
puts the same machine on two surfaces that do not share an identity.
|
||||
|
||||
The Machines threshold tab owns one responsive SMART rules card ahead of the
|
||||
resource groups. It composes the existing threshold tab layout and native
|
||||
accessible checkbox/number controls rather than creating a second settings
|
||||
shell. Failed-health is presented as a toggle; sector, media, CRC-growth,
|
||||
remaining-life, and spare policies are integer inputs; percentage inputs are
|
||||
bounded to 0..100; and the card states that zero disables an individual numeric
|
||||
rule. Every edit uses the canonical agent-default setter and dirty-state path,
|
||||
so the shared save/discard bar owns persistence on desktop and narrow layouts.
|
||||
The single-column narrow layout must keep labels, help, values, percent suffixes,
|
||||
and the full scroll journey inside the app scroll shell without horizontal
|
||||
overflow.
|
||||
|
||||
## Canonical Files
|
||||
|
||||
1. `frontend-modern/src/components/shared/`
|
||||
|
||||
@@ -386,6 +386,16 @@ also preserve the unreported sentinel rather than collapsing an absent facet
|
||||
onto the struct zero value, because `0` is a real reading meaning no endurance
|
||||
remains.
|
||||
|
||||
Storage health assessment accepts an explicit SMART policy from alerting for
|
||||
agent-only host disks while retaining the factory-policy entry points for all
|
||||
other consumers. The policy controls failed-health, sector, media-error,
|
||||
remaining-life, spare, and reallocated-sector classification; a zero threshold
|
||||
disables only that rule. It must not alter collection truth, device identity,
|
||||
the wearout evidence predicate, or the separate temperature classification.
|
||||
This separation keeps provider physical-disk risk deterministic while allowing
|
||||
the alert subsystem to apply resolved per-host settings without forking SMART
|
||||
parsing or risk reason codes.
|
||||
|
||||
Discovery suppression for configured connections is a fail-closed obligation,
|
||||
not a best-effort optimisation. Every configured PVE, PBS and PMG host is
|
||||
resolved into the discovery IP blocklist so the scanner never fingerprints a
|
||||
|
||||
@@ -1,11 +1,23 @@
|
||||
{
|
||||
"version": 1,
|
||||
"base_sha": "118c90d1f0df2ff007a3494e25139f019a38137d",
|
||||
"verified_at": "2026-08-27T11:18:00Z",
|
||||
"base_sha": "3d91709ebad0332858e4615d72daf8e424ab42a8",
|
||||
"verified_at": "2026-08-27T12:19:53Z",
|
||||
"result": "passed",
|
||||
"changed_paths": [],
|
||||
"content_sha256": {},
|
||||
"routes": ["/proxmox/backups"],
|
||||
"changed_paths": [
|
||||
"frontend-modern/src/components/Alerts/ThresholdsTableAgentsTab.tsx",
|
||||
"frontend-modern/src/components/Alerts/ThresholdsTableSMARTDefaultsCard.tsx",
|
||||
"frontend-modern/src/features/alerts/alertsConfigurationModel.ts",
|
||||
"frontend-modern/src/types/alerts.ts",
|
||||
"frontend-modern/src/utils/alertThresholdDefaults.ts"
|
||||
],
|
||||
"content_sha256": {
|
||||
"frontend-modern/src/components/Alerts/ThresholdsTableAgentsTab.tsx": "89646984deb982699d3359953ab85a2ceae15830465ccf20b5603f5a426e3193",
|
||||
"frontend-modern/src/components/Alerts/ThresholdsTableSMARTDefaultsCard.tsx": "c1bd1947dbcbab5806e11fa905271039131cebb0a736642e73fd3652f63e6b66",
|
||||
"frontend-modern/src/features/alerts/alertsConfigurationModel.ts": "2100da1f350cbbea386f1ae669a655d213ea710f1636eb94fe80c6d47d29313d",
|
||||
"frontend-modern/src/types/alerts.ts": "cc8b88ec5d8de87dbb69116114e9cd889347972b59dd81bec081096ab791c693",
|
||||
"frontend-modern/src/utils/alertThresholdDefaults.ts": "6ea4a3f99c8bd27a4fcca98f67c866d06a0f5d3e3211550a58447688cb61352a"
|
||||
},
|
||||
"routes": ["/alerts/thresholds/systems"],
|
||||
"viewports": [
|
||||
{
|
||||
"width": 1440,
|
||||
@@ -17,17 +29,17 @@
|
||||
}
|
||||
],
|
||||
"states": [
|
||||
"mock-mode Proxmox Backups loaded with standalone PBS service and correlated Agent host resources",
|
||||
"backup-vault expanded into the canonical resource detail row with PBS service identity and Agent metrics target",
|
||||
"History rendered stored utilization, network, disk, and thermal samples instead of Collecting History",
|
||||
"History range changed from 24 hours to 7 days and retained populated charts",
|
||||
"backup-vault collapsed back to the originating summary row"
|
||||
"Machines thresholds loaded with the backward-compatible SMART factory rules",
|
||||
"failed-health disabled with pending sectors set to 4, life warning set to 15 percent, and NVMe spare critical disabled at 0",
|
||||
"edited SMART rules persisted after Save Changes and a full route reload",
|
||||
"factory SMART values restored and saved from the narrow layout",
|
||||
"complete single-column SMART card inspected through the app scroll shell at 390x844"
|
||||
],
|
||||
"interactions": [
|
||||
"opened backup-vault from the Backup servers table and selected the shared History tab at 1440x1000",
|
||||
"confirmed the desktop drawer showed populated Agent-backed charts and the document and body scroll widths matched the viewport",
|
||||
"repeated the whole-row touch expansion and History flow at 390x844, selected the 7 day range, and confirmed every chart card remained contained",
|
||||
"collapsed the phone detail through the originating backup-vault row and confirmed focus returned to that row",
|
||||
"confirmed the browser console contained no errors during the final desktop and phone verification flows"
|
||||
"opened Alerts, selected Thresholds, and selected the Machines platform tab at 1440x1000",
|
||||
"toggled failed-health, edited counter and percentage rules including an explicit 0 disable value, and saved through the shared dirty-state bar",
|
||||
"reloaded /alerts/thresholds/systems and confirmed every edited value persisted",
|
||||
"scrolled the complete SMART rules card at 390x844, confirmed scroll width matched client width, restored factory values, and saved from the narrow layout",
|
||||
"confirmed labels, help text, controls, percentage suffixes, focusable save actions, and the browser console remained free of errors"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { ThresholdsTableAgentDisksSection } from './ThresholdsTableAgentDisksSection';
|
||||
import { ThresholdsTableAgentsResourcesSection } from './ThresholdsTableAgentsResourcesSection';
|
||||
import { ThresholdsTableSMARTDefaultsCard } from './ThresholdsTableSMARTDefaultsCard';
|
||||
import type { ThresholdsTableSectionProps } from '@/features/alerts/thresholds/thresholdsTableSectionProps';
|
||||
|
||||
export function ThresholdsTableAgentsTab(props: ThresholdsTableSectionProps) {
|
||||
return (
|
||||
<>
|
||||
<ThresholdsTableSMARTDefaultsCard {...props} />
|
||||
<ThresholdsTableAgentsResourcesSection {...props} />
|
||||
<ThresholdsTableAgentDisksSection {...props} />
|
||||
</>
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { For } from 'solid-js';
|
||||
import ShieldAlert from 'lucide-solid/icons/shield-alert';
|
||||
|
||||
import type { ThresholdsTableSectionProps } from '@/features/alerts/thresholds/thresholdsTableSectionProps';
|
||||
|
||||
const counterRules = [
|
||||
{
|
||||
key: 'smartReallocated',
|
||||
label: 'Reallocated sectors',
|
||||
help: 'Alert when the current counter reaches this value.',
|
||||
},
|
||||
{
|
||||
key: 'smartPending',
|
||||
label: 'Pending sectors',
|
||||
help: 'Alert when sectors are waiting to be remapped.',
|
||||
},
|
||||
{
|
||||
key: 'smartUncorrectable',
|
||||
label: 'Uncorrectable sectors',
|
||||
help: 'Alert when offline scans find unreadable sectors.',
|
||||
},
|
||||
{
|
||||
key: 'smartMediaErrors',
|
||||
label: 'Media errors',
|
||||
help: 'Alert when the drive reports media or data-integrity errors.',
|
||||
},
|
||||
{
|
||||
key: 'smartCrcErrorDelta',
|
||||
label: 'CRC increase',
|
||||
help: 'Alert when the CRC counter grows by at least this much between reports.',
|
||||
},
|
||||
] as const;
|
||||
|
||||
const percentageRules = [
|
||||
{ key: 'smartLifeWarning', label: 'Life remaining warning', help: 'Warn below this value.' },
|
||||
{ key: 'smartLifeCritical', label: 'Life remaining critical', help: 'Critical at or below.' },
|
||||
{ key: 'smartSpareWarning', label: 'NVMe spare warning', help: 'Warn below this value.' },
|
||||
{ key: 'smartSpareCritical', label: 'NVMe spare critical', help: 'Critical at or below.' },
|
||||
] as const;
|
||||
|
||||
export function ThresholdsTableSMARTDefaultsCard(props: ThresholdsTableSectionProps) {
|
||||
const updateValue = (key: string, value: number) => {
|
||||
const normalized = Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0;
|
||||
props.tableProps.setAgentDefaults((previous) => ({ ...previous, [key]: normalized }));
|
||||
props.tableProps.setHasUnsavedChanges(true);
|
||||
};
|
||||
|
||||
return (
|
||||
<section
|
||||
class="rounded-lg border border-base-300 bg-surface p-4"
|
||||
aria-labelledby="smart-alert-defaults-title"
|
||||
>
|
||||
<div class="flex items-start gap-3">
|
||||
<ShieldAlert class="mt-0.5 h-5 w-5 shrink-0 text-warning" aria-hidden="true" />
|
||||
<div>
|
||||
<h3 id="smart-alert-defaults-title" class="font-semibold text-base-content">
|
||||
SMART alert rules
|
||||
</h3>
|
||||
<p class="mt-1 text-sm text-muted">
|
||||
Defaults for agent-reported disks. Set a numeric rule to 0 to disable it.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<label class="mt-4 flex min-h-11 items-center justify-between gap-4 rounded-md border border-base-300 px-3 py-2 sm:min-h-0">
|
||||
<span>
|
||||
<span class="block text-sm font-medium text-base-content">Failed health status</span>
|
||||
<span class="block text-xs text-muted">
|
||||
Alert when SMART reports a failed health state.
|
||||
</span>
|
||||
</span>
|
||||
<input
|
||||
type="checkbox"
|
||||
class="toggle toggle-warning toggle-sm"
|
||||
checked={(props.tableProps.agentDefaults.smartHealthFailure ?? 1) > 0}
|
||||
onChange={(event) =>
|
||||
updateValue('smartHealthFailure', event.currentTarget.checked ? 1 : 0)
|
||||
}
|
||||
aria-label="Alert on failed SMART health status"
|
||||
/>
|
||||
</label>
|
||||
|
||||
<div class="mt-3 grid gap-3 md:grid-cols-2 xl:grid-cols-3">
|
||||
<For each={counterRules}>
|
||||
{(rule) => (
|
||||
<label class="rounded-md border border-base-300 px-3 py-2">
|
||||
<span class="block text-sm font-medium text-base-content">{rule.label}</span>
|
||||
<span class="block min-h-8 text-xs text-muted">{rule.help}</span>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
step="1"
|
||||
class="input input-bordered input-sm mt-2 w-full"
|
||||
value={props.tableProps.agentDefaults[rule.key] ?? 0}
|
||||
onInput={(event) => updateValue(rule.key, event.currentTarget.valueAsNumber)}
|
||||
aria-label={`${rule.label} threshold`}
|
||||
/>
|
||||
</label>
|
||||
)}
|
||||
</For>
|
||||
<For each={percentageRules}>
|
||||
{(rule) => (
|
||||
<label class="rounded-md border border-base-300 px-3 py-2">
|
||||
<span class="block text-sm font-medium text-base-content">{rule.label}</span>
|
||||
<span class="block min-h-8 text-xs text-muted">{rule.help}</span>
|
||||
<div class="mt-2 flex items-center gap-2">
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="100"
|
||||
step="1"
|
||||
class="input input-bordered input-sm w-full"
|
||||
value={props.tableProps.agentDefaults[rule.key] ?? 0}
|
||||
onInput={(event) =>
|
||||
updateValue(rule.key, Math.min(100, event.currentTarget.valueAsNumber))
|
||||
}
|
||||
aria-label={`${rule.label} percentage`}
|
||||
/>
|
||||
<span class="text-sm text-muted">%</span>
|
||||
</div>
|
||||
</label>
|
||||
)}
|
||||
</For>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
import { cleanup, fireEvent, render, screen } from '@solidjs/testing-library';
|
||||
import { createSignal } from 'solid-js';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { ThresholdsTableSMARTDefaultsCard } from '../ThresholdsTableSMARTDefaultsCard';
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
describe('ThresholdsTableSMARTDefaultsCard', () => {
|
||||
it('updates health, counter, and percentage defaults and marks the form dirty', async () => {
|
||||
const [agentDefaults, setAgentDefaults] = createSignal<Record<string, number | undefined>>({
|
||||
smartHealthFailure: 1,
|
||||
smartPending: 1,
|
||||
smartLifeWarning: 10,
|
||||
});
|
||||
const setHasUnsavedChanges = vi.fn();
|
||||
|
||||
render(() => (
|
||||
<ThresholdsTableSMARTDefaultsCard
|
||||
state={{} as never}
|
||||
tableProps={
|
||||
{
|
||||
get agentDefaults() {
|
||||
return agentDefaults();
|
||||
},
|
||||
setAgentDefaults,
|
||||
setHasUnsavedChanges,
|
||||
} as never
|
||||
}
|
||||
/>
|
||||
));
|
||||
|
||||
await fireEvent.click(screen.getByLabelText('Alert on failed SMART health status'));
|
||||
await fireEvent.input(screen.getByLabelText('Pending sectors threshold'), {
|
||||
target: { value: '4' },
|
||||
});
|
||||
await fireEvent.input(screen.getByLabelText('Life remaining warning percentage'), {
|
||||
target: { value: '25' },
|
||||
});
|
||||
|
||||
expect(agentDefaults().smartHealthFailure).toBe(0);
|
||||
expect(agentDefaults().smartPending).toBe(4);
|
||||
expect(agentDefaults().smartLifeWarning).toBe(25);
|
||||
expect(setHasUnsavedChanges).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('clamps negative counters and percentages above 100', async () => {
|
||||
const [agentDefaults, setAgentDefaults] = createSignal<Record<string, number | undefined>>({});
|
||||
|
||||
render(() => (
|
||||
<ThresholdsTableSMARTDefaultsCard
|
||||
state={{} as never}
|
||||
tableProps={
|
||||
{
|
||||
get agentDefaults() {
|
||||
return agentDefaults();
|
||||
},
|
||||
setAgentDefaults,
|
||||
setHasUnsavedChanges: vi.fn(),
|
||||
} as never
|
||||
}
|
||||
/>
|
||||
));
|
||||
|
||||
await fireEvent.input(screen.getByLabelText('Media errors threshold'), {
|
||||
target: { value: '-3' },
|
||||
});
|
||||
await fireEvent.input(screen.getByLabelText('NVMe spare warning percentage'), {
|
||||
target: { value: '120' },
|
||||
});
|
||||
|
||||
expect(agentDefaults().smartMediaErrors).toBe(0);
|
||||
expect(agentDefaults().smartSpareWarning).toBe(100);
|
||||
});
|
||||
});
|
||||
@@ -83,7 +83,14 @@ const buildProps = (): ThresholdsTabProps =>
|
||||
networkOut: -1,
|
||||
}),
|
||||
setVMwareDefaults: vi.fn(),
|
||||
agentDefaults: () => ({ cpu: 80 }),
|
||||
agentDefaults: () => ({
|
||||
cpu: 80,
|
||||
smartHealthFailure: 1,
|
||||
smartPending: 4,
|
||||
smartCrcErrorDelta: 2,
|
||||
smartLifeWarning: 15,
|
||||
smartSpareCritical: 0,
|
||||
}),
|
||||
setAgentDefaults: vi.fn(),
|
||||
dockerDefaults: () => ({
|
||||
cpu: 80,
|
||||
@@ -156,7 +163,14 @@ const buildProps = (): ThresholdsTabProps =>
|
||||
networkIn: -1,
|
||||
networkOut: -1,
|
||||
},
|
||||
factoryAgentDefaults: { cpu: 80 },
|
||||
factoryAgentDefaults: {
|
||||
cpu: 80,
|
||||
smartHealthFailure: 1,
|
||||
smartPending: 1,
|
||||
smartCrcErrorDelta: 1,
|
||||
smartLifeWarning: 10,
|
||||
smartSpareCritical: 10,
|
||||
},
|
||||
factoryDockerDefaults: {
|
||||
cpu: 80,
|
||||
memory: 85,
|
||||
@@ -290,6 +304,18 @@ describe('ThresholdsTab', () => {
|
||||
expect(props.containerRuntimes).toEqual([]);
|
||||
expect(typeof props.guestDefaults).toBe('object');
|
||||
expect(typeof props.dockerDefaults).toBe('object');
|
||||
expect(props.agentDefaults).toMatchObject({
|
||||
smartHealthFailure: 1,
|
||||
smartPending: 4,
|
||||
smartCrcErrorDelta: 2,
|
||||
smartLifeWarning: 15,
|
||||
smartSpareCritical: 0,
|
||||
});
|
||||
expect(props.factoryAgentDefaults).toMatchObject({
|
||||
smartPending: 1,
|
||||
smartLifeWarning: 10,
|
||||
smartSpareCritical: 10,
|
||||
});
|
||||
});
|
||||
|
||||
it('routes a resource-row delay action into the canonical intent policy panel', async () => {
|
||||
|
||||
+15
@@ -39,6 +39,21 @@ describe('createHysteresisThreshold (via buildAlertsConfigurationPayload)', () =
|
||||
expect(alertConfig?.guestDefaults?.diskRead).toEqual({ trigger: 6, clear: 1 });
|
||||
});
|
||||
|
||||
it('persists SMART rules as discrete agent defaults', () => {
|
||||
const snapshot = createDefaultAlertsConfigurationSnapshot();
|
||||
snapshot.agentDefaults.smartHealthFailure = 0;
|
||||
snapshot.agentDefaults.smartPending = 4;
|
||||
snapshot.agentDefaults.smartCrcErrorDelta = 2;
|
||||
snapshot.agentDefaults.smartLifeWarning = 18;
|
||||
|
||||
const { alertConfig } = buildFromSnapshot(snapshot);
|
||||
|
||||
expect(alertConfig?.agentDefaults?.smartHealthFailure).toBe(0);
|
||||
expect(alertConfig?.agentDefaults?.smartPending).toBe(4);
|
||||
expect(alertConfig?.agentDefaults?.smartCrcErrorDelta).toBe(2);
|
||||
expect(alertConfig?.agentDefaults?.smartLifeWarning).toBe(18);
|
||||
});
|
||||
|
||||
it('applies the threshold factory to each diskTempByType entry', () => {
|
||||
const snapshot = createDefaultAlertsConfigurationSnapshot();
|
||||
snapshot.diskTempByType = { hot: 7, sub: 2 };
|
||||
|
||||
+11
-1
@@ -433,7 +433,7 @@ describe('readAlertsConfigurationSnapshot — vmwareDefaults full extraction', (
|
||||
});
|
||||
|
||||
describe('readAlertsConfigurationSnapshot — agentDefaults full extraction', () => {
|
||||
it('extracts all four metric trigger values', () => {
|
||||
it('extracts metric triggers and SMART rules', () => {
|
||||
const snapshot = readAlertsConfigurationSnapshot(
|
||||
cfg({
|
||||
agentDefaults: {
|
||||
@@ -441,6 +441,11 @@ describe('readAlertsConfigurationSnapshot — agentDefaults full extraction', ()
|
||||
memory: { trigger: 77, clear: 72 },
|
||||
disk: { trigger: 82, clear: 77 },
|
||||
diskTemperature: { trigger: 60, clear: 55 },
|
||||
smartHealthFailure: 0,
|
||||
smartPending: 4,
|
||||
smartCrcErrorDelta: 3,
|
||||
smartLifeWarning: 20,
|
||||
smartLifeCritical: 8,
|
||||
},
|
||||
}),
|
||||
);
|
||||
@@ -448,6 +453,11 @@ describe('readAlertsConfigurationSnapshot — agentDefaults full extraction', ()
|
||||
expect(snapshot.agentDefaults.memory).toBe(77);
|
||||
expect(snapshot.agentDefaults.disk).toBe(82);
|
||||
expect(snapshot.agentDefaults.diskTemperature).toBe(60);
|
||||
expect(snapshot.agentDefaults.smartHealthFailure).toBe(0);
|
||||
expect(snapshot.agentDefaults.smartPending).toBe(4);
|
||||
expect(snapshot.agentDefaults.smartCrcErrorDelta).toBe(3);
|
||||
expect(snapshot.agentDefaults.smartLifeWarning).toBe(20);
|
||||
expect(snapshot.agentDefaults.smartLifeCritical).toBe(8);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -439,6 +439,25 @@ export function readAlertsConfigurationSnapshot(config: AlertConfig): AlertsConf
|
||||
diskTemperature:
|
||||
getTriggerValue(config.agentDefaults.diskTemperature) ??
|
||||
FACTORY_AGENT_DEFAULTS.diskTemperature,
|
||||
smartHealthFailure:
|
||||
config.agentDefaults.smartHealthFailure ?? FACTORY_AGENT_DEFAULTS.smartHealthFailure,
|
||||
smartReallocated:
|
||||
config.agentDefaults.smartReallocated ?? FACTORY_AGENT_DEFAULTS.smartReallocated,
|
||||
smartPending: config.agentDefaults.smartPending ?? FACTORY_AGENT_DEFAULTS.smartPending,
|
||||
smartUncorrectable:
|
||||
config.agentDefaults.smartUncorrectable ?? FACTORY_AGENT_DEFAULTS.smartUncorrectable,
|
||||
smartMediaErrors:
|
||||
config.agentDefaults.smartMediaErrors ?? FACTORY_AGENT_DEFAULTS.smartMediaErrors,
|
||||
smartCrcErrorDelta:
|
||||
config.agentDefaults.smartCrcErrorDelta ?? FACTORY_AGENT_DEFAULTS.smartCrcErrorDelta,
|
||||
smartLifeWarning:
|
||||
config.agentDefaults.smartLifeWarning ?? FACTORY_AGENT_DEFAULTS.smartLifeWarning,
|
||||
smartLifeCritical:
|
||||
config.agentDefaults.smartLifeCritical ?? FACTORY_AGENT_DEFAULTS.smartLifeCritical,
|
||||
smartSpareWarning:
|
||||
config.agentDefaults.smartSpareWarning ?? FACTORY_AGENT_DEFAULTS.smartSpareWarning,
|
||||
smartSpareCritical:
|
||||
config.agentDefaults.smartSpareCritical ?? FACTORY_AGENT_DEFAULTS.smartSpareCritical,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -770,6 +789,16 @@ export function buildAlertsConfigurationPayload({
|
||||
memory: createHysteresisThreshold(snapshot.agentDefaults.memory),
|
||||
disk: createHysteresisThreshold(snapshot.agentDefaults.disk),
|
||||
diskTemperature: createHysteresisThreshold(snapshot.agentDefaults.diskTemperature),
|
||||
smartHealthFailure: snapshot.agentDefaults.smartHealthFailure,
|
||||
smartReallocated: snapshot.agentDefaults.smartReallocated,
|
||||
smartPending: snapshot.agentDefaults.smartPending,
|
||||
smartUncorrectable: snapshot.agentDefaults.smartUncorrectable,
|
||||
smartMediaErrors: snapshot.agentDefaults.smartMediaErrors,
|
||||
smartCrcErrorDelta: snapshot.agentDefaults.smartCrcErrorDelta,
|
||||
smartLifeWarning: snapshot.agentDefaults.smartLifeWarning,
|
||||
smartLifeCritical: snapshot.agentDefaults.smartLifeCritical,
|
||||
smartSpareWarning: snapshot.agentDefaults.smartSpareWarning,
|
||||
smartSpareCritical: snapshot.agentDefaults.smartSpareCritical,
|
||||
},
|
||||
diskTempByType: Object.fromEntries(
|
||||
Object.entries(snapshot.diskTempByType).map(([key, trigger]) => [
|
||||
|
||||
@@ -15,6 +15,16 @@ export interface AlertThresholds {
|
||||
networkOut?: HysteresisThreshold;
|
||||
temperature?: HysteresisThreshold;
|
||||
diskTemperature?: HysteresisThreshold;
|
||||
smartHealthFailure?: number;
|
||||
smartReallocated?: number;
|
||||
smartPending?: number;
|
||||
smartUncorrectable?: number;
|
||||
smartMediaErrors?: number;
|
||||
smartCrcErrorDelta?: number;
|
||||
smartLifeWarning?: number;
|
||||
smartLifeCritical?: number;
|
||||
smartSpareWarning?: number;
|
||||
smartSpareCritical?: number;
|
||||
usage?: HysteresisThreshold;
|
||||
disableConnectivity?: boolean; // Disable connectivity/powered-off alerts
|
||||
poweredOffSeverity?: 'warning' | 'critical';
|
||||
|
||||
@@ -123,6 +123,16 @@ describe('alertThresholdDefaults', () => {
|
||||
memory: 85,
|
||||
disk: 90,
|
||||
diskTemperature: 55,
|
||||
smartHealthFailure: 1,
|
||||
smartReallocated: 1,
|
||||
smartPending: 1,
|
||||
smartUncorrectable: 1,
|
||||
smartMediaErrors: 1,
|
||||
smartCrcErrorDelta: 1,
|
||||
smartLifeWarning: 10,
|
||||
smartLifeCritical: 5,
|
||||
smartSpareWarning: 20,
|
||||
smartSpareCritical: 10,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
type MetricType,
|
||||
} from '@/utils/metricThresholds';
|
||||
import {
|
||||
FACTORY_AGENT_DEFAULTS,
|
||||
FACTORY_KUBERNETES_DEFAULTS,
|
||||
FACTORY_SNAPSHOT_DEFAULTS,
|
||||
FACTORY_TRUENAS_DEFAULTS,
|
||||
@@ -451,5 +452,20 @@ describe('metricThresholds', () => {
|
||||
networkOut: -1,
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps SMART display defaults aligned with host alert evaluation', () => {
|
||||
expect(FACTORY_AGENT_DEFAULTS).toMatchObject({
|
||||
smartHealthFailure: 1,
|
||||
smartReallocated: 1,
|
||||
smartPending: 1,
|
||||
smartUncorrectable: 1,
|
||||
smartMediaErrors: 1,
|
||||
smartCrcErrorDelta: 1,
|
||||
smartLifeWarning: 10,
|
||||
smartLifeCritical: 5,
|
||||
smartSpareWarning: 20,
|
||||
smartSpareCritical: 10,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -64,6 +64,16 @@ export const FACTORY_AGENT_DEFAULTS = {
|
||||
memory: 85,
|
||||
disk: 90,
|
||||
diskTemperature: 55,
|
||||
smartHealthFailure: 1,
|
||||
smartReallocated: 1,
|
||||
smartPending: 1,
|
||||
smartUncorrectable: 1,
|
||||
smartMediaErrors: 1,
|
||||
smartCrcErrorDelta: 1,
|
||||
smartLifeWarning: 10,
|
||||
smartLifeCritical: 5,
|
||||
smartSpareWarning: 20,
|
||||
smartSpareCritical: 10,
|
||||
};
|
||||
|
||||
// Mirrors the backend's seeded DiskTempByType defaults (trigger °C).
|
||||
|
||||
@@ -20746,6 +20746,48 @@ func TestCheckHostAlertsWhenSMARTCRCCountIncreases(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHostUsesConfiguredSMARTCounterThresholds(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
m.ClearActiveAlerts()
|
||||
config := m.GetConfig()
|
||||
pendingThreshold := int64(3)
|
||||
crcDelta := int64(2)
|
||||
config.AgentDefaults.SMARTPending = &pendingThreshold
|
||||
config.AgentDefaults.SMARTCRCErrorDelta = &crcDelta
|
||||
m.UpdateConfig(config)
|
||||
|
||||
pending := int64(2)
|
||||
crcErrors := int64(10)
|
||||
host := models.Host{
|
||||
ID: "tuned-smart-host",
|
||||
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
|
||||
Device: "/dev/sda",
|
||||
Health: "PASSED",
|
||||
Attributes: &models.SMARTAttributes{
|
||||
PendingSectors: &pending,
|
||||
UDMACRCErrors: &crcErrors,
|
||||
},
|
||||
}}},
|
||||
}
|
||||
alertKey := buildCanonicalStateID("agent:tuned-smart-host/disk:sda", "agent:tuned-smart-host/disk:sda-disk-health")
|
||||
|
||||
m.CheckHost(host)
|
||||
crcErrors = 11
|
||||
m.CheckHost(host)
|
||||
if testHasActiveAlert(t, m, alertKey) {
|
||||
t.Fatal("values below configured SMART thresholds must not alert")
|
||||
}
|
||||
|
||||
pending = 3
|
||||
crcErrors = 12
|
||||
m.CheckHost(host)
|
||||
alert := testRequireActiveAlert(t, m, alertKey)
|
||||
codes, ok := alert.Metadata["riskCodes"].([]string)
|
||||
if !ok || !reflect.DeepEqual(codes, []string{"pending_sectors"}) {
|
||||
t.Fatalf("riskCodes = %#v, want pending_sectors at configured boundary", alert.Metadata["riskCodes"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHostSMARTCRCCounterResetEstablishesNewBaseline(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
m.ClearActiveAlerts()
|
||||
|
||||
@@ -344,11 +344,57 @@ func NormalizeAgentDefaults(config *AlertConfig) {
|
||||
}
|
||||
}
|
||||
EnsureValidHysteresis(config.AgentDefaults.DiskTemperature, "agent.diskTemperature")
|
||||
normalizeSMARTDefaults(&config.AgentDefaults)
|
||||
|
||||
NormalizeDiskFillByType(config)
|
||||
NormalizeDiskTempByType(config)
|
||||
}
|
||||
|
||||
func normalizeSMARTDefaults(config *ThresholdConfig) {
|
||||
if config.SMARTHealthFailure == nil {
|
||||
config.SMARTHealthFailure = smartIntPtr(1)
|
||||
} else if *config.SMARTHealthFailure != 0 {
|
||||
*config.SMARTHealthFailure = 1
|
||||
}
|
||||
config.SMARTReallocated = normalizeNonNegativeInt64(config.SMARTReallocated, 1)
|
||||
config.SMARTPending = normalizeNonNegativeInt64(config.SMARTPending, 1)
|
||||
config.SMARTUncorrectable = normalizeNonNegativeInt64(config.SMARTUncorrectable, 1)
|
||||
config.SMARTMediaErrors = normalizeNonNegativeInt64(config.SMARTMediaErrors, 1)
|
||||
config.SMARTCRCErrorDelta = normalizeNonNegativeInt64(config.SMARTCRCErrorDelta, 1)
|
||||
config.SMARTLifeWarning = normalizePercentage(config.SMARTLifeWarning, 10)
|
||||
config.SMARTLifeCritical = normalizePercentage(config.SMARTLifeCritical, 5)
|
||||
config.SMARTSpareWarning = normalizePercentage(config.SMARTSpareWarning, 20)
|
||||
config.SMARTSpareCritical = normalizePercentage(config.SMARTSpareCritical, 10)
|
||||
|
||||
if *config.SMARTLifeWarning > 0 && *config.SMARTLifeCritical > *config.SMARTLifeWarning {
|
||||
*config.SMARTLifeCritical = *config.SMARTLifeWarning
|
||||
}
|
||||
if *config.SMARTSpareWarning > 0 && *config.SMARTSpareCritical > *config.SMARTSpareWarning {
|
||||
*config.SMARTSpareCritical = *config.SMARTSpareWarning
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeNonNegativeInt64(value *int64, fallback int64) *int64 {
|
||||
if value == nil || *value < 0 {
|
||||
return smartInt64Ptr(fallback)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func normalizePercentage(value *int, fallback int) *int {
|
||||
if value == nil || *value < 0 {
|
||||
return smartIntPtr(fallback)
|
||||
}
|
||||
if *value > 100 {
|
||||
*value = 100
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func smartIntPtr(value int) *int { return &value }
|
||||
|
||||
func smartInt64Ptr(value int64) *int64 { return &value }
|
||||
|
||||
func normalizeThresholdPointer(
|
||||
current *HysteresisThreshold,
|
||||
defaultTrigger float64,
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package config_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
alertconfig "github.com/rcourtman/pulse-go-rewrite/internal/alerts/config"
|
||||
)
|
||||
|
||||
func smartInt(value int) *int { return &value }
|
||||
|
||||
func smartInt64(value int64) *int64 { return &value }
|
||||
|
||||
func TestNormalizeAgentDefaultsSMARTBackwardCompatibilityAndDisableSemantics(t *testing.T) {
|
||||
config := &alertconfig.AlertConfig{}
|
||||
alertconfig.NormalizeAgentDefaults(config)
|
||||
|
||||
if config.AgentDefaults.SMARTHealthFailure == nil || *config.AgentDefaults.SMARTHealthFailure != 1 ||
|
||||
config.AgentDefaults.SMARTCRCErrorDelta == nil || *config.AgentDefaults.SMARTCRCErrorDelta != 1 ||
|
||||
config.AgentDefaults.SMARTLifeWarning == nil || *config.AgentDefaults.SMARTLifeWarning != 10 {
|
||||
t.Fatalf("missing SMART defaults were not seeded: %+v", config.AgentDefaults)
|
||||
}
|
||||
|
||||
config.AgentDefaults.SMARTHealthFailure = smartInt(0)
|
||||
config.AgentDefaults.SMARTPending = smartInt64(0)
|
||||
config.AgentDefaults.SMARTCRCErrorDelta = smartInt64(0)
|
||||
config.AgentDefaults.SMARTLifeWarning = smartInt(0)
|
||||
config.AgentDefaults.SMARTLifeCritical = smartInt(0)
|
||||
alertconfig.NormalizeAgentDefaults(config)
|
||||
|
||||
if *config.AgentDefaults.SMARTHealthFailure != 0 || *config.AgentDefaults.SMARTPending != 0 ||
|
||||
*config.AgentDefaults.SMARTCRCErrorDelta != 0 || *config.AgentDefaults.SMARTLifeWarning != 0 ||
|
||||
*config.AgentDefaults.SMARTLifeCritical != 0 {
|
||||
t.Fatalf("explicit zero SMART rules must stay disabled: %+v", config.AgentDefaults)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAgentDefaultsSMARTClampsInvalidRanges(t *testing.T) {
|
||||
config := &alertconfig.AlertConfig{AgentDefaults: alertconfig.ThresholdConfig{
|
||||
SMARTHealthFailure: smartInt(9),
|
||||
SMARTPending: smartInt64(-2),
|
||||
SMARTLifeWarning: smartInt(150),
|
||||
SMARTLifeCritical: smartInt(120),
|
||||
SMARTSpareWarning: smartInt(12),
|
||||
SMARTSpareCritical: smartInt(40),
|
||||
}}
|
||||
alertconfig.NormalizeAgentDefaults(config)
|
||||
|
||||
if *config.AgentDefaults.SMARTHealthFailure != 1 || *config.AgentDefaults.SMARTPending != 1 {
|
||||
t.Fatalf("invalid discrete SMART values were not normalized: %+v", config.AgentDefaults)
|
||||
}
|
||||
if *config.AgentDefaults.SMARTLifeWarning != 100 || *config.AgentDefaults.SMARTLifeCritical != 100 {
|
||||
t.Fatalf("life thresholds were not clamped: %+v", config.AgentDefaults)
|
||||
}
|
||||
if *config.AgentDefaults.SMARTSpareWarning != 12 || *config.AgentDefaults.SMARTSpareCritical != 12 {
|
||||
t.Fatalf("critical spare threshold must not exceed warning: %+v", config.AgentDefaults)
|
||||
}
|
||||
}
|
||||
@@ -50,9 +50,19 @@ type ThresholdConfig struct {
|
||||
DiskWrite *HysteresisThreshold `json:"diskWrite,omitempty"`
|
||||
NetworkIn *HysteresisThreshold `json:"networkIn,omitempty"`
|
||||
NetworkOut *HysteresisThreshold `json:"networkOut,omitempty"`
|
||||
Usage *HysteresisThreshold `json:"usage,omitempty"` // For storage devices
|
||||
Temperature *HysteresisThreshold `json:"temperature,omitempty"` // For node CPU temperature
|
||||
DiskTemperature *HysteresisThreshold `json:"diskTemperature,omitempty"` // For host SMART temperatures
|
||||
Usage *HysteresisThreshold `json:"usage,omitempty"` // For storage devices
|
||||
Temperature *HysteresisThreshold `json:"temperature,omitempty"` // For node CPU temperature
|
||||
DiskTemperature *HysteresisThreshold `json:"diskTemperature,omitempty"` // For host SMART temperatures
|
||||
SMARTHealthFailure *int `json:"smartHealthFailure,omitempty"` // 1 alerts on failed SMART health, 0 disables
|
||||
SMARTReallocated *int64 `json:"smartReallocated,omitempty"` // Reallocated-sector count; 0 disables
|
||||
SMARTPending *int64 `json:"smartPending,omitempty"` // Pending-sector count; 0 disables
|
||||
SMARTUncorrectable *int64 `json:"smartUncorrectable,omitempty"` // Offline-uncorrectable count; 0 disables
|
||||
SMARTMediaErrors *int64 `json:"smartMediaErrors,omitempty"` // Media-error count; 0 disables
|
||||
SMARTCRCErrorDelta *int64 `json:"smartCrcErrorDelta,omitempty"` // Increase between reports; 0 disables
|
||||
SMARTLifeWarning *int `json:"smartLifeWarning,omitempty"` // Remaining-life warning percentage; 0 disables
|
||||
SMARTLifeCritical *int `json:"smartLifeCritical,omitempty"` // Remaining-life critical percentage; 0 disables
|
||||
SMARTSpareWarning *int `json:"smartSpareWarning,omitempty"` // NVMe spare warning percentage; 0 disables
|
||||
SMARTSpareCritical *int `json:"smartSpareCritical,omitempty"` // NVMe spare critical percentage; 0 disables
|
||||
Backup *BackupAlertConfig `json:"backup,omitempty"`
|
||||
Snapshot *SnapshotAlertConfig `json:"snapshot,omitempty"`
|
||||
Note *string `json:"note,omitempty"`
|
||||
|
||||
@@ -641,6 +641,22 @@ func cloneStringPtr(value *string) *string {
|
||||
return &v
|
||||
}
|
||||
|
||||
func cloneInt(value *int) *int {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
clone := *value
|
||||
return &clone
|
||||
}
|
||||
|
||||
func cloneInt64(value *int64) *int64 {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
clone := *value
|
||||
return &clone
|
||||
}
|
||||
|
||||
func cloneSnapshotConfig(cfg *SnapshotAlertConfig) *SnapshotAlertConfig {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
@@ -675,6 +691,16 @@ func cloneThresholdConfig(cfg ThresholdConfig) ThresholdConfig {
|
||||
clone.NetworkOut = cloneThreshold(cfg.NetworkOut)
|
||||
clone.Temperature = cloneThreshold(cfg.Temperature)
|
||||
clone.DiskTemperature = cloneThreshold(cfg.DiskTemperature)
|
||||
clone.SMARTHealthFailure = cloneInt(cfg.SMARTHealthFailure)
|
||||
clone.SMARTReallocated = cloneInt64(cfg.SMARTReallocated)
|
||||
clone.SMARTPending = cloneInt64(cfg.SMARTPending)
|
||||
clone.SMARTUncorrectable = cloneInt64(cfg.SMARTUncorrectable)
|
||||
clone.SMARTMediaErrors = cloneInt64(cfg.SMARTMediaErrors)
|
||||
clone.SMARTCRCErrorDelta = cloneInt64(cfg.SMARTCRCErrorDelta)
|
||||
clone.SMARTLifeWarning = cloneInt(cfg.SMARTLifeWarning)
|
||||
clone.SMARTLifeCritical = cloneInt(cfg.SMARTLifeCritical)
|
||||
clone.SMARTSpareWarning = cloneInt(cfg.SMARTSpareWarning)
|
||||
clone.SMARTSpareCritical = cloneInt(cfg.SMARTSpareCritical)
|
||||
clone.Usage = cloneThreshold(cfg.Usage)
|
||||
clone.Backup = cloneBackupConfig(cfg.Backup)
|
||||
clone.Snapshot = cloneSnapshotConfig(cfg.Snapshot)
|
||||
@@ -728,6 +754,36 @@ func (m *Manager) applyThresholdOverride(base ThresholdConfig, override Threshol
|
||||
if override.DiskTemperature != nil {
|
||||
result.DiskTemperature = ensureHysteresisThreshold(cloneThreshold(override.DiskTemperature))
|
||||
}
|
||||
if override.SMARTHealthFailure != nil {
|
||||
result.SMARTHealthFailure = cloneInt(override.SMARTHealthFailure)
|
||||
}
|
||||
if override.SMARTReallocated != nil {
|
||||
result.SMARTReallocated = cloneInt64(override.SMARTReallocated)
|
||||
}
|
||||
if override.SMARTPending != nil {
|
||||
result.SMARTPending = cloneInt64(override.SMARTPending)
|
||||
}
|
||||
if override.SMARTUncorrectable != nil {
|
||||
result.SMARTUncorrectable = cloneInt64(override.SMARTUncorrectable)
|
||||
}
|
||||
if override.SMARTMediaErrors != nil {
|
||||
result.SMARTMediaErrors = cloneInt64(override.SMARTMediaErrors)
|
||||
}
|
||||
if override.SMARTCRCErrorDelta != nil {
|
||||
result.SMARTCRCErrorDelta = cloneInt64(override.SMARTCRCErrorDelta)
|
||||
}
|
||||
if override.SMARTLifeWarning != nil {
|
||||
result.SMARTLifeWarning = cloneInt(override.SMARTLifeWarning)
|
||||
}
|
||||
if override.SMARTLifeCritical != nil {
|
||||
result.SMARTLifeCritical = cloneInt(override.SMARTLifeCritical)
|
||||
}
|
||||
if override.SMARTSpareWarning != nil {
|
||||
result.SMARTSpareWarning = cloneInt(override.SMARTSpareWarning)
|
||||
}
|
||||
if override.SMARTSpareCritical != nil {
|
||||
result.SMARTSpareCritical = cloneInt(override.SMARTSpareCritical)
|
||||
}
|
||||
if override.Usage != nil {
|
||||
result.Usage = ensureHysteresisThreshold(cloneThreshold(override.Usage))
|
||||
}
|
||||
|
||||
@@ -23,10 +23,20 @@ func defaultAlertConfig() AlertConfig {
|
||||
Temperature: &HysteresisThreshold{Trigger: 80, Clear: 75}, // Warning at 80°C, clear at 75°C
|
||||
},
|
||||
AgentDefaults: ThresholdConfig{
|
||||
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
||||
DiskTemperature: &HysteresisThreshold{Trigger: 55, Clear: 50},
|
||||
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
|
||||
DiskTemperature: &HysteresisThreshold{Trigger: 55, Clear: 50},
|
||||
SMARTHealthFailure: defaultIntPtr(1),
|
||||
SMARTReallocated: defaultInt64Ptr(1),
|
||||
SMARTPending: defaultInt64Ptr(1),
|
||||
SMARTUncorrectable: defaultInt64Ptr(1),
|
||||
SMARTMediaErrors: defaultInt64Ptr(1),
|
||||
SMARTCRCErrorDelta: defaultInt64Ptr(1),
|
||||
SMARTLifeWarning: defaultIntPtr(10),
|
||||
SMARTLifeCritical: defaultIntPtr(5),
|
||||
SMARTSpareWarning: defaultIntPtr(20),
|
||||
SMARTSpareCritical: defaultIntPtr(10),
|
||||
},
|
||||
DockerDefaults: DockerThresholdConfig{
|
||||
CPU: HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
@@ -191,3 +201,7 @@ func defaultAlertConfig() AlertConfig {
|
||||
FlappingCooldownMinutes: 15, // 15 minute cooldown
|
||||
}
|
||||
}
|
||||
|
||||
func defaultIntPtr(value int) *int { return &value }
|
||||
|
||||
func defaultInt64Ptr(value int64) *int64 { return &value }
|
||||
|
||||
+31
-6
@@ -393,7 +393,7 @@ func (m *Manager) CheckHost(host models.Host) {
|
||||
diskResourceID, diskName := hostSMARTDiskResourceID(host, disk)
|
||||
if host.LinkedNodeID == "" {
|
||||
seenDisks[diskResourceID] = struct{}{}
|
||||
m.syncHostSMARTDiskRiskAlerts(host, disk, diskResourceID, diskName, nodeName, instanceName, baseMetadata)
|
||||
m.syncHostSMARTDiskRiskAlerts(host, disk, diskResourceID, diskName, nodeName, instanceName, baseMetadata, thresholds)
|
||||
continue
|
||||
}
|
||||
m.syncHostSMARTDiskAlert(host, disk, diskResourceID, diskName, nodeName, instanceName, baseMetadata, "disk-health", nil)
|
||||
@@ -1141,9 +1141,20 @@ func (m *Manager) cleanupHostDiskAlerts(host models.Host, seen map[string]struct
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) syncHostSMARTDiskRiskAlerts(host models.Host, disk models.HostDiskSMART, resourceID, resourceName, nodeName, instanceName string, baseMetadata map[string]interface{}) {
|
||||
assessment := storagehealth.AssessHostSMARTDisk(disk)
|
||||
assessment.Reasons = append(assessment.Reasons, m.hostSMARTCounterGrowthReasons(resourceID, disk)...)
|
||||
func (m *Manager) syncHostSMARTDiskRiskAlerts(host models.Host, disk models.HostDiskSMART, resourceID, resourceName, nodeName, instanceName string, baseMetadata map[string]interface{}, thresholds ThresholdConfig) {
|
||||
smartThresholds := storagehealth.SMARTThresholds{
|
||||
HealthFailure: intValue(thresholds.SMARTHealthFailure) > 0,
|
||||
ReallocatedSectors: int64Value(thresholds.SMARTReallocated),
|
||||
PendingSectors: int64Value(thresholds.SMARTPending),
|
||||
OfflineUncorrectable: int64Value(thresholds.SMARTUncorrectable),
|
||||
MediaErrors: int64Value(thresholds.SMARTMediaErrors),
|
||||
LifeWarning: intValue(thresholds.SMARTLifeWarning),
|
||||
LifeCritical: intValue(thresholds.SMARTLifeCritical),
|
||||
AvailableSpareWarn: intValue(thresholds.SMARTSpareWarning),
|
||||
AvailableSpareCrit: intValue(thresholds.SMARTSpareCritical),
|
||||
}
|
||||
assessment := storagehealth.AssessHostSMARTDiskWithThresholds(disk, smartThresholds)
|
||||
assessment.Reasons = append(assessment.Reasons, m.hostSMARTCounterGrowthReasons(resourceID, disk, int64Value(thresholds.SMARTCRCErrorDelta))...)
|
||||
healthReasons, wearReasons := splitSMARTAlertReasons(assessment.Reasons)
|
||||
|
||||
m.syncHostSMARTDiskAlert(host, disk, resourceID, resourceName, nodeName, instanceName, baseMetadata, "disk-health", healthReasons)
|
||||
@@ -1154,7 +1165,7 @@ func (m *Manager) syncHostSMARTDiskRiskAlerts(host models.Host, disk models.Host
|
||||
// alertable health reason without warning on an old, non-zero counter at first
|
||||
// observation. A stable value clears the transient growth reason on the next
|
||||
// report, while the alert and notification histories retain the event.
|
||||
func (m *Manager) hostSMARTCounterGrowthReasons(resourceID string, disk models.HostDiskSMART) []storagehealth.Reason {
|
||||
func (m *Manager) hostSMARTCounterGrowthReasons(resourceID string, disk models.HostDiskSMART, minimumDelta int64) []storagehealth.Reason {
|
||||
if disk.Attributes == nil || disk.Attributes.UDMACRCErrors == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -1173,7 +1184,7 @@ func (m *Manager) hostSMARTCounterGrowthReasons(resourceID string, disk models.H
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
if !observed || current <= previous.UDMACRCErrors {
|
||||
if !observed || minimumDelta <= 0 || current-previous.UDMACRCErrors < minimumDelta {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1184,6 +1195,20 @@ func (m *Manager) hostSMARTCounterGrowthReasons(resourceID string, disk models.H
|
||||
}}
|
||||
}
|
||||
|
||||
func intValue(value *int) int {
|
||||
if value == nil {
|
||||
return 0
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func int64Value(value *int64) int64 {
|
||||
if value == nil {
|
||||
return 0
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func splitSMARTAlertReasons(reasons []storagehealth.Reason) ([]storagehealth.Reason, []storagehealth.Reason) {
|
||||
healthReasons := make([]storagehealth.Reason, 0, len(reasons))
|
||||
wearReasons := make([]storagehealth.Reason, 0, len(reasons))
|
||||
|
||||
@@ -28,6 +28,35 @@ type Assessment struct {
|
||||
Reasons []Reason `json:"reasons,omitempty"`
|
||||
}
|
||||
|
||||
// SMARTThresholds controls the discrete health evidence that becomes a disk
|
||||
// risk. Counter values at zero disable that rule; the endurance percentages
|
||||
// use the same convention. Temperature remains a separate metric threshold.
|
||||
type SMARTThresholds struct {
|
||||
HealthFailure bool
|
||||
ReallocatedSectors int64
|
||||
PendingSectors int64
|
||||
OfflineUncorrectable int64
|
||||
MediaErrors int64
|
||||
LifeWarning int
|
||||
LifeCritical int
|
||||
AvailableSpareWarn int
|
||||
AvailableSpareCrit int
|
||||
}
|
||||
|
||||
func DefaultSMARTThresholds() SMARTThresholds {
|
||||
return SMARTThresholds{
|
||||
HealthFailure: true,
|
||||
ReallocatedSectors: 1,
|
||||
PendingSectors: 1,
|
||||
OfflineUncorrectable: 1,
|
||||
MediaErrors: 1,
|
||||
LifeWarning: 10,
|
||||
LifeCritical: 5,
|
||||
AvailableSpareWarn: 20,
|
||||
AvailableSpareCrit: 10,
|
||||
}
|
||||
}
|
||||
|
||||
type Sample struct {
|
||||
Model string
|
||||
Health string
|
||||
@@ -60,6 +89,10 @@ func AssessPhysicalDisk(disk models.PhysicalDisk) Assessment {
|
||||
}
|
||||
|
||||
func AssessHostSMARTDisk(disk models.HostDiskSMART) Assessment {
|
||||
return AssessHostSMARTDiskWithThresholds(disk, DefaultSMARTThresholds())
|
||||
}
|
||||
|
||||
func AssessHostSMARTDiskWithThresholds(disk models.HostDiskSMART, thresholds SMARTThresholds) Assessment {
|
||||
sample := Sample{
|
||||
Model: disk.Model,
|
||||
Health: disk.Health,
|
||||
@@ -67,7 +100,7 @@ func AssessHostSMARTDisk(disk models.HostDiskSMART) Assessment {
|
||||
Wearout: -1,
|
||||
}
|
||||
applySMARTAttributes(&sample, disk.Attributes)
|
||||
return AssessSample(sample)
|
||||
return AssessSampleWithThresholds(sample, thresholds)
|
||||
}
|
||||
|
||||
func applySMARTAttributes(sample *Sample, attrs *models.SMARTAttributes) {
|
||||
@@ -115,6 +148,10 @@ func applySMARTAttributes(sample *Sample, attrs *models.SMARTAttributes) {
|
||||
}
|
||||
|
||||
func AssessSample(sample Sample) Assessment {
|
||||
return AssessSampleWithThresholds(sample, DefaultSMARTThresholds())
|
||||
}
|
||||
|
||||
func AssessSampleWithThresholds(sample Sample, thresholds SMARTThresholds) Assessment {
|
||||
assessment := Assessment{Level: RiskHealthy}
|
||||
addReason := func(code string, severity RiskLevel, summary string) {
|
||||
if summary == "" {
|
||||
@@ -131,31 +168,33 @@ func AssessSample(sample Sample) Assessment {
|
||||
}
|
||||
|
||||
normalizedHealth := normalizeHealth(sample.Health)
|
||||
if normalizedHealth != "" && normalizedHealth != "UNKNOWN" && normalizedHealth != "PASSED" && normalizedHealth != "OK" && !HasKnownFirmwareBug(sample.Model) {
|
||||
if thresholds.HealthFailure && normalizedHealth != "" && normalizedHealth != "UNKNOWN" && normalizedHealth != "PASSED" && normalizedHealth != "OK" && !HasKnownFirmwareBug(sample.Model) {
|
||||
addReason("health_status", RiskCritical, fmt.Sprintf("Disk reports health status %s", normalizedHealth))
|
||||
}
|
||||
if sample.PendingSectors > 0 {
|
||||
if thresholds.PendingSectors > 0 && sample.PendingSectors >= thresholds.PendingSectors {
|
||||
addReason("pending_sectors", RiskCritical, fmt.Sprintf("Pending sectors detected (%d)", sample.PendingSectors))
|
||||
}
|
||||
if sample.OfflineUncorrectable > 0 {
|
||||
if thresholds.OfflineUncorrectable > 0 && sample.OfflineUncorrectable >= thresholds.OfflineUncorrectable {
|
||||
addReason("offline_uncorrectable", RiskCritical, fmt.Sprintf("Offline uncorrectable sectors detected (%d)", sample.OfflineUncorrectable))
|
||||
}
|
||||
if sample.MediaErrors > 0 {
|
||||
if thresholds.MediaErrors > 0 && sample.MediaErrors >= thresholds.MediaErrors {
|
||||
addReason("media_errors", RiskCritical, fmt.Sprintf("Media errors detected (%d)", sample.MediaErrors))
|
||||
}
|
||||
if (sample.WearoutKnown || sample.Wearout > 0) && sample.Wearout >= 0 && sample.Wearout <= 5 {
|
||||
if thresholds.LifeCritical > 0 && (sample.WearoutKnown || sample.Wearout > 0) && sample.Wearout >= 0 && sample.Wearout <= thresholds.LifeCritical {
|
||||
addReason("wearout_low", RiskCritical, fmt.Sprintf("SSD life remaining is %d%%", sample.Wearout))
|
||||
} else if sample.Wearout > 5 && sample.Wearout < 10 {
|
||||
} else if thresholds.LifeWarning > 0 && sample.Wearout > thresholds.LifeCritical && sample.Wearout < thresholds.LifeWarning {
|
||||
addReason("wearout_low", RiskWarning, fmt.Sprintf("SSD life remaining is %d%%", sample.Wearout))
|
||||
}
|
||||
if (sample.AvailableSpareKnown || sample.AvailableSpare > 0) && sample.AvailableSpare <= 10 {
|
||||
if thresholds.AvailableSpareCrit > 0 && (sample.AvailableSpareKnown || sample.AvailableSpare > 0) && sample.AvailableSpare <= thresholds.AvailableSpareCrit {
|
||||
addReason("nvme_available_spare_low", RiskCritical, fmt.Sprintf("NVMe available spare is %d%%", sample.AvailableSpare))
|
||||
} else if sample.AvailableSpare > 0 && sample.AvailableSpare < 20 {
|
||||
} else if thresholds.AvailableSpareWarn > 0 && sample.AvailableSpare > thresholds.AvailableSpareCrit && sample.AvailableSpare < thresholds.AvailableSpareWarn {
|
||||
addReason("nvme_available_spare_low", RiskWarning, fmt.Sprintf("NVMe available spare is %d%%", sample.AvailableSpare))
|
||||
}
|
||||
if sample.PercentageUsed >= 95 {
|
||||
percentageCritical := thresholds.LifeCritical > 0 && sample.PercentageUsed >= 100-thresholds.LifeCritical
|
||||
percentageWarning := thresholds.LifeWarning > 0 && sample.PercentageUsed >= 100-thresholds.LifeWarning
|
||||
if percentageCritical {
|
||||
addReason("nvme_percentage_used_high", RiskCritical, fmt.Sprintf("NVMe endurance used is %d%%", sample.PercentageUsed))
|
||||
} else if sample.PercentageUsed >= 90 {
|
||||
} else if percentageWarning {
|
||||
addReason("nvme_percentage_used_high", RiskWarning, fmt.Sprintf("NVMe endurance used is %d%%", sample.PercentageUsed))
|
||||
}
|
||||
if sample.Temperature >= 70 {
|
||||
@@ -163,7 +202,7 @@ func AssessSample(sample Sample) Assessment {
|
||||
} else if sample.Temperature >= 60 {
|
||||
addReason("temperature_high", RiskWarning, fmt.Sprintf("Disk temperature is %dC", sample.Temperature))
|
||||
}
|
||||
if sample.ReallocatedSectors > 0 {
|
||||
if thresholds.ReallocatedSectors > 0 && sample.ReallocatedSectors >= thresholds.ReallocatedSectors {
|
||||
addReason("reallocated_sectors", RiskWarning, fmt.Sprintf("Reallocated sectors detected (%d)", sample.ReallocatedSectors))
|
||||
}
|
||||
if sample.UDMACRCErrors > 0 {
|
||||
|
||||
@@ -22,6 +22,35 @@ func TestAssessSample_HealthyDisk(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssessSampleWithThresholdsTunesAndDisablesSMARTRules(t *testing.T) {
|
||||
thresholds := DefaultSMARTThresholds()
|
||||
thresholds.HealthFailure = false
|
||||
thresholds.PendingSectors = 3
|
||||
thresholds.MediaErrors = 0
|
||||
thresholds.LifeWarning = 20
|
||||
thresholds.LifeCritical = 10
|
||||
|
||||
assessment := AssessSampleWithThresholds(Sample{
|
||||
Health: "FAILED",
|
||||
PendingSectors: 2,
|
||||
MediaErrors: 50,
|
||||
Wearout: 15,
|
||||
WearoutKnown: true,
|
||||
}, thresholds)
|
||||
|
||||
if assessment.Level != RiskWarning {
|
||||
t.Fatalf("Level = %q, want warning from configured life threshold", assessment.Level)
|
||||
}
|
||||
if len(assessment.Reasons) != 1 || assessment.Reasons[0].Code != "wearout_low" {
|
||||
t.Fatalf("Reasons = %+v, want only wearout_low", assessment.Reasons)
|
||||
}
|
||||
|
||||
assessment = AssessSampleWithThresholds(Sample{PendingSectors: 3}, thresholds)
|
||||
if assessment.Level != RiskCritical || len(assessment.Reasons) != 1 || assessment.Reasons[0].Code != "pending_sectors" {
|
||||
t.Fatalf("threshold boundary did not trigger: %+v", assessment)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssessSample_FailedHealthStatus(t *testing.T) {
|
||||
assessment := AssessSample(Sample{
|
||||
Health: "FAILED",
|
||||
|
||||
Reference in New Issue
Block a user