mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-10 10:49:35 +00:00
fix(monitor): collapse repeated host-metric alerts into per-window summary (F-11) (#1175)
* fix(monitor): collapse repeated host-metric alerts into per-window summary (F-11) A host metric over threshold previously dispatched one notification every 5 minutes for the duration of the breach, producing 7+ identical messages in 35 minutes and spamming Discord/Slack routes. Replace the hardcoded 5-minute cooldown for CPU/RAM/disk with a per-metric suppression window (default 60 min, configurable via host_alert_suppression_mins). The first breach fires immediately; subsequent cycles within the window are silently counted; the next dispatch after the window elapses carries a summary suffix listing how many cycles were suppressed and when the breach first crossed threshold. Recovery clears the counter so re-breach fires fresh. The pattern mirrors PolicyEnforcement.notifyTrivyMissingOnce: module-scope Map, in-memory only, in-cycle dedup, with a test-reset helper. The existing system_state row keeps post-restart re-fires bounded. Janitor and per-stack alert rules are unchanged; they already have adequate cadence and per-rule cooldown respectively. * fix(ci): restore backend and frontend checks * fix(e2e): remove create button timing race * fix(e2e): harden create double-click test * fix(monitor): clear persisted F-11 timestamp on recovery + clamp suppression window Independent audit on the previous commit surfaced two issues. 1. clearHostMetricSuppression early-returned on missing in-memory state, leaving a stale system_state.last_host_*_alert_ts row alive after a process restart. Scenario: breach fires + persists timestamp, process restarts, metric recovers before another evaluate cycle re-seeds the in-memory Map, recovery cleanup early-returns. Next re-breach inside the original window hits the restart-survivability branch and is silently suppressed instead of firing fresh. Fix: read persisted state in clearHostMetricSuppression and reset to '0' independently of in-memory presence. The read-before-write also skips redundant writes when the row is already cleared. 2. host_alert_suppression_mins is validated by zod on the bulk PATCH path but the single-key POST /api/settings path accepts allowlisted keys without re-validation. A 999999999-minute value would silence host alerts for centuries. Add MAX_HOST_ALERT_SUPPRESSION_MIN = 1440 mirroring the zod max, and clamp via Math.min in evaluateGlobalSettings. Two new vitest cases (restart-then-recovery-then-rebreach; the 1440 clamp) confirmed failing before the fix, passing after. The existing "metric drop" case updated to use a mock-backed persistence pattern consistent with the new restart-scenario tests. 73/73 monitor-service tests green; full backend suite 2507/2510 (same pre-existing Windows EBUSY flake on filesystem-backup.test.ts as baseline).
This commit is contained in:
@@ -132,12 +132,13 @@ function SettingsSkeleton() {
|
||||
);
|
||||
}
|
||||
|
||||
type SystemFields = Pick<PatchableSettings, 'host_cpu_limit' | 'host_ram_limit' | 'host_disk_limit' | 'docker_janitor_gb' | 'global_crash'>;
|
||||
type SystemFields = Pick<PatchableSettings, 'host_cpu_limit' | 'host_ram_limit' | 'host_disk_limit' | 'host_alert_suppression_mins' | 'docker_janitor_gb' | 'global_crash'>;
|
||||
|
||||
const DEFAULT_SYSTEM: SystemFields = {
|
||||
host_cpu_limit: DEFAULT_SETTINGS.host_cpu_limit,
|
||||
host_ram_limit: DEFAULT_SETTINGS.host_ram_limit,
|
||||
host_disk_limit: DEFAULT_SETTINGS.host_disk_limit,
|
||||
host_alert_suppression_mins: DEFAULT_SETTINGS.host_alert_suppression_mins,
|
||||
docker_janitor_gb: DEFAULT_SETTINGS.docker_janitor_gb,
|
||||
global_crash: DEFAULT_SETTINGS.global_crash,
|
||||
};
|
||||
@@ -155,6 +156,7 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) {
|
||||
if (settings.host_cpu_limit !== baseline.host_cpu_limit) n++;
|
||||
if (settings.host_ram_limit !== baseline.host_ram_limit) n++;
|
||||
if (settings.host_disk_limit !== baseline.host_disk_limit) n++;
|
||||
if (settings.host_alert_suppression_mins !== baseline.host_alert_suppression_mins) n++;
|
||||
if (settings.docker_janitor_gb !== baseline.docker_janitor_gb) n++;
|
||||
if (settings.global_crash !== baseline.global_crash) n++;
|
||||
return n;
|
||||
@@ -188,6 +190,7 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) {
|
||||
host_cpu_limit: nodeData.host_cpu_limit ?? DEFAULT_SETTINGS.host_cpu_limit,
|
||||
host_ram_limit: nodeData.host_ram_limit ?? DEFAULT_SETTINGS.host_ram_limit,
|
||||
host_disk_limit: nodeData.host_disk_limit ?? DEFAULT_SETTINGS.host_disk_limit,
|
||||
host_alert_suppression_mins: nodeData.host_alert_suppression_mins ?? DEFAULT_SETTINGS.host_alert_suppression_mins,
|
||||
docker_janitor_gb: nodeData.docker_janitor_gb ?? DEFAULT_SETTINGS.docker_janitor_gb,
|
||||
global_crash: (nodeData.global_crash as '0' | '1') ?? DEFAULT_SETTINGS.global_crash,
|
||||
};
|
||||
@@ -235,7 +238,7 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) {
|
||||
<SettingsSection title="Host thresholds">
|
||||
<SettingsField
|
||||
label="CPU limit"
|
||||
helper="Alerts fire when the 5-minute average exceeds this percentage."
|
||||
helper="Alerts fire when host CPU utilization exceeds this percentage."
|
||||
>
|
||||
<NumberChip
|
||||
value={settings.host_cpu_limit || '90'}
|
||||
@@ -272,6 +275,18 @@ export function SystemSection({ onDirtyChange }: SystemSectionProps) {
|
||||
warnOver={95}
|
||||
/>
|
||||
</SettingsField>
|
||||
<SettingsField
|
||||
label="Alert suppression"
|
||||
helper="How long to wait before resending a host alert while the metric stays over threshold. The follow-up message includes a count of suppressed cycles."
|
||||
>
|
||||
<NumberChip
|
||||
value={settings.host_alert_suppression_mins || '60'}
|
||||
onChange={(v) => onSettingChange('host_alert_suppression_mins', v)}
|
||||
suffix="min"
|
||||
min={1}
|
||||
max={1440}
|
||||
/>
|
||||
</SettingsField>
|
||||
</SettingsSection>
|
||||
|
||||
<SettingsSection title="Docker hygiene">
|
||||
|
||||
@@ -2,6 +2,7 @@ export interface PatchableSettings {
|
||||
host_cpu_limit?: string;
|
||||
host_ram_limit?: string;
|
||||
host_disk_limit?: string;
|
||||
host_alert_suppression_mins?: string;
|
||||
docker_janitor_gb?: string;
|
||||
global_crash?: '0' | '1';
|
||||
developer_mode?: '0' | '1';
|
||||
@@ -15,6 +16,7 @@ export const DEFAULT_SETTINGS: PatchableSettings = {
|
||||
host_cpu_limit: '90',
|
||||
host_ram_limit: '90',
|
||||
host_disk_limit: '90',
|
||||
host_alert_suppression_mins: '60',
|
||||
global_crash: '1',
|
||||
docker_janitor_gb: '5',
|
||||
developer_mode: '0',
|
||||
|
||||
Reference in New Issue
Block a user