gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-08-28 11:17:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(notifications): replace polling with Docker event stream for container lifecycle detection (#588)

Browse Source 
* fix(notifications): replace polling with Docker event stream for container lifecycle detection

Replaces the 30-second MonitorService crash-detection poll with a causal,
per-node Docker events stream. Eliminates false crash alerts on intentional
stops (docker stop, compose down, stack restart/update), detects OOM kills
as a distinct alert category, and surfaces real crashes in real time.

A new DockerEventManager spawns one DockerEventService per local node. Each
service consumes the filtered container event stream, classifies die events
against recent kill/oom state, and reconciles container state via snapshot
diffing on connect and reconnect. Rate limiting, exponential backoff with
jitter, and parse-error tolerance keep the stream resilient under load and
during daemon interruptions.

MonitorService retains host limits, janitor, version check, and stack metric
alerts; crash and healthcheck detection move out entirely.

* fix(tests): silence require-imports lint in hoisted mock factory
This commit is contained in:
Anso
2026-04-14 13:47:48 -04:00
committed by GitHub
parent 6ffac2a0db
commit ad9a6859e6
11 changed files with 1720 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/MonitorService.ts
+6 -60
View File
@@ -69,10 +69,9 @@ export class MonitorService {
// key: rule_id, value: AlertState
private activeBreaches = new Map<number, AlertState>();
// Track containers that have already been alerted as crashed to avoid
// duplicate alerts. key: containerId, value: timestamp when alerted.
private alertedCrashes = new Map<string, number>();
private static readonly CRASH_ALERT_TTL_MS = 60 * 60 * 1000; // 1 hour
// Crash and healthcheck detection live in DockerEventService (event-driven,
// causal classification). MonitorService no longer polls for container
// exits; see backend/src/services/DockerEventService.ts.
// Sencho version check cooldown (6 hours between external API calls)
private lastVersionCheckAt = 0;
@@ -126,7 +125,6 @@ export class MonitorService {
}
private async evaluateGlobalSettings(settings: Record<string, string>) {
const notifier = NotificationService.getInstance();
const HOST_ALERT_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes between repeat alerts
// 1. Host Limits
@@ -160,61 +158,9 @@ export class MonitorService {
console.error('Error checking host limits in watchdog', e);
}
// 2. Global Crash Detect
if (settings['global_crash'] === '1') {
// Prune expired entries from the crash tracker
const now = Date.now();
for (const [id, ts] of this.alertedCrashes) {
if (now - ts > MonitorService.CRASH_ALERT_TTL_MS) this.alertedCrashes.delete(id);
}
try {
const nodes = DatabaseService.getInstance().getNodes();
const runningIds = new Set<string>();
for (const node of nodes) {
if (!node.id) continue;
// Remote nodes run their own MonitorService locally
if (node.type === 'remote') continue;
try {
const docker = DockerController.getInstance(node.id);
const containers = await docker.getAllContainers();
for (const c of containers) {
if (c.State === 'running') {
runningIds.add(c.Id);
continue;
}
// Skip containers already alerted
if (this.alertedCrashes.has(c.Id)) continue;
const containerStack = c.Labels?.['com.docker.compose.project'] || undefined;
if (c.State === 'exited') {
const match = c.Status.match(/Exited \((\d+)\)/i);
const exitCode = match ? parseInt(match[1], 10) : null;
const intentionalExitCodes = [0, 137, 143, 255];
if (exitCode !== null && !intentionalExitCodes.includes(exitCode)) {
await notifier.dispatchAlert('error', `[Node: ${node.name}] Container Crash Detected: ${c.Names[0]} exited unexpectedly (Code: ${exitCode}).`, containerStack);
this.alertedCrashes.set(c.Id, now);
}
} else if (String(c.Status).includes('unhealthy')) {
await notifier.dispatchAlert('error', `[Node: ${node.name}] Healthcheck Failed: Container ${c.Names[0]} is unhealthy.`, containerStack);
this.alertedCrashes.set(c.Id, now);
}
}
} catch (err) {
console.error(`Error checking crashes on node ${node.name}`, err);
}
}
// Clear crash tracking for containers that are running again
for (const id of this.alertedCrashes.keys()) {
if (runningIds.has(id)) this.alertedCrashes.delete(id);
}
} catch (e) {
console.error('Error checking global crashes', e);
}
}
// 2. (Removed) Container crash + healthcheck detection moved to
// DockerEventService: event-driven, causal, distinguishes
// intentional stops from real crashes, detects OOM kills.
// 3. Docker Janitor Check
try {