mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-31 06:02:22 +00:00
Anso
44a89d9d2e
* fix(docker-events): harden crash detection against edge cases - Isolate per-container failures in the reconcile path so one failed container inspect cannot abort classification for the rest of the batch after a reconnect. - Fall back to inspecting State.OOMKilled when a container exits with code 137 and no oom event preceded the die, so cgroup OOM kills that lose the oom event are still classified correctly. The dedup check runs before the fallback so crashloops do not hammer the daemon. - Guard the intentional-kill window against wildly future-dated Docker timestamps by switching to a signed age comparison with a bounded negative-skew tolerance, so clock skew cannot flip a genuine crash into an intentional stop. - Align diagnostic logging with the codebase [Service:diag] convention and add one informational lifecycle log on boot and shutdown so operators running in developer mode can confirm the watcher started. - New tests cover gap-inspect isolation, the OOM inspect fallback (both success and failure paths), duplicate die events collapsing within the grace window, and clock-skew bounds on the classifier. * docs(alerts): add troubleshooting entries for crash detection toggle and rate limits
83 lines
3.0 KiB
TypeScript
83 lines
3.0 KiB
TypeScript
/**
|
|
* ContainerLifecycleClassifier
|
|
*
|
|
* Pure classification helpers for Docker container lifecycle events. No I/O,
|
|
* no side effects, no singletons. Consumed by DockerEventService.
|
|
*
|
|
* The classifier answers a single question: given a `die` event and the
|
|
* container's recent lifecycle state, is this exit intentional, a clean exit,
|
|
* an OOM kill, or a crash worth alerting on?
|
|
*/
|
|
|
|
export type Classification = 'intentional' | 'clean' | 'crash' | 'oom';
|
|
|
|
/** Window (ms) after a `kill` event within which a subsequent `die` is considered intentional. */
|
|
export const INTENTIONAL_KILL_WINDOW_MS = 60_000;
|
|
|
|
/**
|
|
* Maximum negative age (ms) tolerated when matching a kill to a later die.
|
|
* Absorbs small out-of-order deliveries and minor clock skew, but rejects
|
|
* wildly future-dated Docker timestamps that could otherwise flip a genuine
|
|
* crash into an "intentional" classification.
|
|
*/
|
|
export const MAX_NEGATIVE_SKEW_MS = 10_000;
|
|
|
|
export interface ContainerLifecycleState {
|
|
/** Timestamp (ms) of the most recent `kill` event for this container, if any. */
|
|
lastKillAt?: number;
|
|
/** True when an `oom` event has been observed and the matching `die` has not yet arrived. */
|
|
oomPending?: boolean;
|
|
}
|
|
|
|
export interface DieEventInput {
|
|
/** Time the die event occurred (ms). Typically Date.now() when the event was parsed. */
|
|
at: number;
|
|
/** Exit code reported by Docker. May be undefined for malformed events (treated as non-zero). */
|
|
exitCode: number | undefined;
|
|
}
|
|
|
|
/**
|
|
* Classify a die event against the container's current lifecycle state.
|
|
*
|
|
* Priority order:
|
|
* 1. OOM pending → 'oom' (OOM kills are meaningful even if exitCode looks clean)
|
|
* 2. Recent kill within window → 'intentional'
|
|
* 3. Exit code 0 → 'clean'
|
|
* 4. Anything else → 'crash'
|
|
*/
|
|
export function classifyDie(
|
|
input: DieEventInput,
|
|
state: ContainerLifecycleState,
|
|
): Classification {
|
|
if (state.oomPending) return 'oom';
|
|
|
|
if (typeof state.lastKillAt === 'number') {
|
|
// Use signed age so a die with a wildly future-dated timestamp
|
|
// (clock skew / bad container clock) cannot match a past kill.
|
|
// Allow modest negative skew so out-of-order deliveries still
|
|
// classify as intentional within DockerEventService's grace window.
|
|
const age = input.at - state.lastKillAt;
|
|
if (age >= -MAX_NEGATIVE_SKEW_MS && age <= INTENTIONAL_KILL_WINDOW_MS) {
|
|
return 'intentional';
|
|
}
|
|
}
|
|
|
|
if (input.exitCode === 0) return 'clean';
|
|
return 'crash';
|
|
}
|
|
|
|
/**
|
|
* Classify a gap exit discovered during reconciliation (no die event observed
|
|
* because the stream was disconnected). Uses the container inspect result
|
|
* rather than event state.
|
|
*/
|
|
export function classifyGapExit(inspect: {
|
|
State?: { OOMKilled?: boolean; ExitCode?: number };
|
|
}): Classification {
|
|
const oom = inspect.State?.OOMKilled === true;
|
|
if (oom) return 'oom';
|
|
const code = inspect.State?.ExitCode;
|
|
if (code === 0) return 'clean';
|
|
return 'crash';
|
|
}
|