mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-19 14:56:27 +00:00
feat: auto-heal policies for unhealthy containers (#671)
* feat(db): add auto_heal_policies and auto_heal_history schema and CRUD Adds two new SQLite tables (auto_heal_policies, auto_heal_history) to DatabaseService.initSchema() and exposes CRUD methods: getAutoHealPolicies, getAutoHealPolicy, addAutoHealPolicy, updateAutoHealPolicy, deleteAutoHealPolicy, recordAutoHealHistory, getAutoHealHistory, incrementConsecutiveFailures, resetConsecutiveFailures, setPolicyEnabled. Also adds AutoHealPolicy and AutoHealHistoryEntry TypeScript interfaces. * feat(events): track health-status duration and expose state accessors - Add healthStatus and unhealthySince fields to InternalContainerState - onHealthStatus now records unhealthySince timestamp on first transition to unhealthy, and clears it when the container recovers or restarts - onStart resets both fields so a restarted container begins from 'starting' - Add listContainerStates() and getContainerState() public accessors for use by the upcoming AutoHealService evaluator * fix(auto-heal): key allowlist in updateAutoHealPolicy, cascade delete, extract ContainerHealthSnapshot * feat: add AutoHealService evaluator singleton Polls every 30 s, matches containers to enabled policies via Compose labels, and restarts containers that have been unhealthy beyond the configured threshold. Enforces cooldown, per-hour rate cap, and recent-user-action suppression; auto-disables policies after repeated consecutive failures. Also adds DockerEventManager.getService() accessor required by the evaluator. * fix(auto-heal): prune stale restartTimestamps, guard undefined policy id - Prune restartTimestamps entries for containers no longer running after each container list fetch, preventing unbounded map growth from dead container IDs. - Guard against policies with undefined id at the start of the per-policy loop; warn and skip rather than proceed with a non-null assertion. - Extract handleAutoDisable private helper to bring executeHeal under 30 lines and isolate the auto-disable side-effect sequence. - Move ContainerInfo type to module scope. * feat: add auto-heal API routes and wire AutoHealService lifecycle Registers five REST endpoints under /api/auto-heal/policies (list, create, patch, delete, history) with requirePaid + requireAdmin guards and Zod validation. Wires AutoHealService.start()/stop() into the server startup and graceful-shutdown blocks alongside MonitorService. * test: add AutoHealService and DatabaseService auto-heal unit tests - 15 unit tests for AutoHealService.shouldHeal covering all decision branches (healthy state, duration threshold, user-action suppression, cooldown, rate limiting, and correct skipReason values) - 13 integration tests for DatabaseService auto-heal CRUD: policy round-trip, stack-name filter, partial update, cascade delete, history ordering/limit, consecutive failure counters, and setPolicyEnabled toggle * fix: log AutoHealService shutdown errors consistently * fix(api): requireAdmin-first guard order and try/catch on auto-heal routes * feat(ui): add StackAutoHealSheet component * feat(ui): add Auto-Heal context menu item to EditorLayout * fix(ui): StackAutoHealSheet label, token, a11y, and useEffect fixes - Rename 'All services in stack' to 'All services' in combobox options and placeholder - Replace text-green-600 with text-success design token in actionColorClass - Add htmlFor/id pairs to all four numeric form inputs for accessibility - Inline fetch logic into useEffect, removing stale closure risk and eslint-disable comment - Remove now-unused fetchPolicies and fetchServices standalone functions - Update 'Auto-disable after' label to 'Auto-disable after (failures)' for clarity - Add toast.error in policy fetch failure path; services fetch silently skips as before * docs: add auto-heal-policies feature documentation * test(e2e): add auto-heal policies CRUD spec * fix(docs): correct auto-heal-policies nav position in docs.json
This commit is contained in:
@@ -24,6 +24,16 @@ import { getErrorMessage } from '../utils/errors';
|
||||
* See docs/features/alerts-notifications.mdx for user-facing behaviour.
|
||||
*/
|
||||
|
||||
/** Snapshot of a single container's health tracking state, exposed to AutoHealService. */
|
||||
export interface ContainerHealthSnapshot {
|
||||
id: string;
|
||||
name?: string;
|
||||
stackName?: string;
|
||||
healthStatus?: 'healthy' | 'unhealthy' | 'starting';
|
||||
unhealthySince?: number;
|
||||
lastKillAt?: number;
|
||||
}
|
||||
|
||||
/** Grace window after a `die` before classifying, to absorb out-of-order kill events. */
|
||||
const DIE_GRACE_WINDOW_MS = 500;
|
||||
|
||||
@@ -61,6 +71,8 @@ interface InternalContainerState extends ContainerLifecycleState {
|
||||
stackName?: string;
|
||||
lastCrashAlertAt?: number;
|
||||
lastActivityAt: number;
|
||||
healthStatus?: 'healthy' | 'unhealthy' | 'starting';
|
||||
unhealthySince?: number;
|
||||
}
|
||||
|
||||
interface DockerEventPayload {
|
||||
@@ -400,16 +412,29 @@ export class DockerEventService {
|
||||
}
|
||||
|
||||
private onHealthStatus(id: string, action: string, event: DockerEventPayload): void {
|
||||
if (!action.includes('unhealthy')) return;
|
||||
const state = this.getOrCreateState(id, event);
|
||||
state.lastActivityAt = Date.now();
|
||||
if (!this.isCrashAlertsEnabled()) return;
|
||||
const name = state.name ?? id.slice(0, 12);
|
||||
const stackName = state.stackName;
|
||||
void this.emitError(
|
||||
`Healthcheck failed: ${name} is unhealthy.`,
|
||||
stackName,
|
||||
);
|
||||
|
||||
if (action.includes('unhealthy')) {
|
||||
if (state.healthStatus !== 'unhealthy') {
|
||||
state.unhealthySince = Date.now();
|
||||
}
|
||||
state.healthStatus = 'unhealthy';
|
||||
if (!this.isCrashAlertsEnabled()) return;
|
||||
const name = state.name ?? id.slice(0, 12);
|
||||
const stackName = state.stackName;
|
||||
void this.emitError(
|
||||
`Healthcheck failed: ${name} is unhealthy.`,
|
||||
stackName,
|
||||
);
|
||||
} else {
|
||||
state.unhealthySince = undefined;
|
||||
if (action.includes('starting')) {
|
||||
state.healthStatus = 'starting';
|
||||
} else {
|
||||
state.healthStatus = 'healthy';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private onStart(id: string): void {
|
||||
@@ -419,6 +444,8 @@ export class DockerEventService {
|
||||
state.lastKillAt = undefined;
|
||||
state.oomPending = undefined;
|
||||
state.lastCrashAlertAt = undefined;
|
||||
state.unhealthySince = undefined;
|
||||
state.healthStatus = 'starting';
|
||||
state.lastActivityAt = Date.now();
|
||||
}
|
||||
|
||||
@@ -649,4 +676,32 @@ export class DockerEventService {
|
||||
trackedContainers: this.containerState.size,
|
||||
};
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Container state accessors (used by AutoHealService)
|
||||
// ========================================================================
|
||||
|
||||
public listContainerStates(): ContainerHealthSnapshot[] {
|
||||
return Array.from(this.containerState.entries()).map(([id, s]) => ({
|
||||
id,
|
||||
name: s.name,
|
||||
stackName: s.stackName,
|
||||
healthStatus: s.healthStatus,
|
||||
unhealthySince: s.unhealthySince,
|
||||
lastKillAt: s.lastKillAt,
|
||||
}));
|
||||
}
|
||||
|
||||
public getContainerState(id: string): ContainerHealthSnapshot | undefined {
|
||||
const s = this.containerState.get(id);
|
||||
if (!s) return undefined;
|
||||
return {
|
||||
id,
|
||||
name: s.name,
|
||||
stackName: s.stackName,
|
||||
healthStatus: s.healthStatus,
|
||||
unhealthySince: s.unhealthySince,
|
||||
lastKillAt: s.lastKillAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user