Files
sencho/backend/src/services/DockerEventManager.ts
T
Anso 5bb4b01953 feat: auto-heal policies for unhealthy containers (#671)
* feat(db): add auto_heal_policies and auto_heal_history schema and CRUD

Adds two new SQLite tables (auto_heal_policies, auto_heal_history) to
DatabaseService.initSchema() and exposes CRUD methods: getAutoHealPolicies,
getAutoHealPolicy, addAutoHealPolicy, updateAutoHealPolicy,
deleteAutoHealPolicy, recordAutoHealHistory, getAutoHealHistory,
incrementConsecutiveFailures, resetConsecutiveFailures, setPolicyEnabled.
Also adds AutoHealPolicy and AutoHealHistoryEntry TypeScript interfaces.

* feat(events): track health-status duration and expose state accessors

- Add healthStatus and unhealthySince fields to InternalContainerState
- onHealthStatus now records unhealthySince timestamp on first transition
  to unhealthy, and clears it when the container recovers or restarts
- onStart resets both fields so a restarted container begins from 'starting'
- Add listContainerStates() and getContainerState() public accessors for
  use by the upcoming AutoHealService evaluator

* fix(auto-heal): key allowlist in updateAutoHealPolicy, cascade delete, extract ContainerHealthSnapshot

* feat: add AutoHealService evaluator singleton

Polls every 30 s, matches containers to enabled policies via Compose
labels, and restarts containers that have been unhealthy beyond the
configured threshold. Enforces cooldown, per-hour rate cap, and
recent-user-action suppression; auto-disables policies after repeated
consecutive failures. Also adds DockerEventManager.getService() accessor
required by the evaluator.

* fix(auto-heal): prune stale restartTimestamps, guard undefined policy id

- Prune restartTimestamps entries for containers no longer running after
  each container list fetch, preventing unbounded map growth from dead
  container IDs.
- Guard against policies with undefined id at the start of the per-policy
  loop; warn and skip rather than proceed with a non-null assertion.
- Extract handleAutoDisable private helper to bring executeHeal under 30
  lines and isolate the auto-disable side-effect sequence.
- Move ContainerInfo type to module scope.

* feat: add auto-heal API routes and wire AutoHealService lifecycle

Registers five REST endpoints under /api/auto-heal/policies (list, create,
patch, delete, history) with requirePaid + requireAdmin guards and Zod
validation. Wires AutoHealService.start()/stop() into the server startup
and graceful-shutdown blocks alongside MonitorService.

* test: add AutoHealService and DatabaseService auto-heal unit tests

- 15 unit tests for AutoHealService.shouldHeal covering all decision branches
  (healthy state, duration threshold, user-action suppression, cooldown,
  rate limiting, and correct skipReason values)
- 13 integration tests for DatabaseService auto-heal CRUD: policy round-trip,
  stack-name filter, partial update, cascade delete, history ordering/limit,
  consecutive failure counters, and setPolicyEnabled toggle

* fix: log AutoHealService shutdown errors consistently

* fix(api): requireAdmin-first guard order and try/catch on auto-heal routes

* feat(ui): add StackAutoHealSheet component

* feat(ui): add Auto-Heal context menu item to EditorLayout

* fix(ui): StackAutoHealSheet label, token, a11y, and useEffect fixes

- Rename 'All services in stack' to 'All services' in combobox options and placeholder
- Replace text-green-600 with text-success design token in actionColorClass
- Add htmlFor/id pairs to all four numeric form inputs for accessibility
- Inline fetch logic into useEffect, removing stale closure risk and eslint-disable comment
- Remove now-unused fetchPolicies and fetchServices standalone functions
- Update 'Auto-disable after' label to 'Auto-disable after (failures)' for clarity
- Add toast.error in policy fetch failure path; services fetch silently skips as before

* docs: add auto-heal-policies feature documentation

* test(e2e): add auto-heal policies CRUD spec

* fix(docs): correct auto-heal-policies nav position in docs.json
2026-04-17 22:45:06 -04:00

137 lines
5.1 KiB
TypeScript

import { DatabaseService, Node } from './DatabaseService';
import { NodeRegistry } from './NodeRegistry';
import { DockerEventService } from './DockerEventService';
import { isDebugEnabled } from '../utils/debug';
/**
* DockerEventManager
*
* Singleton coordinator that owns one DockerEventService per local node.
* Spawns services on boot for every existing local node, and reacts to
* NodeRegistry 'node-added' / 'node-removed' / 'node-updated' events to
* keep the service map in sync with the database.
*
* Remote nodes self-monitor on their own Sencho instance; this manager does
* not subscribe to remote Docker daemons.
*/
export class DockerEventManager {
private static instance: DockerEventManager;
private services: Map<number, DockerEventService> = new Map();
private started = false;
private readonly onNodeAdded = (id: number) => { void this.handleNodeAdded(id); };
private readonly onNodeRemoved = (id: number) => { this.handleNodeRemoved(id); };
private readonly onNodeUpdated = (id: number) => { void this.handleNodeUpdated(id); };
private constructor() { /* private: use getInstance */ }
public static getInstance(): DockerEventManager {
if (!DockerEventManager.instance) {
DockerEventManager.instance = new DockerEventManager();
}
return DockerEventManager.instance;
}
/** Boot: spawn a DockerEventService for every existing local node. */
public async start(): Promise<void> {
if (this.started) return;
this.started = true;
const registry = NodeRegistry.getInstance();
registry.on('node-added', this.onNodeAdded);
registry.on('node-removed', this.onNodeRemoved);
registry.on('node-updated', this.onNodeUpdated);
// Spawn in parallel so one slow node can't block boot for the others.
const nodes = DatabaseService.getInstance().getNodes()
.filter(n => n.type === 'local' && typeof n.id === 'number');
await Promise.all(nodes.map(n => this.spawn(n)));
console.log(`[DockerEvents] Started; watching ${this.services.size} local node(s) for container lifecycle events`);
}
/** Shutdown: stop every service and unsubscribe from registry events. */
public stop(): void {
if (!this.started) return;
this.started = false;
const registry = NodeRegistry.getInstance();
registry.off('node-added', this.onNodeAdded);
registry.off('node-removed', this.onNodeRemoved);
registry.off('node-updated', this.onNodeUpdated);
for (const service of this.services.values()) service.shutdown();
this.services.clear();
console.log('[DockerEvents] Stopped');
}
/** Aggregated status for diagnostics (e.g. /api/health). */
public getStatus(): Array<ReturnType<DockerEventService['getStatus']>> {
return Array.from(this.services.values()).map(s => s.getStatus());
}
/** Returns the DockerEventService for a given local node, or undefined if not tracked. */
public getService(nodeId: number): DockerEventService | undefined {
return this.services.get(nodeId);
}
// ========================================================================
// Node lifecycle handlers
// ========================================================================
private async handleNodeAdded(nodeId: number): Promise<void> {
if (this.services.has(nodeId)) return;
const node = DatabaseService.getInstance().getNode(nodeId);
if (!node || node.type !== 'local') return;
await this.spawn(node);
}
private handleNodeRemoved(nodeId: number): void {
const service = this.services.get(nodeId);
if (!service) return;
service.shutdown();
this.services.delete(nodeId);
}
private async handleNodeUpdated(nodeId: number): Promise<void> {
const node = DatabaseService.getInstance().getNode(nodeId);
const existing = this.services.get(nodeId);
// Node became remote (or was deleted): tear down.
if (!node || node.type !== 'local') {
if (existing) {
existing.shutdown();
this.services.delete(nodeId);
}
return;
}
// Node is local: ensure a service exists (respawn if missing).
if (!existing) {
await this.spawn(node);
}
}
// ========================================================================
// Service spawning
// ========================================================================
private async spawn(node: Node): Promise<void> {
if (typeof node.id !== 'number') return;
if (this.services.has(node.id)) return;
const service = new DockerEventService(node.id, node.name);
this.services.set(node.id, service);
try {
await service.start();
} catch (err) {
if (isDebugEnabled()) {
console.log(`[DockerEvents:diag] failed to start service for node ${node.name}:`,
err instanceof Error ? err.message : err);
}
}
}
}