mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-07 17:34:23 +00:00
5bb4b01953
* feat(db): add auto_heal_policies and auto_heal_history schema and CRUD Adds two new SQLite tables (auto_heal_policies, auto_heal_history) to DatabaseService.initSchema() and exposes CRUD methods: getAutoHealPolicies, getAutoHealPolicy, addAutoHealPolicy, updateAutoHealPolicy, deleteAutoHealPolicy, recordAutoHealHistory, getAutoHealHistory, incrementConsecutiveFailures, resetConsecutiveFailures, setPolicyEnabled. Also adds AutoHealPolicy and AutoHealHistoryEntry TypeScript interfaces. * feat(events): track health-status duration and expose state accessors - Add healthStatus and unhealthySince fields to InternalContainerState - onHealthStatus now records unhealthySince timestamp on first transition to unhealthy, and clears it when the container recovers or restarts - onStart resets both fields so a restarted container begins from 'starting' - Add listContainerStates() and getContainerState() public accessors for use by the upcoming AutoHealService evaluator * fix(auto-heal): key allowlist in updateAutoHealPolicy, cascade delete, extract ContainerHealthSnapshot * feat: add AutoHealService evaluator singleton Polls every 30 s, matches containers to enabled policies via Compose labels, and restarts containers that have been unhealthy beyond the configured threshold. Enforces cooldown, per-hour rate cap, and recent-user-action suppression; auto-disables policies after repeated consecutive failures. Also adds DockerEventManager.getService() accessor required by the evaluator. * fix(auto-heal): prune stale restartTimestamps, guard undefined policy id - Prune restartTimestamps entries for containers no longer running after each container list fetch, preventing unbounded map growth from dead container IDs. - Guard against policies with undefined id at the start of the per-policy loop; warn and skip rather than proceed with a non-null assertion. - Extract handleAutoDisable private helper to bring executeHeal under 30 lines and isolate the auto-disable side-effect sequence. - Move ContainerInfo type to module scope. * feat: add auto-heal API routes and wire AutoHealService lifecycle Registers five REST endpoints under /api/auto-heal/policies (list, create, patch, delete, history) with requirePaid + requireAdmin guards and Zod validation. Wires AutoHealService.start()/stop() into the server startup and graceful-shutdown blocks alongside MonitorService. * test: add AutoHealService and DatabaseService auto-heal unit tests - 15 unit tests for AutoHealService.shouldHeal covering all decision branches (healthy state, duration threshold, user-action suppression, cooldown, rate limiting, and correct skipReason values) - 13 integration tests for DatabaseService auto-heal CRUD: policy round-trip, stack-name filter, partial update, cascade delete, history ordering/limit, consecutive failure counters, and setPolicyEnabled toggle * fix: log AutoHealService shutdown errors consistently * fix(api): requireAdmin-first guard order and try/catch on auto-heal routes * feat(ui): add StackAutoHealSheet component * feat(ui): add Auto-Heal context menu item to EditorLayout * fix(ui): StackAutoHealSheet label, token, a11y, and useEffect fixes - Rename 'All services in stack' to 'All services' in combobox options and placeholder - Replace text-green-600 with text-success design token in actionColorClass - Add htmlFor/id pairs to all four numeric form inputs for accessibility - Inline fetch logic into useEffect, removing stale closure risk and eslint-disable comment - Remove now-unused fetchPolicies and fetchServices standalone functions - Update 'Auto-disable after' label to 'Auto-disable after (failures)' for clarity - Add toast.error in policy fetch failure path; services fetch silently skips as before * docs: add auto-heal-policies feature documentation * test(e2e): add auto-heal policies CRUD spec * fix(docs): correct auto-heal-policies nav position in docs.json
137 lines
5.1 KiB
TypeScript
137 lines
5.1 KiB
TypeScript
import { DatabaseService, Node } from './DatabaseService';
|
|
import { NodeRegistry } from './NodeRegistry';
|
|
import { DockerEventService } from './DockerEventService';
|
|
import { isDebugEnabled } from '../utils/debug';
|
|
|
|
/**
|
|
* DockerEventManager
|
|
*
|
|
* Singleton coordinator that owns one DockerEventService per local node.
|
|
* Spawns services on boot for every existing local node, and reacts to
|
|
* NodeRegistry 'node-added' / 'node-removed' / 'node-updated' events to
|
|
* keep the service map in sync with the database.
|
|
*
|
|
* Remote nodes self-monitor on their own Sencho instance; this manager does
|
|
* not subscribe to remote Docker daemons.
|
|
*/
|
|
export class DockerEventManager {
|
|
private static instance: DockerEventManager;
|
|
private services: Map<number, DockerEventService> = new Map();
|
|
private started = false;
|
|
|
|
private readonly onNodeAdded = (id: number) => { void this.handleNodeAdded(id); };
|
|
private readonly onNodeRemoved = (id: number) => { this.handleNodeRemoved(id); };
|
|
private readonly onNodeUpdated = (id: number) => { void this.handleNodeUpdated(id); };
|
|
|
|
private constructor() { /* private: use getInstance */ }
|
|
|
|
public static getInstance(): DockerEventManager {
|
|
if (!DockerEventManager.instance) {
|
|
DockerEventManager.instance = new DockerEventManager();
|
|
}
|
|
return DockerEventManager.instance;
|
|
}
|
|
|
|
/** Boot: spawn a DockerEventService for every existing local node. */
|
|
public async start(): Promise<void> {
|
|
if (this.started) return;
|
|
this.started = true;
|
|
|
|
const registry = NodeRegistry.getInstance();
|
|
registry.on('node-added', this.onNodeAdded);
|
|
registry.on('node-removed', this.onNodeRemoved);
|
|
registry.on('node-updated', this.onNodeUpdated);
|
|
|
|
// Spawn in parallel so one slow node can't block boot for the others.
|
|
const nodes = DatabaseService.getInstance().getNodes()
|
|
.filter(n => n.type === 'local' && typeof n.id === 'number');
|
|
await Promise.all(nodes.map(n => this.spawn(n)));
|
|
|
|
console.log(`[DockerEvents] Started; watching ${this.services.size} local node(s) for container lifecycle events`);
|
|
}
|
|
|
|
/** Shutdown: stop every service and unsubscribe from registry events. */
|
|
public stop(): void {
|
|
if (!this.started) return;
|
|
this.started = false;
|
|
|
|
const registry = NodeRegistry.getInstance();
|
|
registry.off('node-added', this.onNodeAdded);
|
|
registry.off('node-removed', this.onNodeRemoved);
|
|
registry.off('node-updated', this.onNodeUpdated);
|
|
|
|
for (const service of this.services.values()) service.shutdown();
|
|
this.services.clear();
|
|
|
|
console.log('[DockerEvents] Stopped');
|
|
}
|
|
|
|
/** Aggregated status for diagnostics (e.g. /api/health). */
|
|
public getStatus(): Array<ReturnType<DockerEventService['getStatus']>> {
|
|
return Array.from(this.services.values()).map(s => s.getStatus());
|
|
}
|
|
|
|
/** Returns the DockerEventService for a given local node, or undefined if not tracked. */
|
|
public getService(nodeId: number): DockerEventService | undefined {
|
|
return this.services.get(nodeId);
|
|
}
|
|
|
|
// ========================================================================
|
|
// Node lifecycle handlers
|
|
// ========================================================================
|
|
|
|
private async handleNodeAdded(nodeId: number): Promise<void> {
|
|
if (this.services.has(nodeId)) return;
|
|
const node = DatabaseService.getInstance().getNode(nodeId);
|
|
if (!node || node.type !== 'local') return;
|
|
await this.spawn(node);
|
|
}
|
|
|
|
private handleNodeRemoved(nodeId: number): void {
|
|
const service = this.services.get(nodeId);
|
|
if (!service) return;
|
|
service.shutdown();
|
|
this.services.delete(nodeId);
|
|
}
|
|
|
|
private async handleNodeUpdated(nodeId: number): Promise<void> {
|
|
const node = DatabaseService.getInstance().getNode(nodeId);
|
|
const existing = this.services.get(nodeId);
|
|
|
|
// Node became remote (or was deleted): tear down.
|
|
if (!node || node.type !== 'local') {
|
|
if (existing) {
|
|
existing.shutdown();
|
|
this.services.delete(nodeId);
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Node is local: ensure a service exists (respawn if missing).
|
|
if (!existing) {
|
|
await this.spawn(node);
|
|
}
|
|
}
|
|
|
|
// ========================================================================
|
|
// Service spawning
|
|
// ========================================================================
|
|
|
|
private async spawn(node: Node): Promise<void> {
|
|
if (typeof node.id !== 'number') return;
|
|
if (this.services.has(node.id)) return;
|
|
|
|
const service = new DockerEventService(node.id, node.name);
|
|
this.services.set(node.id, service);
|
|
|
|
try {
|
|
await service.start();
|
|
} catch (err) {
|
|
if (isDebugEnabled()) {
|
|
console.log(`[DockerEvents:diag] failed to start service for node ${node.name}:`,
|
|
err instanceof Error ? err.message : err);
|
|
}
|
|
}
|
|
}
|
|
}
|