mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-27 18:57:09 +00:00
feat: auto-heal policies for unhealthy containers (#671)
* feat(db): add auto_heal_policies and auto_heal_history schema and CRUD Adds two new SQLite tables (auto_heal_policies, auto_heal_history) to DatabaseService.initSchema() and exposes CRUD methods: getAutoHealPolicies, getAutoHealPolicy, addAutoHealPolicy, updateAutoHealPolicy, deleteAutoHealPolicy, recordAutoHealHistory, getAutoHealHistory, incrementConsecutiveFailures, resetConsecutiveFailures, setPolicyEnabled. Also adds AutoHealPolicy and AutoHealHistoryEntry TypeScript interfaces. * feat(events): track health-status duration and expose state accessors - Add healthStatus and unhealthySince fields to InternalContainerState - onHealthStatus now records unhealthySince timestamp on first transition to unhealthy, and clears it when the container recovers or restarts - onStart resets both fields so a restarted container begins from 'starting' - Add listContainerStates() and getContainerState() public accessors for use by the upcoming AutoHealService evaluator * fix(auto-heal): key allowlist in updateAutoHealPolicy, cascade delete, extract ContainerHealthSnapshot * feat: add AutoHealService evaluator singleton Polls every 30 s, matches containers to enabled policies via Compose labels, and restarts containers that have been unhealthy beyond the configured threshold. Enforces cooldown, per-hour rate cap, and recent-user-action suppression; auto-disables policies after repeated consecutive failures. Also adds DockerEventManager.getService() accessor required by the evaluator. * fix(auto-heal): prune stale restartTimestamps, guard undefined policy id - Prune restartTimestamps entries for containers no longer running after each container list fetch, preventing unbounded map growth from dead container IDs. - Guard against policies with undefined id at the start of the per-policy loop; warn and skip rather than proceed with a non-null assertion. - Extract handleAutoDisable private helper to bring executeHeal under 30 lines and isolate the auto-disable side-effect sequence. - Move ContainerInfo type to module scope. * feat: add auto-heal API routes and wire AutoHealService lifecycle Registers five REST endpoints under /api/auto-heal/policies (list, create, patch, delete, history) with requirePaid + requireAdmin guards and Zod validation. Wires AutoHealService.start()/stop() into the server startup and graceful-shutdown blocks alongside MonitorService. * test: add AutoHealService and DatabaseService auto-heal unit tests - 15 unit tests for AutoHealService.shouldHeal covering all decision branches (healthy state, duration threshold, user-action suppression, cooldown, rate limiting, and correct skipReason values) - 13 integration tests for DatabaseService auto-heal CRUD: policy round-trip, stack-name filter, partial update, cascade delete, history ordering/limit, consecutive failure counters, and setPolicyEnabled toggle * fix: log AutoHealService shutdown errors consistently * fix(api): requireAdmin-first guard order and try/catch on auto-heal routes * feat(ui): add StackAutoHealSheet component * feat(ui): add Auto-Heal context menu item to EditorLayout * fix(ui): StackAutoHealSheet label, token, a11y, and useEffect fixes - Rename 'All services in stack' to 'All services' in combobox options and placeholder - Replace text-green-600 with text-success design token in actionColorClass - Add htmlFor/id pairs to all four numeric form inputs for accessibility - Inline fetch logic into useEffect, removing stale closure risk and eslint-disable comment - Remove now-unused fetchPolicies and fetchServices standalone functions - Update 'Auto-disable after' label to 'Auto-disable after (failures)' for clarity - Add toast.error in policy fetch failure path; services fetch silently skips as before * docs: add auto-heal-policies feature documentation * test(e2e): add auto-heal policies CRUD spec * fix(docs): correct auto-heal-policies nav position in docs.json
This commit is contained in:
Anso
@@ -28,6 +28,35 @@ export interface StackAlert {
|
||||
|
||||
export type NodeMode = 'proxy' | 'pilot_agent';
|
||||
|
||||
export interface AutoHealPolicy {
|
||||
id?: number;
|
||||
stack_name: string;
|
||||
service_name: string | null;
|
||||
unhealthy_duration_mins: number;
|
||||
cooldown_mins: number;
|
||||
max_restarts_per_hour: number;
|
||||
auto_disable_after_failures: number;
|
||||
enabled: number;
|
||||
consecutive_failures: number;
|
||||
last_fired_at: number;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
export interface AutoHealHistoryEntry {
|
||||
id?: number;
|
||||
policy_id: number;
|
||||
stack_name: string;
|
||||
service_name: string | null;
|
||||
container_name: string;
|
||||
container_id: string;
|
||||
action: 'restarted' | 'skipped_user_action' | 'skipped_cooldown' | 'skipped_rate_limit' | 'failed' | 'policy_auto_disabled';
|
||||
reason: string;
|
||||
success: number;
|
||||
error: string | null;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
export interface Node {
|
||||
id: number;
|
||||
name: string;
|
||||
@@ -807,6 +836,38 @@ export class DatabaseService {
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auto_heal_policies (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
stack_name TEXT NOT NULL,
|
||||
service_name TEXT,
|
||||
unhealthy_duration_mins INTEGER NOT NULL,
|
||||
cooldown_mins INTEGER NOT NULL DEFAULT 5,
|
||||
max_restarts_per_hour INTEGER NOT NULL DEFAULT 3,
|
||||
auto_disable_after_failures INTEGER NOT NULL DEFAULT 5,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
last_fired_at INTEGER NOT NULL DEFAULT 0,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS auto_heal_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
policy_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
service_name TEXT,
|
||||
container_name TEXT NOT NULL,
|
||||
container_id TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
success INTEGER NOT NULL,
|
||||
error TEXT,
|
||||
timestamp INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_auto_heal_history_policy_ts
|
||||
ON auto_heal_history(policy_id, timestamp DESC);
|
||||
`);
|
||||
|
||||
// Apply migrations safely (ignore if columns already exist)
|
||||
@@ -1205,6 +1266,94 @@ export class DatabaseService {
|
||||
stmt.run(timestamp, id);
|
||||
}
|
||||
|
||||
// --- Auto-Heal Policies ---
|
||||
|
||||
public getAutoHealPolicies(stackName?: string): AutoHealPolicy[] {
|
||||
if (stackName) {
|
||||
return this.db.prepare('SELECT * FROM auto_heal_policies WHERE stack_name = ?').all(stackName) as AutoHealPolicy[];
|
||||
}
|
||||
return this.db.prepare('SELECT * FROM auto_heal_policies').all() as AutoHealPolicy[];
|
||||
}
|
||||
|
||||
public getAutoHealPolicy(id: number): AutoHealPolicy | undefined {
|
||||
return this.db.prepare('SELECT * FROM auto_heal_policies WHERE id = ?').get(id) as AutoHealPolicy | undefined;
|
||||
}
|
||||
|
||||
public addAutoHealPolicy(policy: Omit<AutoHealPolicy, 'id'>): AutoHealPolicy {
|
||||
const stmt = this.db.prepare(
|
||||
'INSERT INTO auto_heal_policies (stack_name, service_name, unhealthy_duration_mins, cooldown_mins, max_restarts_per_hour, auto_disable_after_failures, enabled, consecutive_failures, last_fired_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
);
|
||||
const result = stmt.run(
|
||||
policy.stack_name,
|
||||
policy.service_name ?? null,
|
||||
policy.unhealthy_duration_mins,
|
||||
policy.cooldown_mins,
|
||||
policy.max_restarts_per_hour,
|
||||
policy.auto_disable_after_failures,
|
||||
policy.enabled,
|
||||
policy.consecutive_failures,
|
||||
policy.last_fired_at,
|
||||
policy.created_at,
|
||||
policy.updated_at
|
||||
);
|
||||
return this.db.prepare('SELECT * FROM auto_heal_policies WHERE id = ?').get(result.lastInsertRowid) as AutoHealPolicy;
|
||||
}
|
||||
|
||||
public updateAutoHealPolicy(id: number, patch: Partial<Omit<AutoHealPolicy, 'id' | 'stack_name' | 'created_at'>>): void {
|
||||
const ALLOWED_KEYS = new Set([
|
||||
'service_name', 'unhealthy_duration_mins', 'cooldown_mins',
|
||||
'max_restarts_per_hour', 'auto_disable_after_failures',
|
||||
'enabled', 'consecutive_failures', 'last_fired_at',
|
||||
]);
|
||||
const entries = Object.entries(patch).filter(([k, v]) => ALLOWED_KEYS.has(k) && v !== undefined);
|
||||
if (entries.length === 0) return;
|
||||
const fields = entries.map(([k]) => `${k} = ?`).join(', ');
|
||||
const values = entries.map(([, v]) => v);
|
||||
this.db.prepare(`UPDATE auto_heal_policies SET ${fields}, updated_at = ? WHERE id = ?`).run(...values, Date.now(), id);
|
||||
}
|
||||
|
||||
public deleteAutoHealPolicy(id: number): void {
|
||||
this.db.transaction(() => {
|
||||
this.db.prepare('DELETE FROM auto_heal_history WHERE policy_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM auto_heal_policies WHERE id = ?').run(id);
|
||||
})();
|
||||
}
|
||||
|
||||
public recordAutoHealHistory(entry: Omit<AutoHealHistoryEntry, 'id'>): void {
|
||||
this.db.prepare(
|
||||
'INSERT INTO auto_heal_history (policy_id, stack_name, service_name, container_name, container_id, action, reason, success, error, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
).run(
|
||||
entry.policy_id,
|
||||
entry.stack_name,
|
||||
entry.service_name ?? null,
|
||||
entry.container_name,
|
||||
entry.container_id,
|
||||
entry.action,
|
||||
entry.reason,
|
||||
entry.success,
|
||||
entry.error ?? null,
|
||||
entry.timestamp
|
||||
);
|
||||
}
|
||||
|
||||
public getAutoHealHistory(policyId: number, limit = 50): AutoHealHistoryEntry[] {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM auto_heal_history WHERE policy_id = ? ORDER BY timestamp DESC LIMIT ?'
|
||||
).all(policyId, limit) as AutoHealHistoryEntry[];
|
||||
}
|
||||
|
||||
public incrementConsecutiveFailures(policyId: number): void {
|
||||
this.db.prepare('UPDATE auto_heal_policies SET consecutive_failures = consecutive_failures + 1, updated_at = ? WHERE id = ?').run(Date.now(), policyId);
|
||||
}
|
||||
|
||||
public resetConsecutiveFailures(policyId: number): void {
|
||||
this.db.prepare('UPDATE auto_heal_policies SET consecutive_failures = 0, updated_at = ? WHERE id = ?').run(Date.now(), policyId);
|
||||
}
|
||||
|
||||
public setPolicyEnabled(policyId: number, enabled: boolean): void {
|
||||
this.db.prepare('UPDATE auto_heal_policies SET enabled = ?, updated_at = ? WHERE id = ?').run(enabled ? 1 : 0, Date.now(), policyId);
|
||||
}
|
||||
|
||||
// --- Notification History ---
|
||||
|
||||
public getNotificationHistory(limit = 50): NotificationHistory[] {
|
||||
|
||||
Reference in New Issue
Block a user