diff --git a/backend/src/__tests__/AutoHealService.test.ts b/backend/src/__tests__/AutoHealService.test.ts new file mode 100644 index 00000000..23a3ee90 --- /dev/null +++ b/backend/src/__tests__/AutoHealService.test.ts @@ -0,0 +1,168 @@ +/** + * Unit tests for AutoHealService.shouldHeal decision logic. + * + * shouldHeal is private; accessed via type cast (service as any) to avoid + * exposing it in production API surface. All tests are pure (no I/O, no + * timers) - they exercise the decision function directly. + */ +import { describe, it, expect, beforeEach } from 'vitest'; +import { INTENTIONAL_KILL_WINDOW_MS } from '../services/ContainerLifecycleClassifier'; +import { AutoHealService } from '../services/AutoHealService'; + +describe('AutoHealService.shouldHeal', () => { + let service: any; + + beforeEach(() => { + // Reset singleton so each test starts with a clean restartTimestamps map + (AutoHealService as any).instance = undefined; + service = AutoHealService.getInstance(); + }); + + const basePolicy = { + id: 1, + stack_name: 'mystack', + service_name: null, + unhealthy_duration_mins: 5, + cooldown_mins: 10, + max_restarts_per_hour: 3, + auto_disable_after_failures: 5, + enabled: 1, + consecutive_failures: 0, + last_fired_at: 0, + created_at: Date.now(), + updated_at: Date.now(), + }; + + const baseState = { + id: 'container123', + name: 'mystack-web-1', + stackName: 'mystack', + healthStatus: 'unhealthy' as const, + unhealthySince: Date.now() - 6 * 60_000, // 6 minutes ago (past 5 min threshold) + lastKillAt: undefined, + }; + + it('returns heal:true when all conditions are met', () => { + const result = service.shouldHeal(baseState, basePolicy, 'container123', Date.now()); + expect(result.heal).toBe(true); + }); + + it('returns heal:false when healthStatus is not unhealthy', () => { + const result = service.shouldHeal( + { ...baseState, healthStatus: 'healthy' }, + basePolicy, + 'container123', + Date.now(), + ); + expect(result.heal).toBe(false); + }); + + it('returns heal:false when healthStatus is undefined', () => { + const result = service.shouldHeal( + { ...baseState, healthStatus: undefined }, + basePolicy, + 'container123', + Date.now(), + ); + expect(result.heal).toBe(false); + }); + + it('returns heal:false when state is undefined', () => { + const result = service.shouldHeal(undefined, basePolicy, 'container123', Date.now()); + expect(result.heal).toBe(false); + }); + + it('returns heal:false when unhealthySince is undefined', () => { + const result = service.shouldHeal( + { ...baseState, unhealthySince: undefined }, + basePolicy, + 'container123', + Date.now(), + ); + expect(result.heal).toBe(false); + }); + + it('returns heal:false when duration threshold is not yet met', () => { + // Only 2 minutes, threshold is 5 + const state = { ...baseState, unhealthySince: Date.now() - 2 * 60_000 }; + const result = service.shouldHeal(state, basePolicy, 'container123', Date.now()); + expect(result.heal).toBe(false); + expect(result.skipReason).toBe('duration_not_met'); + }); + + it('returns skipped_user_action when lastKillAt is within the window', () => { + // 30s ago, well within the 60s INTENTIONAL_KILL_WINDOW_MS + const state = { ...baseState, lastKillAt: Date.now() - 30_000 }; + const result = service.shouldHeal(state, basePolicy, 'container123', Date.now()); + expect(result.heal).toBe(false); + expect(result.skipReason).toBe('skipped_user_action'); + }); + + it('does not suppress when lastKillAt is outside the intentional kill window', () => { + const state = { + ...baseState, + lastKillAt: Date.now() - (INTENTIONAL_KILL_WINDOW_MS + 5_000), + }; + const result = service.shouldHeal(state, basePolicy, 'container123', Date.now()); + expect(result.heal).toBe(true); + }); + + it('returns skipped_cooldown when last_fired_at is within cooldown period', () => { + // Fired 5 min ago, cooldown is 10 min + const policy = { ...basePolicy, last_fired_at: Date.now() - 5 * 60_000, cooldown_mins: 10 }; + const result = service.shouldHeal(baseState, policy, 'container123', Date.now()); + expect(result.heal).toBe(false); + expect(result.skipReason).toBe('skipped_cooldown'); + }); + + it('does not apply cooldown when last_fired_at is 0', () => { + const policy = { ...basePolicy, last_fired_at: 0 }; + const result = service.shouldHeal(baseState, policy, 'container123', Date.now()); + expect(result.heal).toBe(true); + }); + + it('does not apply cooldown when last_fired_at exceeds the cooldown window', () => { + // Fired 15 min ago, cooldown is 10 min + const policy = { ...basePolicy, last_fired_at: Date.now() - 15 * 60_000, cooldown_mins: 10 }; + const result = service.shouldHeal(baseState, policy, 'container123', Date.now()); + expect(result.heal).toBe(true); + }); + + it('returns skipped_rate_limit when hourly restart count is at the configured max', () => { + const now = Date.now(); + // Pre-populate with 3 entries within the last hour (policy max is 3) + const map = (service as any).restartTimestamps as Map; + map.set('container123', [now - 10_000, now - 20_000, now - 30_000]); + const result = service.shouldHeal(baseState, basePolicy, 'container123', now); + expect(result.heal).toBe(false); + expect(result.skipReason).toBe('skipped_rate_limit'); + }); + + it('does not rate-limit when all timestamps are older than one hour', () => { + const now = Date.now(); + const map = (service as any).restartTimestamps as Map; + // All entries are >1 hour old, so they fall outside the rate-limit window + map.set('container123', [now - 70 * 60_000, now - 80 * 60_000, now - 90 * 60_000]); + const result = service.shouldHeal(baseState, basePolicy, 'container123', now); + expect(result.heal).toBe(true); + }); + + it('counts only recent timestamps toward the rate limit', () => { + const now = Date.now(); + const map = (service as any).restartTimestamps as Map; + // 2 old (outside window) + 1 recent = 1 active restart; max is 3, so still allowed + map.set('container123', [now - 70 * 60_000, now - 80 * 60_000, now - 5_000]); + const result = service.shouldHeal(baseState, basePolicy, 'container123', now); + expect(result.heal).toBe(true); + }); + + it('returns not_unhealthy as skipReason when container is healthy', () => { + const result = service.shouldHeal( + { ...baseState, healthStatus: 'healthy' }, + basePolicy, + 'container123', + Date.now(), + ); + expect(result.skipReason).toBe('not_unhealthy'); + }); +}); diff --git a/backend/src/__tests__/DatabaseService.autoHeal.test.ts b/backend/src/__tests__/DatabaseService.autoHeal.test.ts new file mode 100644 index 00000000..eb2db88d --- /dev/null +++ b/backend/src/__tests__/DatabaseService.autoHeal.test.ts @@ -0,0 +1,271 @@ +/** + * Integration tests for DatabaseService auto-heal CRUD. + * + * Uses a real temp SQLite database (same pattern as database-metrics.test.ts). + * All operations are tested against live SQL; no mocking. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let db: any; + +const makePolicy = (overrides: Record = {}) => { + const now = Date.now(); + return { + stack_name: 'teststack', + service_name: null, + unhealthy_duration_mins: 5, + cooldown_mins: 10, + max_restarts_per_hour: 3, + auto_disable_after_failures: 5, + enabled: 1, + consecutive_failures: 0, + last_fired_at: 0, + created_at: now, + updated_at: now, + ...overrides, + }; +}; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + const { DatabaseService } = await import('../services/DatabaseService'); + db = DatabaseService.getInstance(); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +describe('DatabaseService - auto-heal policy CRUD', () => { + it('addAutoHealPolicy + getAutoHealPolicy round-trip preserves all fields', () => { + const input = makePolicy({ + stack_name: 'roundtrip-stack', + service_name: 'web', + unhealthy_duration_mins: 3, + cooldown_mins: 15, + max_restarts_per_hour: 5, + auto_disable_after_failures: 10, + enabled: 1, + consecutive_failures: 0, + last_fired_at: 0, + }); + + const created = db.addAutoHealPolicy(input); + expect(created.id).toBeDefined(); + expect(typeof created.id).toBe('number'); + + const fetched = db.getAutoHealPolicy(created.id); + expect(fetched).toBeDefined(); + expect(fetched.stack_name).toBe('roundtrip-stack'); + expect(fetched.service_name).toBe('web'); + expect(fetched.unhealthy_duration_mins).toBe(3); + expect(fetched.cooldown_mins).toBe(15); + expect(fetched.max_restarts_per_hour).toBe(5); + expect(fetched.auto_disable_after_failures).toBe(10); + expect(fetched.enabled).toBe(1); + expect(fetched.consecutive_failures).toBe(0); + expect(fetched.last_fired_at).toBe(0); + expect(fetched.created_at).toBe(input.created_at); + }); + + it('getAutoHealPolicies without filter returns all policies', () => { + const before = db.getAutoHealPolicies().length; + + db.addAutoHealPolicy(makePolicy({ stack_name: 'all-stack-a' })); + db.addAutoHealPolicy(makePolicy({ stack_name: 'all-stack-b' })); + + const after = db.getAutoHealPolicies(); + expect(after.length).toBe(before + 2); + }); + + it('getAutoHealPolicies with stackName filter returns only matching rows', () => { + db.addAutoHealPolicy(makePolicy({ stack_name: 'filter-stack-x' })); + db.addAutoHealPolicy(makePolicy({ stack_name: 'filter-stack-y' })); + + const xPolicies = db.getAutoHealPolicies('filter-stack-x'); + const yPolicies = db.getAutoHealPolicies('filter-stack-y'); + + expect(xPolicies.length).toBeGreaterThanOrEqual(1); + expect(xPolicies.every((p: any) => p.stack_name === 'filter-stack-x')).toBe(true); + + expect(yPolicies.length).toBeGreaterThanOrEqual(1); + expect(yPolicies.every((p: any) => p.stack_name === 'filter-stack-y')).toBe(true); + }); + + it('updateAutoHealPolicy partial update changes only specified fields', () => { + const created = db.addAutoHealPolicy(makePolicy({ stack_name: 'update-stack', cooldown_mins: 10 })); + + db.updateAutoHealPolicy(created.id, { cooldown_mins: 30, max_restarts_per_hour: 7 }); + + const updated = db.getAutoHealPolicy(created.id); + // Changed fields + expect(updated.cooldown_mins).toBe(30); + expect(updated.max_restarts_per_hour).toBe(7); + // Unchanged fields must be preserved + expect(updated.stack_name).toBe('update-stack'); + expect(updated.unhealthy_duration_mins).toBe(5); + // updated_at should be refreshed + expect(updated.updated_at).toBeGreaterThanOrEqual(created.updated_at); + }); + + it('deleteAutoHealPolicy removes the policy row', () => { + const created = db.addAutoHealPolicy(makePolicy({ stack_name: 'delete-stack' })); + expect(db.getAutoHealPolicy(created.id)).toBeDefined(); + + db.deleteAutoHealPolicy(created.id); + + expect(db.getAutoHealPolicy(created.id)).toBeUndefined(); + }); + + it('deleteAutoHealPolicy cascades and removes history rows', () => { + const created = db.addAutoHealPolicy(makePolicy({ stack_name: 'cascade-stack' })); + const policyId: number = created.id; + + // Write a history entry tied to this policy + db.recordAutoHealHistory({ + policy_id: policyId, + stack_name: 'cascade-stack', + service_name: null, + container_name: 'cascade-stack-web-1', + container_id: 'cascade-abc', + action: 'restarted', + reason: 'test cascade', + success: 1, + error: null, + timestamp: Date.now(), + }); + + const beforeDelete = db.getAutoHealHistory(policyId); + expect(beforeDelete.length).toBe(1); + + db.deleteAutoHealPolicy(policyId); + + const afterDelete = db.getAutoHealHistory(policyId); + expect(afterDelete.length).toBe(0); + }); +}); + +describe('DatabaseService - auto-heal history', () => { + it('recordAutoHealHistory + getAutoHealHistory orders results by timestamp DESC', () => { + const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'history-order-stack' })); + const policyId: number = policy.id; + const baseTs = Date.now(); + + db.recordAutoHealHistory({ + policy_id: policyId, + stack_name: 'history-order-stack', + service_name: null, + container_name: 'history-order-stack-web-1', + container_id: 'hist-001', + action: 'restarted', + reason: 'first', + success: 1, + error: null, + timestamp: baseTs, + }); + db.recordAutoHealHistory({ + policy_id: policyId, + stack_name: 'history-order-stack', + service_name: null, + container_name: 'history-order-stack-web-1', + container_id: 'hist-001', + action: 'skipped_cooldown', + reason: 'second', + success: 0, + error: null, + timestamp: baseTs + 1_000, + }); + + const history = db.getAutoHealHistory(policyId); + expect(history.length).toBe(2); + // Most recent first + expect(history[0].reason).toBe('second'); + expect(history[1].reason).toBe('first'); + }); + + it('getAutoHealHistory respects the limit parameter', () => { + const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'history-limit-stack' })); + const policyId: number = policy.id; + const baseTs = Date.now(); + + for (let i = 0; i < 10; i++) { + db.recordAutoHealHistory({ + policy_id: policyId, + stack_name: 'history-limit-stack', + service_name: null, + container_name: 'history-limit-stack-web-1', + container_id: 'hist-limit', + action: 'restarted', + reason: `entry-${i}`, + success: 1, + error: null, + timestamp: baseTs + i, + }); + } + + const limited = db.getAutoHealHistory(policyId, 3); + expect(limited.length).toBe(3); + // Should contain the 3 most recent entries (highest timestamps) + expect(limited[0].reason).toBe('entry-9'); + expect(limited[1].reason).toBe('entry-8'); + expect(limited[2].reason).toBe('entry-7'); + }); +}); + +describe('DatabaseService - consecutive failure counters', () => { + it('incrementConsecutiveFailures increments the counter each call', () => { + const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'fail-inc-stack', consecutive_failures: 0 })); + const id: number = policy.id; + + db.incrementConsecutiveFailures(id); + expect(db.getAutoHealPolicy(id).consecutive_failures).toBe(1); + + db.incrementConsecutiveFailures(id); + expect(db.getAutoHealPolicy(id).consecutive_failures).toBe(2); + }); + + it('resetConsecutiveFailures sets the counter back to 0', () => { + const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'fail-reset-stack', consecutive_failures: 0 })); + const id: number = policy.id; + + db.incrementConsecutiveFailures(id); + db.incrementConsecutiveFailures(id); + expect(db.getAutoHealPolicy(id).consecutive_failures).toBe(2); + + db.resetConsecutiveFailures(id); + expect(db.getAutoHealPolicy(id).consecutive_failures).toBe(0); + }); +}); + +describe('DatabaseService - setPolicyEnabled', () => { + it('setPolicyEnabled(id, false) sets enabled to 0', () => { + const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'enable-toggle-stack', enabled: 1 })); + const id: number = policy.id; + + db.setPolicyEnabled(id, false); + expect(db.getAutoHealPolicy(id).enabled).toBe(0); + }); + + it('setPolicyEnabled(id, true) sets enabled to 1', () => { + const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'enable-on-stack', enabled: 0 })); + const id: number = policy.id; + + db.setPolicyEnabled(id, true); + expect(db.getAutoHealPolicy(id).enabled).toBe(1); + }); + + it('getAutoHealPolicies returns only enabled policies after filtering', () => { + const enabledPolicy = db.addAutoHealPolicy(makePolicy({ stack_name: 'enabled-filter-stack', enabled: 1 })); + const disabledPolicy = db.addAutoHealPolicy(makePolicy({ stack_name: 'disabled-filter-stack', enabled: 0 })); + + const allPolicies = db.getAutoHealPolicies(); + const enabledIds = allPolicies.filter((p: any) => p.enabled === 1).map((p: any) => p.id); + const disabledIds = allPolicies.filter((p: any) => p.enabled === 0).map((p: any) => p.id); + + expect(enabledIds).toContain(enabledPolicy.id); + expect(disabledIds).toContain(disabledPolicy.id); + expect(enabledIds).not.toContain(disabledPolicy.id); + }); +}); diff --git a/backend/src/index.ts b/backend/src/index.ts index 55fae5be..afc61b3b 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -23,6 +23,7 @@ import { HostTerminalService } from './services/HostTerminalService'; import { DatabaseService, Node, AuthProvider, ScheduledTask, UserRole, ResourceType } from './services/DatabaseService'; import { NotificationService } from './services/NotificationService'; import { MonitorService } from './services/MonitorService'; +import { AutoHealService } from './services/AutoHealService'; import { DockerEventManager } from './services/DockerEventManager'; import { ImageUpdateService } from './services/ImageUpdateService'; import { templateService } from './services/TemplateService'; @@ -5697,6 +5698,16 @@ const AlertCreateSchema = z.object({ cooldown_mins: z.coerce.number().int().min(0).max(10080), }); +const AutoHealPolicyCreateSchema = z.object({ + stack_name: z.string().min(1).max(255), + service_name: z.string().min(1).max(255).nullable().optional(), + unhealthy_duration_mins: z.coerce.number().int().min(1).max(1440), + cooldown_mins: z.coerce.number().int().min(1).max(1440).default(5), + max_restarts_per_hour: z.coerce.number().int().min(1).max(60).default(3), + auto_disable_after_failures: z.coerce.number().int().min(1).max(100).default(5), +}); +const AutoHealPolicyUpdateSchema = AutoHealPolicyCreateSchema.partial().omit({ stack_name: true }); + app.post('/api/alerts', authMiddleware, async (req: Request, res: Response) => { if (!requireAdmin(req, res)) return; const parsed = AlertCreateSchema.safeParse(req.body); @@ -5724,6 +5735,100 @@ app.delete('/api/alerts/:id', authMiddleware, async (req: Request, res: Response } }); +// ─── Auto-Heal Policies ─────────────────────────────────────────────────────── + +app.get('/api/auto-heal/policies', authMiddleware, (req: Request, res: Response): void => { + if (!requirePaid(req, res)) return; + const stackName = typeof req.query.stackName === 'string' ? req.query.stackName : undefined; + try { + res.json(DatabaseService.getInstance().getAutoHealPolicies(stackName)); + } catch (err) { + console.error('[AutoHeal] Failed to list policies:', err instanceof Error ? err.message : err); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +app.post('/api/auto-heal/policies', authMiddleware, (req: Request, res: Response): void => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + const parsed = AutoHealPolicyCreateSchema.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: parsed.error.issues[0]?.message ?? 'Invalid input' }); + return; + } + const { stack_name, service_name, unhealthy_duration_mins, cooldown_mins, max_restarts_per_hour, auto_disable_after_failures } = parsed.data; + const now = Date.now(); + try { + const policy = DatabaseService.getInstance().addAutoHealPolicy({ + stack_name, + service_name: service_name ?? null, + unhealthy_duration_mins, + cooldown_mins, + max_restarts_per_hour, + auto_disable_after_failures, + enabled: 1, + consecutive_failures: 0, + last_fired_at: 0, + created_at: now, + updated_at: now, + }); + res.status(201).json(policy); + } catch (err) { + console.error('[AutoHeal] Failed to create policy:', err instanceof Error ? err.message : err); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +app.patch('/api/auto-heal/policies/:id', authMiddleware, (req: Request, res: Response): void => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + const id = parseInt(req.params.id as string, 10); + if (isNaN(id)) { res.status(400).json({ error: 'Invalid id' }); return; } + const parsed = AutoHealPolicyUpdateSchema.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: parsed.error.issues[0]?.message ?? 'Invalid input' }); + return; + } + try { + const db = DatabaseService.getInstance(); + if (!db.getAutoHealPolicy(id)) { res.status(404).json({ error: 'Policy not found' }); return; } + db.updateAutoHealPolicy(id, parsed.data); + res.json(db.getAutoHealPolicy(id)); + } catch (err) { + console.error('[AutoHeal] Failed to update policy:', err instanceof Error ? err.message : err); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +app.delete('/api/auto-heal/policies/:id', authMiddleware, (req: Request, res: Response): void => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + const id = parseInt(req.params.id as string, 10); + if (isNaN(id)) { res.status(400).json({ error: 'Invalid id' }); return; } + try { + const db = DatabaseService.getInstance(); + if (!db.getAutoHealPolicy(id)) { res.status(404).json({ error: 'Policy not found' }); return; } + db.deleteAutoHealPolicy(id); + res.json({ success: true }); + } catch (err) { + console.error('[AutoHeal] Failed to delete policy:', err instanceof Error ? err.message : err); + res.status(500).json({ error: 'Internal server error' }); + } +}); + +app.get('/api/auto-heal/policies/:id/history', authMiddleware, (req: Request, res: Response): void => { + if (!requirePaid(req, res)) return; + const id = parseInt(req.params.id as string, 10); + if (isNaN(id)) { res.status(400).json({ error: 'Invalid id' }); return; } + const limit = Math.min(parseInt(String(req.query.limit ?? '50'), 10) || 50, 100); + try { + res.json(DatabaseService.getInstance().getAutoHealHistory(id, limit)); + } catch (err) { + console.error('[AutoHeal] Failed to fetch history:', err instanceof Error ? err.message : err); + res.status(500).json({ error: 'Internal server error' }); + } +}); + app.get('/api/notifications', authMiddleware, async (req: Request, res: Response) => { try { const history = DatabaseService.getInstance().getNotificationHistory(); @@ -8535,6 +8640,7 @@ async function startServer() { // Start Background Watchdog MonitorService.getInstance().start(); + AutoHealService.getInstance().start(); // Start Docker Event Stream (causal crash/OOM/health detection per local node) await DockerEventManager.getInstance().start(); @@ -8609,6 +8715,7 @@ const gracefulShutdown = (signal: string) => { try { MonitorService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] MonitorService cleanup failed:', (e as Error).message); } + try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); } try { DockerEventManager.getInstance().stop(); } catch (e) { console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message); } diff --git a/backend/src/services/AutoHealService.ts b/backend/src/services/AutoHealService.ts new file mode 100644 index 00000000..0e8e78a2 --- /dev/null +++ b/backend/src/services/AutoHealService.ts @@ -0,0 +1,318 @@ +import { INTENTIONAL_KILL_WINDOW_MS } from './ContainerLifecycleClassifier'; +import { DatabaseService, AutoHealPolicy, AutoHealHistoryEntry } from './DatabaseService'; +import DockerController from './DockerController'; +import { DockerEventManager } from './DockerEventManager'; +import { ContainerHealthSnapshot } from './DockerEventService'; +import { NotificationService } from './NotificationService'; + +// Dockerode listContainers shape (subset used here) +type ContainerInfo = { + Id: string; + Names?: string[]; + Labels?: Record; +}; + +const EVAL_INTERVAL_MS = 30_000; +const INITIAL_DELAY_MS = 10_000; +const RATE_LIMIT_WINDOW_MS = 60 * 60_000; // 1 hour + +export class AutoHealService { + private static instance: AutoHealService; + private intervalId: NodeJS.Timeout | null = null; + private initialTimer: NodeJS.Timeout | null = null; + private isProcessing = false; + private restartTimestamps = new Map(); + + private constructor() {} + + static getInstance(): AutoHealService { + if (!AutoHealService.instance) { + AutoHealService.instance = new AutoHealService(); + } + return AutoHealService.instance; + } + + start(): void { + this.initialTimer = setTimeout(() => { + void this.evaluate(); + this.intervalId = setInterval(() => void this.evaluate(), EVAL_INTERVAL_MS); + }, INITIAL_DELAY_MS); + } + + stop(): void { + if (this.initialTimer) { + clearTimeout(this.initialTimer); + this.initialTimer = null; + } + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + } + + async evaluate(): Promise { + if (this.isProcessing) return; + this.isProcessing = true; + try { + const db = DatabaseService.getInstance(); + const policies = db.getAutoHealPolicies().filter(p => p.enabled === 1); + if (policies.length === 0) return; + + // Evaluate only on local nodes (remote nodes self-monitor via their own instance) + const nodes = db.getNodes().filter(n => n.type === 'local'); + for (const node of nodes) { + await this.evaluateForNode(node.id, policies); + } + } catch (err) { + console.error('[AutoHeal] evaluate error:', err instanceof Error ? err.message : err); + } finally { + this.isProcessing = false; + } + } + + private async evaluateForNode(nodeId: number, policies: AutoHealPolicy[]): Promise { + let containers: ContainerInfo[]; + try { + containers = await DockerController.getInstance(nodeId).getRunningContainers(); + } catch (err) { + console.error( + `[AutoHeal] failed to list containers on node ${nodeId}:`, + err instanceof Error ? err.message : err, + ); + return; + } + + const db = DatabaseService.getInstance(); + const eventSvc = DockerEventManager.getInstance().getService(nodeId); + const now = Date.now(); + + // Prune stale entries for containers no longer running on this node + const liveIds = new Set(containers.map(c => c.Id)); + for (const [cid, timestamps] of this.restartTimestamps.entries()) { + const recent = timestamps.filter(t => now - t < RATE_LIMIT_WINDOW_MS); + if (recent.length === 0 || !liveIds.has(cid)) { + this.restartTimestamps.delete(cid); + } else { + this.restartTimestamps.set(cid, recent); + } + } + + for (const policy of policies) { + if (policy.id === undefined) { + console.warn('[AutoHeal] skipping policy without id:', policy.stack_name); + continue; + } + const candidates = containers.filter(c => { + const labels = c.Labels ?? {}; + if (labels['com.docker.compose.project'] !== policy.stack_name) return false; + if (policy.service_name) { + return labels['com.docker.compose.service'] === policy.service_name; + } + return true; + }); + + for (const container of candidates) { + const containerName = + container.Names?.[0]?.replace(/^\//, '') ?? container.Id.slice(0, 12); + const serviceOverride = container.Labels?.['com.docker.compose.service'] ?? null; + const state = eventSvc?.getContainerState(container.Id); + const decision = this.shouldHeal(state, policy, container.Id, now); + + if (!decision.heal) { + if ( + decision.skipReason && + decision.skipReason !== 'not_unhealthy' && + decision.skipReason !== 'duration_not_met' + ) { + db.recordAutoHealHistory({ + policy_id: policy.id!, + stack_name: policy.stack_name, + service_name: policy.service_name ?? serviceOverride, + container_name: containerName, + container_id: container.Id, + action: decision.skipReason as AutoHealHistoryEntry['action'], + reason: this.skipReasonText(decision.skipReason), + success: 0, + error: null, + timestamp: now, + }); + } + continue; + } + + await this.executeHeal( + policy, + nodeId, + container.Id, + containerName, + policy.service_name ?? serviceOverride, + ); + } + } + } + + private shouldHeal( + state: ContainerHealthSnapshot | undefined, + policy: AutoHealPolicy, + containerId: string, + now: number, + ): { heal: boolean; skipReason?: string } { + // No state tracked yet, or container is not unhealthy + if (!state || state.healthStatus !== 'unhealthy' || !state.unhealthySince) { + return { heal: false, skipReason: 'not_unhealthy' }; + } + + // Duration threshold not yet met + const unhealthyMs = now - state.unhealthySince; + if (unhealthyMs < policy.unhealthy_duration_mins * 60_000) { + return { heal: false, skipReason: 'duration_not_met' }; + } + + // Suppress if user recently killed the container + if (state.lastKillAt !== undefined && now - state.lastKillAt < INTENTIONAL_KILL_WINDOW_MS) { + return { heal: false, skipReason: 'skipped_user_action' }; + } + + // Cooldown: respect last_fired_at + if (policy.last_fired_at > 0 && now - policy.last_fired_at < policy.cooldown_mins * 60_000) { + return { heal: false, skipReason: 'skipped_cooldown' }; + } + + // Rate limit: max restarts per hour + const recentRestarts = (this.restartTimestamps.get(containerId) ?? []).filter( + t => now - t < RATE_LIMIT_WINDOW_MS, + ); + if (recentRestarts.length >= policy.max_restarts_per_hour) { + return { heal: false, skipReason: 'skipped_rate_limit' }; + } + + return { heal: true }; + } + + private async executeHeal( + policy: AutoHealPolicy, + nodeId: number, + containerId: string, + containerName: string, + serviceName: string | null, + ): Promise { + const db = DatabaseService.getInstance(); + const now = Date.now(); + const baseEntry = { + policy_id: policy.id!, + stack_name: policy.stack_name, + service_name: serviceName, + container_name: containerName, + container_id: containerId, + timestamp: now, + }; + + try { + await DockerController.getInstance(nodeId).restartContainer(containerId); + + db.resetConsecutiveFailures(policy.id!); + db.updateAutoHealPolicy(policy.id!, { last_fired_at: now }); + + const timestamps = this.restartTimestamps.get(containerId) ?? []; + timestamps.push(now); + this.restartTimestamps.set( + containerId, + timestamps.filter(t => now - t < RATE_LIMIT_WINDOW_MS), + ); + + db.recordAutoHealHistory({ + ...baseEntry, + action: 'restarted', + reason: `Container unhealthy for ${policy.unhealthy_duration_mins} minute(s); auto-restarted.`, + success: 1, + error: null, + }); + + db.insertAuditLog({ + timestamp: now, + username: 'system', + method: 'POST', + path: '/api/auto-heal/execute', + status_code: 200, + node_id: nodeId, + ip_address: '127.0.0.1', + summary: `Auto-healed container ${containerName} on stack ${policy.stack_name}`, + }); + + NotificationService.getInstance() + .dispatchAlert( + 'info', + `Auto-Heal: Restarted ${containerName} on stack ${policy.stack_name} after being unhealthy for ${policy.unhealthy_duration_mins} minute(s).`, + policy.stack_name, + ) + .catch(err => console.error('[AutoHeal] notification dispatch failed:', err)); + } catch (err) { + const errorMsg = err instanceof Error ? err.message : String(err); + + db.incrementConsecutiveFailures(policy.id!); + // Re-read to get updated consecutive_failures count + const updated = db.getAutoHealPolicy(policy.id!); + const failures = updated?.consecutive_failures ?? policy.consecutive_failures + 1; + + db.recordAutoHealHistory({ + ...baseEntry, + action: 'failed', + reason: `Restart failed: ${errorMsg}`, + success: 0, + error: errorMsg, + }); + + NotificationService.getInstance() + .dispatchAlert( + 'warning', + `Auto-Heal: Failed to restart ${containerName} on stack ${policy.stack_name}. Error: ${errorMsg}`, + policy.stack_name, + ) + .catch(e => console.error('[AutoHeal] notification dispatch failed:', e)); + + // Auto-disable if failure threshold reached + if (failures >= policy.auto_disable_after_failures) { + this.handleAutoDisable(policy.id!, policy, baseEntry, failures); + } + + console.error(`[AutoHeal] restart failed for ${containerName} (${containerId}):`, errorMsg); + } + } + + private handleAutoDisable( + policyId: number, + policy: AutoHealPolicy, + baseEntry: Omit[0], 'action' | 'reason' | 'success' | 'error'>, + failures: number, + ): void { + const db = DatabaseService.getInstance(); + db.setPolicyEnabled(policyId, false); + db.recordAutoHealHistory({ + ...baseEntry, + action: 'policy_auto_disabled', + reason: `Policy disabled after ${failures} consecutive restart failures. Check container logs and re-enable when resolved.`, + success: 0, + error: null, + }); + NotificationService.getInstance() + .dispatchAlert( + 'warning', + `Auto-Heal: Policy for ${policy.stack_name}${policy.service_name ? '/' + policy.service_name : ''} has been auto-disabled after ${failures} consecutive failures.`, + policy.stack_name, + ) + .catch(e => console.error('[AutoHeal] notification dispatch failed:', e)); + } + + private skipReasonText(reason: string): string { + switch (reason) { + case 'skipped_user_action': + return 'Skipped: recent user action detected on this container.'; + case 'skipped_cooldown': + return 'Skipped: cooldown period has not elapsed since last restart.'; + case 'skipped_rate_limit': + return 'Skipped: hourly restart limit reached for this container.'; + default: + return reason; + } + } +} diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index c5b9b921..d1e3fe1b 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -28,6 +28,35 @@ export interface StackAlert { export type NodeMode = 'proxy' | 'pilot_agent'; +export interface AutoHealPolicy { + id?: number; + stack_name: string; + service_name: string | null; + unhealthy_duration_mins: number; + cooldown_mins: number; + max_restarts_per_hour: number; + auto_disable_after_failures: number; + enabled: number; + consecutive_failures: number; + last_fired_at: number; + created_at: number; + updated_at: number; +} + +export interface AutoHealHistoryEntry { + id?: number; + policy_id: number; + stack_name: string; + service_name: string | null; + container_name: string; + container_id: string; + action: 'restarted' | 'skipped_user_action' | 'skipped_cooldown' | 'skipped_rate_limit' | 'failed' | 'policy_auto_disabled'; + reason: string; + success: number; + error: string | null; + timestamp: number; +} + export interface Node { id: number; name: string; @@ -807,6 +836,38 @@ export class DatabaseService { created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); + + CREATE TABLE IF NOT EXISTS auto_heal_policies ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + stack_name TEXT NOT NULL, + service_name TEXT, + unhealthy_duration_mins INTEGER NOT NULL, + cooldown_mins INTEGER NOT NULL DEFAULT 5, + max_restarts_per_hour INTEGER NOT NULL DEFAULT 3, + auto_disable_after_failures INTEGER NOT NULL DEFAULT 5, + enabled INTEGER NOT NULL DEFAULT 1, + consecutive_failures INTEGER NOT NULL DEFAULT 0, + last_fired_at INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS auto_heal_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + policy_id INTEGER NOT NULL, + stack_name TEXT NOT NULL, + service_name TEXT, + container_name TEXT NOT NULL, + container_id TEXT NOT NULL, + action TEXT NOT NULL, + reason TEXT NOT NULL, + success INTEGER NOT NULL, + error TEXT, + timestamp INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_auto_heal_history_policy_ts + ON auto_heal_history(policy_id, timestamp DESC); `); // Apply migrations safely (ignore if columns already exist) @@ -1205,6 +1266,94 @@ export class DatabaseService { stmt.run(timestamp, id); } + // --- Auto-Heal Policies --- + + public getAutoHealPolicies(stackName?: string): AutoHealPolicy[] { + if (stackName) { + return this.db.prepare('SELECT * FROM auto_heal_policies WHERE stack_name = ?').all(stackName) as AutoHealPolicy[]; + } + return this.db.prepare('SELECT * FROM auto_heal_policies').all() as AutoHealPolicy[]; + } + + public getAutoHealPolicy(id: number): AutoHealPolicy | undefined { + return this.db.prepare('SELECT * FROM auto_heal_policies WHERE id = ?').get(id) as AutoHealPolicy | undefined; + } + + public addAutoHealPolicy(policy: Omit): AutoHealPolicy { + const stmt = this.db.prepare( + 'INSERT INTO auto_heal_policies (stack_name, service_name, unhealthy_duration_mins, cooldown_mins, max_restarts_per_hour, auto_disable_after_failures, enabled, consecutive_failures, last_fired_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' + ); + const result = stmt.run( + policy.stack_name, + policy.service_name ?? null, + policy.unhealthy_duration_mins, + policy.cooldown_mins, + policy.max_restarts_per_hour, + policy.auto_disable_after_failures, + policy.enabled, + policy.consecutive_failures, + policy.last_fired_at, + policy.created_at, + policy.updated_at + ); + return this.db.prepare('SELECT * FROM auto_heal_policies WHERE id = ?').get(result.lastInsertRowid) as AutoHealPolicy; + } + + public updateAutoHealPolicy(id: number, patch: Partial>): void { + const ALLOWED_KEYS = new Set([ + 'service_name', 'unhealthy_duration_mins', 'cooldown_mins', + 'max_restarts_per_hour', 'auto_disable_after_failures', + 'enabled', 'consecutive_failures', 'last_fired_at', + ]); + const entries = Object.entries(patch).filter(([k, v]) => ALLOWED_KEYS.has(k) && v !== undefined); + if (entries.length === 0) return; + const fields = entries.map(([k]) => `${k} = ?`).join(', '); + const values = entries.map(([, v]) => v); + this.db.prepare(`UPDATE auto_heal_policies SET ${fields}, updated_at = ? WHERE id = ?`).run(...values, Date.now(), id); + } + + public deleteAutoHealPolicy(id: number): void { + this.db.transaction(() => { + this.db.prepare('DELETE FROM auto_heal_history WHERE policy_id = ?').run(id); + this.db.prepare('DELETE FROM auto_heal_policies WHERE id = ?').run(id); + })(); + } + + public recordAutoHealHistory(entry: Omit): void { + this.db.prepare( + 'INSERT INTO auto_heal_history (policy_id, stack_name, service_name, container_name, container_id, action, reason, success, error, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' + ).run( + entry.policy_id, + entry.stack_name, + entry.service_name ?? null, + entry.container_name, + entry.container_id, + entry.action, + entry.reason, + entry.success, + entry.error ?? null, + entry.timestamp + ); + } + + public getAutoHealHistory(policyId: number, limit = 50): AutoHealHistoryEntry[] { + return this.db.prepare( + 'SELECT * FROM auto_heal_history WHERE policy_id = ? ORDER BY timestamp DESC LIMIT ?' + ).all(policyId, limit) as AutoHealHistoryEntry[]; + } + + public incrementConsecutiveFailures(policyId: number): void { + this.db.prepare('UPDATE auto_heal_policies SET consecutive_failures = consecutive_failures + 1, updated_at = ? WHERE id = ?').run(Date.now(), policyId); + } + + public resetConsecutiveFailures(policyId: number): void { + this.db.prepare('UPDATE auto_heal_policies SET consecutive_failures = 0, updated_at = ? WHERE id = ?').run(Date.now(), policyId); + } + + public setPolicyEnabled(policyId: number, enabled: boolean): void { + this.db.prepare('UPDATE auto_heal_policies SET enabled = ?, updated_at = ? WHERE id = ?').run(enabled ? 1 : 0, Date.now(), policyId); + } + // --- Notification History --- public getNotificationHistory(limit = 50): NotificationHistory[] { diff --git a/backend/src/services/DockerEventManager.ts b/backend/src/services/DockerEventManager.ts index aaaa06de..1612cbdd 100644 --- a/backend/src/services/DockerEventManager.ts +++ b/backend/src/services/DockerEventManager.ts @@ -71,6 +71,11 @@ export class DockerEventManager { return Array.from(this.services.values()).map(s => s.getStatus()); } + /** Returns the DockerEventService for a given local node, or undefined if not tracked. */ + public getService(nodeId: number): DockerEventService | undefined { + return this.services.get(nodeId); + } + // ======================================================================== // Node lifecycle handlers // ======================================================================== diff --git a/backend/src/services/DockerEventService.ts b/backend/src/services/DockerEventService.ts index c2ee4865..bdedfc25 100644 --- a/backend/src/services/DockerEventService.ts +++ b/backend/src/services/DockerEventService.ts @@ -24,6 +24,16 @@ import { getErrorMessage } from '../utils/errors'; * See docs/features/alerts-notifications.mdx for user-facing behaviour. */ +/** Snapshot of a single container's health tracking state, exposed to AutoHealService. */ +export interface ContainerHealthSnapshot { + id: string; + name?: string; + stackName?: string; + healthStatus?: 'healthy' | 'unhealthy' | 'starting'; + unhealthySince?: number; + lastKillAt?: number; +} + /** Grace window after a `die` before classifying, to absorb out-of-order kill events. */ const DIE_GRACE_WINDOW_MS = 500; @@ -61,6 +71,8 @@ interface InternalContainerState extends ContainerLifecycleState { stackName?: string; lastCrashAlertAt?: number; lastActivityAt: number; + healthStatus?: 'healthy' | 'unhealthy' | 'starting'; + unhealthySince?: number; } interface DockerEventPayload { @@ -400,16 +412,29 @@ export class DockerEventService { } private onHealthStatus(id: string, action: string, event: DockerEventPayload): void { - if (!action.includes('unhealthy')) return; const state = this.getOrCreateState(id, event); state.lastActivityAt = Date.now(); - if (!this.isCrashAlertsEnabled()) return; - const name = state.name ?? id.slice(0, 12); - const stackName = state.stackName; - void this.emitError( - `Healthcheck failed: ${name} is unhealthy.`, - stackName, - ); + + if (action.includes('unhealthy')) { + if (state.healthStatus !== 'unhealthy') { + state.unhealthySince = Date.now(); + } + state.healthStatus = 'unhealthy'; + if (!this.isCrashAlertsEnabled()) return; + const name = state.name ?? id.slice(0, 12); + const stackName = state.stackName; + void this.emitError( + `Healthcheck failed: ${name} is unhealthy.`, + stackName, + ); + } else { + state.unhealthySince = undefined; + if (action.includes('starting')) { + state.healthStatus = 'starting'; + } else { + state.healthStatus = 'healthy'; + } + } } private onStart(id: string): void { @@ -419,6 +444,8 @@ export class DockerEventService { state.lastKillAt = undefined; state.oomPending = undefined; state.lastCrashAlertAt = undefined; + state.unhealthySince = undefined; + state.healthStatus = 'starting'; state.lastActivityAt = Date.now(); } @@ -649,4 +676,32 @@ export class DockerEventService { trackedContainers: this.containerState.size, }; } + + // ======================================================================== + // Container state accessors (used by AutoHealService) + // ======================================================================== + + public listContainerStates(): ContainerHealthSnapshot[] { + return Array.from(this.containerState.entries()).map(([id, s]) => ({ + id, + name: s.name, + stackName: s.stackName, + healthStatus: s.healthStatus, + unhealthySince: s.unhealthySince, + lastKillAt: s.lastKillAt, + })); + } + + public getContainerState(id: string): ContainerHealthSnapshot | undefined { + const s = this.containerState.get(id); + if (!s) return undefined; + return { + id, + name: s.name, + stackName: s.stackName, + healthStatus: s.healthStatus, + unhealthySince: s.unhealthySince, + lastKillAt: s.lastKillAt, + }; + } } diff --git a/docs/docs.json b/docs/docs.json index 8ae35aba..e8bbe188 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -120,6 +120,7 @@ "features/vulnerability-scanning", "features/cve-suppressions", "features/auto-update-policies", + "features/auto-heal-policies", "features/scheduled-operations", "features/sso", "features/two-factor-authentication", diff --git a/docs/features/auto-heal-policies.mdx b/docs/features/auto-heal-policies.mdx new file mode 100644 index 00000000..a4de5e30 --- /dev/null +++ b/docs/features/auto-heal-policies.mdx @@ -0,0 +1,77 @@ +--- +title: "Auto-Heal Policies" +description: "Automatically restart containers that fail Docker healthchecks." +--- + + + Auto-Heal Policies require a **Skipper** or **Admiral** license. + + +## Overview + +Auto-Heal Policies let you define rules that restart containers when they have been in an `unhealthy` Docker healthcheck state for longer than a specified threshold. This keeps long-running services recoverable without manual intervention. + +## Prerequisites + +- Your containers must define a `HEALTHCHECK` instruction in their `Dockerfile` or in the `healthcheck` section of your `docker-compose.yml`. +- You must be an admin user. +- A Skipper or Admiral license. + +## Creating a Policy + +1. In the sidebar, right-click the stack you want to protect. +2. Select **Auto-Heal** from the context menu. +3. In the sheet that opens, fill in the form: + - **Service** — Select a specific service from your stack, or choose **All services** to apply the policy to every container. + - **Unhealthy for (minutes)** — How long a container must be continuously unhealthy before it is restarted. + - **Cooldown (minutes)** — How long to wait after a restart before evaluating the container again. + - **Max restarts per hour** — The maximum number of times this container can be restarted within a rolling hour window. + - **Auto-disable after (failures)** — How many consecutive failed restart attempts disable the policy automatically. +4. Click **Add Policy**. + + + Auto-Heal Policies sheet showing a policy for the web service + + +## Stack vs Service Scope + +- **All services** — The policy applies to every container in the stack that reports `unhealthy` status. +- **Named service** — The policy targets only the containers for that specific Compose service (matched by the `com.docker.compose.service` label). + +Multiple policies can coexist on the same stack. Each policy is evaluated independently. + +## Safety Rails + +Each policy includes four built-in safety mechanisms: + +**Cooldown period** — After a restart is triggered, the policy pauses evaluation for the configured number of minutes. This gives the container time to recover before being evaluated again. + +**Hourly restart cap** — If a container has been restarted the configured maximum number of times within the last hour, further restarts are skipped until the window clears. This prevents a persistently broken container from being restarted in a tight loop. + +**Recent user action suppression** — If you or another operator has manually stopped or restarted the container in the last 60 seconds, the policy skips evaluation for that container. This avoids interfering with in-progress manual interventions. + +**Auto-disable on repeated failures** — If the restart attempt itself fails (for example, because the Docker daemon is temporarily unavailable) the configured number of times in a row, the policy is automatically disabled. A notification is sent, and you can re-enable the policy from the sheet once the underlying issue is resolved. + +## Policy History + +Each policy row in the sheet can be expanded to show recent activity: restarts, skipped evaluations, and any auto-disable events. The history shows the container name, action taken, and the reason. + +## Troubleshooting + + + + The policy disabled itself after the configured number of consecutive restart failures. Check the container logs to understand why the restart is failing. Common causes include the Docker daemon being unavailable, insufficient system resources, or a misconfigured compose file. Once the issue is resolved, re-enable the policy from the Auto-Heal sheet. + + + + Verify that the container's Docker healthcheck actually reports `unhealthy`. A container can fail to start (exit code non-zero) without ever reaching the `unhealthy` state. Use `docker inspect ` and check `State.Health.Status`. + + If a manual restart was performed recently, the policy suppresses evaluation for 60 seconds after the restart to avoid conflicting with operator actions. + + Also check that the **Unhealthy for (minutes)** threshold has elapsed. The policy evaluates every 30 seconds, so there may be up to a 30-second delay between the threshold being crossed and the restart firing. + + + + If the container becomes unhealthy immediately after each restart, the hourly restart cap and the auto-disable-after-failures setting limit how many times this can occur. Review the container logs to address the root cause. You can lower the **Max restarts per hour** value or increase the **Unhealthy for (minutes)** threshold to reduce restart frequency while you investigate. + + diff --git a/e2e/auto-heal-policies.spec.ts b/e2e/auto-heal-policies.spec.ts new file mode 100644 index 00000000..9caa2629 --- /dev/null +++ b/e2e/auto-heal-policies.spec.ts @@ -0,0 +1,101 @@ +/** + * Auto-Heal Policies E2E tests - happy-path CRUD via the sheet UI. + * + * Opens the Auto-Heal sheet from the stack sidebar context menu, creates a + * policy, verifies it appears in the list, then deletes it. + * + * Requires a paid license (Skipper or Admiral) on the test instance. The test + * skips gracefully when the PaidGate upgrade prompt is detected instead. + */ +import { test, expect } from '@playwright/test'; +import { loginAs } from './helpers'; + +/** Wait for the stacks sidebar to finish loading. */ +async function waitForStacksLoaded(page: import('@playwright/test').Page) { + await expect(page.getByRole('button', { name: 'Create Stack' })).toBeVisible({ timeout: 15_000 }); + await expect(page.locator('[data-stacks-loaded="true"]')).toBeAttached({ timeout: 15_000 }); +} + +test.describe('Auto-Heal Policies', () => { + test.beforeEach(async ({ page }) => { + await loginAs(page); + await waitForStacksLoaded(page); + }); + + test('CRUD: create and delete a policy via the sheet', async ({ page }) => { + // Find the first stack item in the sidebar. cmdk renders items with role="option". + const stackItems = page.locator('[data-stacks-loaded="true"] [role="option"]'); + const count = await stackItems.count(); + if (count === 0) { + test.skip(); + return; + } + + // Right-click the first stack to open the context menu + await stackItems.first().click({ button: 'right' }); + + // Wait for the Radix context menu to appear + await expect(page.locator('[role="menu"]')).toBeVisible({ timeout: 5_000 }); + + // Click "Auto-Heal" menu item + await page.locator('[role="menu"]').getByText('Auto-Heal').click(); + + // The sheet title should be visible + await expect(page.getByText(/Auto-Heal Policies/)).toBeVisible({ timeout: 5_000 }); + + // Detect PaidGate: skip if the upgrade prompt blocks the UI (community instance) + const upgradePrompt = page.getByText(/requires a paid license/i); + if (await upgradePrompt.isVisible({ timeout: 2_000 }).catch(() => false)) { + test.skip(); + return; + } + + // Wait for the policy list and form to be ready + await expect(page.getByText('Active Policies')).toBeVisible({ timeout: 5_000 }); + await expect(page.getByRole('button', { name: 'Add Policy' })).toBeVisible({ timeout: 8_000 }); + + // ── Fill in the add-policy form ───────────────────────────────────────── + // Leave Service as default "All services" + + const unhealthyInput = page.locator('#unhealthy-duration'); + await unhealthyInput.clear(); + await unhealthyInput.fill('2'); + + const cooldownInput = page.locator('#cooldown'); + await cooldownInput.clear(); + await cooldownInput.fill('5'); + + const maxRestartsInput = page.locator('#max-restarts'); + await maxRestartsInput.clear(); + await maxRestartsInput.fill('3'); + + const autoDisableInput = page.locator('#auto-disable'); + await autoDisableInput.clear(); + await autoDisableInput.fill('5'); + + // ── Submit ────────────────────────────────────────────────────────────── + await page.getByRole('button', { name: 'Add Policy' }).click(); + + // Wait for the save to complete (button re-enables) then verify the row appears + await expect(page.getByRole('button', { name: 'Add Policy' })).toBeEnabled({ timeout: 8_000 }); + + // The PolicyRow subtitle shows "Unhealthy for 2 min" for the value we entered + const policySubtitle = page.getByText(/Unhealthy for 2 min/i).first(); + await expect(policySubtitle).toBeVisible({ timeout: 8_000 }); + + // ── Delete the policy ─────────────────────────────────────────────────── + // Find the policy card containing the subtitle and click its delete button + const policyCard = page + .locator('.rounded-lg.border') + .filter({ hasText: 'Unhealthy for 2 min' }) + .first(); + await expect(policyCard).toBeVisible({ timeout: 5_000 }); + + const deleteBtn = policyCard.getByRole('button', { name: 'Delete policy' }); + await expect(deleteBtn).toBeVisible({ timeout: 5_000 }); + await deleteBtn.click(); + + // Confirm the policy row is removed + await expect(policySubtitle).not.toBeVisible({ timeout: 8_000 }); + }); +}); diff --git a/frontend/src/components/EditorLayout.tsx b/frontend/src/components/EditorLayout.tsx index 964ef636..95c77350 100644 --- a/frontend/src/components/EditorLayout.tsx +++ b/frontend/src/components/EditorLayout.tsx @@ -46,6 +46,7 @@ import { Sheet, SheetContent, SheetTrigger } from './ui/sheet'; import { cn } from '@/lib/utils'; import { SettingsModal } from './SettingsModal'; import { StackAlertSheet } from './StackAlertSheet'; +import { StackAutoHealSheet } from '@/components/StackAutoHealSheet'; import { GitSourcePanel } from './stack/GitSourcePanel'; import { AppStoreView } from './AppStoreView'; import { LogViewer } from './LogViewer'; @@ -239,6 +240,7 @@ export default function EditorLayout() { const [settingsInitialSection, setSettingsInitialSection] = useState<'account' | 'labels'>('account'); const [alertSheetOpen, setAlertSheetOpen] = useState(false); const [alertSheetStack, setAlertSheetStack] = useState(''); + const [autoHealStackName, setAutoHealStackName] = useState(null); // Mobile navigation sheet state const [mobileNavOpen, setMobileNavOpen] = useState(false); @@ -2126,6 +2128,10 @@ export default function EditorLayout() { Alerts + setAutoHealStackName(file)}> + + Auto-Heal + {isPaid && ( @@ -2923,6 +2929,13 @@ export default function EditorLayout() { stackName={alertSheetStack} /> + {/* Stack Auto-Heal Sheet */} + { if (!open) setAutoHealStackName(null); }} + /> + {/* Git Source Panel */} {stackName && ( void; +} + +const clampNonNegative = (setter: (v: string) => void) => (e: React.ChangeEvent) => { + let val = e.target.value; + if (val !== '' && Number(val) < 0) val = '0'; + setter(val); +}; + +function actionColorClass(action: AutoHealHistoryEntry['action']): string { + if (action === 'restarted') return 'text-success'; + if (action === 'failed' || action === 'policy_auto_disabled') return 'text-destructive'; + return 'text-muted-foreground'; +} + +function actionLabel(action: AutoHealHistoryEntry['action']): string { + switch (action) { + case 'restarted': return 'Restarted'; + case 'skipped_user_action': return 'Skipped (user action)'; + case 'skipped_cooldown': return 'Skipped (cooldown)'; + case 'skipped_rate_limit': return 'Skipped (rate limit)'; + case 'failed': return 'Failed'; + case 'policy_auto_disabled': return 'Auto-disabled'; + } +} + +interface PolicyRowProps { + policy: AutoHealPolicy; + onDelete: (id: number) => void; + onToggle: (id: number, enabled: boolean) => void; + deleting: boolean; + saving: boolean; +} + +function PolicyRow({ policy, onDelete, onToggle, deleting, saving }: PolicyRowProps) { + const [historyOpen, setHistoryOpen] = useState(false); + const [history, setHistory] = useState([]); + const [loadingHistory, setLoadingHistory] = useState(false); + + const toggleHistory = async () => { + if (!historyOpen && history.length === 0 && policy.id != null) { + setLoadingHistory(true); + try { + const res = await apiFetch(`/auto-heal/policies/${policy.id}/history`); + if (res.ok) { + const data: AutoHealHistoryEntry[] = await res.json(); + setHistory(data); + } else { + const err = await res.json().catch(() => ({})) as Record; + toast.error((err?.message as string) || (err?.error as string) || 'Failed to load history.'); + } + } catch (e) { + console.error('[StackAutoHealSheet] Failed to fetch history:', e); + toast.error('Network error. Could not reach the node.'); + } finally { + setLoadingHistory(false); + } + } + setHistoryOpen(prev => !prev); + }; + + return ( +
+
+
+ + {policy.service_name ?? All services} + + + Unhealthy for {policy.unhealthy_duration_mins} min + • Cooldown: {policy.cooldown_mins} min + • Max {policy.max_restarts_per_hour}/hr + + {policy.consecutive_failures > 0 && ( + + + {policy.consecutive_failures} failure{policy.consecutive_failures !== 1 ? 's' : ''} + + + )} +
+
+ policy.id != null && onToggle(policy.id, checked)} + disabled={saving} + aria-label={`Toggle policy for ${policy.service_name ?? 'all services'}`} + /> + + +
+
+ + {historyOpen && ( +
+

Recent Activity

+ {history.length === 0 ? ( +

No history yet.

+ ) : ( + history.map((entry) => ( +
+ + {new Date(entry.timestamp).toLocaleString()} + + + {entry.container_name} + + + {actionLabel(entry.action)} + + + {entry.reason} + +
+ )) + )} +
+ )} +
+ ); +} + +export function StackAutoHealSheet({ stackName, open, onOpenChange }: StackAutoHealSheetProps) { + const [policies, setPolicies] = useState([]); + const [loading, setLoading] = useState(false); + const [saving, setSaving] = useState(false); + const [deleting, setDeleting] = useState(false); + const [serviceOptions, setServiceOptions] = useState<{ value: string; label: string }[]>([]); + + // Form state + const [service, setService] = useState(''); + const [unhealthyFor, setUnhealthyFor] = useState('5'); + const [cooldown, setCooldown] = useState('5'); + const [maxRestarts, setMaxRestarts] = useState('3'); + const [autoDisableAfter, setAutoDisableAfter] = useState('5'); + + useEffect(() => { + if (!open || !stackName) return; + + setLoading(true); + apiFetch(`/auto-heal/policies?stackName=${encodeURIComponent(stackName)}`) + .then(res => res.json() as Promise) + .then(data => setPolicies(data)) + .catch(() => toast.error('Failed to load auto-heal policies.')) + .finally(() => setLoading(false)); + + apiFetch(`/stacks/${encodeURIComponent(stackName)}/services`) + .then(res => res.json() as Promise) + .then(names => setServiceOptions(names.map(n => ({ value: n, label: n })))) + .catch(() => { /* services list is optional, silently skip */ }); + }, [open, stackName]); + + const handleToggle = async (id: number, enabled: boolean) => { + setSaving(true); + try { + const res = await apiFetch(`/auto-heal/policies/${id}`, { + method: 'PATCH', + body: JSON.stringify({ enabled: enabled ? 1 : 0 }), + }); + if (res.ok) { + setPolicies(prev => + prev.map(p => p.id === id ? { ...p, enabled: enabled ? 1 : 0 } : p) + ); + } else { + const err = await res.json().catch(() => ({})) as Record; + toast.error((err?.message as string) || (err?.error as string) || 'Failed to update policy.'); + } + } catch (e) { + console.error('[StackAutoHealSheet] Failed to toggle policy:', e); + toast.error('Network error. Could not reach the node.'); + } finally { + setSaving(false); + } + }; + + const handleDelete = async (id: number) => { + setDeleting(true); + try { + const res = await apiFetch(`/auto-heal/policies/${id}`, { method: 'DELETE' }); + if (res.ok) { + toast.success('Policy deleted.'); + setPolicies(prev => prev.filter(p => p.id !== id)); + } else { + const err = await res.json().catch(() => ({})) as Record; + toast.error((err?.message as string) || (err?.error as string) || 'Failed to delete policy.'); + } + } catch (e) { + console.error('[StackAutoHealSheet] Failed to delete policy:', e); + toast.error('Network error. Could not reach the node.'); + } finally { + setDeleting(false); + } + }; + + const handleAddPolicy = async () => { + setSaving(true); + const body = { + stack_name: stackName, + service_name: service === '' ? null : service, + unhealthy_duration_mins: parseInt(unhealthyFor, 10) || 5, + cooldown_mins: parseInt(cooldown, 10) || 5, + max_restarts_per_hour: parseInt(maxRestarts, 10) || 3, + auto_disable_after_failures: parseInt(autoDisableAfter, 10) || 5, + }; + try { + const res = await apiFetch('/auto-heal/policies', { + method: 'POST', + body: JSON.stringify(body), + }); + if (res.ok) { + toast.success('Policy added.'); + setService(''); + setUnhealthyFor('5'); + setCooldown('5'); + setMaxRestarts('3'); + setAutoDisableAfter('5'); + apiFetch(`/auto-heal/policies?stackName=${encodeURIComponent(stackName)}`) + .then(res => res.json() as Promise) + .then(data => setPolicies(data)) + .catch(() => toast.error('Failed to reload policies.')); + } else { + const err = await res.json().catch(() => ({})) as Record; + toast.error((err?.message as string) || (err?.error as string) || 'Failed to add policy.'); + console.error('[StackAutoHealSheet] addPolicy failed:', err); + } + } catch (e) { + console.error('[StackAutoHealSheet] addPolicy threw:', e); + toast.error('Network error. Could not reach the node.'); + } finally { + setSaving(false); + } + }; + + const serviceComboOptions = [ + { value: '', label: 'All services' }, + ...serviceOptions, + ]; + + return ( + + + + + Auto-Heal Policies: {stackName} + + Configure auto-heal policies to automatically restart unhealthy containers in this stack. + + + + +
+ {/* Existing policies */} +
+

Active Policies

+ {loading ? ( +
+ + Loading policies... +
+ ) : policies.length === 0 ? ( +
+ No auto-heal policies configured for this stack. +
+ ) : ( + policies.map(policy => ( + + )) + )} +
+ +
+ + {/* Add new policy form */} +
+

Add New Policy

+ +
+ + +
+ +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + + ); +}