diff --git a/backend/src/__tests__/AutoHealService.evaluate.test.ts b/backend/src/__tests__/AutoHealService.evaluate.test.ts index e0a72b07..4a84caaa 100644 --- a/backend/src/__tests__/AutoHealService.evaluate.test.ts +++ b/backend/src/__tests__/AutoHealService.evaluate.test.ts @@ -8,6 +8,7 @@ let DockerController: typeof import('../services/DockerController').default; let DockerEventManager: typeof import('../services/DockerEventManager').DockerEventManager; let LicenseService: typeof import('../services/LicenseService').LicenseService; let NotificationService: typeof import('../services/NotificationService').NotificationService; +let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry; function makePolicy( db: import('../services/DatabaseService').DatabaseService, @@ -46,6 +47,7 @@ beforeAll(async () => { ({ DockerEventManager } = await import('../services/DockerEventManager')); ({ LicenseService } = await import('../services/LicenseService')); ({ NotificationService } = await import('../services/NotificationService')); + ({ NodeRegistry } = await import('../services/NodeRegistry')); }); beforeEach(() => { @@ -77,9 +79,9 @@ describe('AutoHealService.evaluate', () => { const db = DatabaseService.getInstance(); makePolicy(db, { proxy_entitled_until: Date.now() + 60_000 }); vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); - const getRunningContainers = vi.fn().mockResolvedValue([]); + const getAllContainers = vi.fn().mockResolvedValue([]); const getInstance = vi.spyOn(DockerController, 'getInstance').mockReturnValue({ - getRunningContainers, + getAllContainers, } as unknown as ReturnType); await resetAutoHealSingleton().evaluate(); @@ -93,7 +95,7 @@ describe('AutoHealService.evaluate', () => { vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); const restartContainer = vi.fn().mockResolvedValue(undefined); vi.spyOn(DockerController, 'getInstance').mockReturnValue({ - getRunningContainers: vi.fn().mockResolvedValue([{ + getAllContainers: vi.fn().mockResolvedValue([{ Id: 'container-1', Names: ['/heal-stack-web-1'], Labels: { @@ -118,6 +120,190 @@ describe('AutoHealService.evaluate', () => { expect(restartContainer).toHaveBeenCalledWith('container-1'); const history = db.getAutoHealHistory(policy.id!); expect(history[0]).toMatchObject({ action: 'restarted', success: 1 }); + expect(history[0].reason).toContain('unhealthy'); + }); + + it('heals a crashed container that stayed down past the threshold', async () => { + const db = DatabaseService.getInstance(); + const policy = makePolicy(db); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const restartContainer = vi.fn().mockResolvedValue(undefined); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getAllContainers: vi.fn().mockResolvedValue([{ + Id: 'crash-1', + Names: ['/heal-stack-worker-1'], + Labels: { + 'com.docker.compose.project': 'heal-stack', + 'com.docker.compose.service': 'worker', + }, + State: 'exited', + Status: 'Exited (1) 2 minutes ago', + }]), + restartContainer, + } as unknown as ReturnType); + vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({ + getContainerState: () => ({ id: 'crash-1', crashedAt: Date.now() - 2 * 60_000 }), + } as unknown as ReturnType['getService']>); + + await resetAutoHealSingleton().evaluate(); + + expect(restartContainer).toHaveBeenCalledWith('crash-1'); + const history = db.getAutoHealHistory(policy.id!); + expect(history[0]).toMatchObject({ action: 'restarted', success: 1 }); + expect(history[0].reason).toContain('crashed'); + }); + + it('heals a container in the "dead" state when it crashed', async () => { + const db = DatabaseService.getInstance(); + makePolicy(db); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const restartContainer = vi.fn().mockResolvedValue(undefined); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getAllContainers: vi.fn().mockResolvedValue([{ + Id: 'dead-1', + Names: ['/heal-stack-worker-1'], + Labels: { + 'com.docker.compose.project': 'heal-stack', + 'com.docker.compose.service': 'worker', + }, + State: 'dead', + Status: 'Dead', + }]), + restartContainer, + } as unknown as ReturnType); + vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({ + getContainerState: () => ({ id: 'dead-1', crashedAt: Date.now() - 2 * 60_000 }), + } as unknown as ReturnType['getService']>); + + await resetAutoHealSingleton().evaluate(); + + expect(restartContainer).toHaveBeenCalledWith('dead-1'); + }); + + it('does not heal a recovered container that still carries a stale crash signal', async () => { + // Container crashed earlier (crashedAt set) but is running again now; the + // live running state must win so we do not restart a healthy container. + const db = DatabaseService.getInstance(); + makePolicy(db); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const restartContainer = vi.fn().mockResolvedValue(undefined); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getAllContainers: vi.fn().mockResolvedValue([{ + Id: 'recovered-1', + Names: ['/heal-stack-worker-1'], + Labels: { + 'com.docker.compose.project': 'heal-stack', + 'com.docker.compose.service': 'worker', + }, + State: 'running', + Status: 'Up 30 seconds', + }]), + restartContainer, + } as unknown as ReturnType); + vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({ + getContainerState: () => ({ id: 'recovered-1', crashedAt: Date.now() - 5 * 60_000 }), + } as unknown as ReturnType['getService']>); + + await resetAutoHealSingleton().evaluate(); + + expect(restartContainer).not.toHaveBeenCalled(); + }); + + it('does not heal an exited container with no crash signal (operator stop / clean exit)', async () => { + const db = DatabaseService.getInstance(); + makePolicy(db); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const restartContainer = vi.fn().mockResolvedValue(undefined); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getAllContainers: vi.fn().mockResolvedValue([{ + Id: 'stopped-1', + Names: ['/heal-stack-worker-1'], + Labels: { + 'com.docker.compose.project': 'heal-stack', + 'com.docker.compose.service': 'worker', + }, + State: 'exited', + Status: 'Exited (0) 2 minutes ago', + }]), + restartContainer, + } as unknown as ReturnType); + vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({ + getContainerState: () => ({ id: 'stopped-1' }), // no crashedAt: not a crash + } as unknown as ReturnType['getService']>); + + await resetAutoHealSingleton().evaluate(); + + expect(restartContainer).not.toHaveBeenCalled(); + }); + + it('restarts a container at most once per pass when policies overlap', async () => { + const db = DatabaseService.getInstance(); + makePolicy(db); // all services + makePolicy(db, { service_name: 'worker' }); // service-specific, same container + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const restartContainer = vi.fn().mockResolvedValue(undefined); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getAllContainers: vi.fn().mockResolvedValue([{ + Id: 'dup-1', + Names: ['/heal-stack-worker-1'], + Labels: { + 'com.docker.compose.project': 'heal-stack', + 'com.docker.compose.service': 'worker', + }, + State: 'running', + Status: 'Up 5 minutes (unhealthy)', + }]), + restartContainer, + } as unknown as ReturnType); + vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({ + getContainerState: () => undefined, + } as unknown as ReturnType['getService']>); + + const service = resetAutoHealSingleton(); + (service as unknown as { observedUnhealthySince: Map }).observedUnhealthySince + .set('1:dup-1', Date.now() - 2 * 60_000); + + await service.evaluate(); + + expect(restartContainer).toHaveBeenCalledTimes(1); + }); + + it('counts a failed restart toward cooldown and the hourly cap', async () => { + const db = DatabaseService.getInstance(); + const policy = makePolicy(db); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const restartContainer = vi.fn().mockRejectedValue(new Error('no such container')); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getAllContainers: vi.fn().mockResolvedValue([{ + Id: 'fail-1', + Names: ['/heal-stack-web-1'], + Labels: { + 'com.docker.compose.project': 'heal-stack', + 'com.docker.compose.service': 'web', + }, + State: 'running', + Status: 'Up 1 minute (unhealthy)', + }]), + restartContainer, + } as unknown as ReturnType); + vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({ + getContainerState: () => undefined, + } as unknown as ReturnType['getService']>); + + const service = resetAutoHealSingleton(); + (service as unknown as { observedUnhealthySince: Map }).observedUnhealthySince + .set('1:fail-1', Date.now() - 2 * 60_000); + + await service.evaluate(); + + expect(restartContainer).toHaveBeenCalledTimes(1); + const updated = db.getAutoHealPolicy(policy.id!); + expect(updated!.last_fired_at).toBeGreaterThan(0); + expect(updated!.consecutive_failures).toBe(1); + const ts = (service as unknown as { restartTimestamps: Map }).restartTimestamps.get('1:fail-1'); + expect(ts).toHaveLength(1); + const history = db.getAutoHealHistory(policy.id!); + expect(history[0]).toMatchObject({ action: 'failed', success: 0 }); }); it('records Docker unavailable history once per throttle window when listing containers fails', async () => { @@ -125,7 +311,7 @@ describe('AutoHealService.evaluate', () => { const policy = makePolicy(db); vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); vi.spyOn(DockerController, 'getInstance').mockReturnValue({ - getRunningContainers: vi.fn().mockRejectedValue(new Error('permission denied')), + getAllContainers: vi.fn().mockRejectedValue(new Error('permission denied')), } as unknown as ReturnType); const service = resetAutoHealSingleton(); @@ -153,9 +339,9 @@ describe('AutoHealService.evaluate', () => { }); makePolicy(db, { node_id: secondNodeId, stack_name: 'second-stack' }); vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); - const getRunningContainers = vi.fn().mockResolvedValue([]); + const getAllContainers = vi.fn().mockResolvedValue([]); const getInstance = vi.spyOn(DockerController, 'getInstance').mockReturnValue({ - getRunningContainers, + getAllContainers, } as unknown as ReturnType); await resetAutoHealSingleton().evaluate(); @@ -176,7 +362,7 @@ describe('AutoHealService.evaluate', () => { }); const policy = makePolicy(db, { node_id: secondNodeId, stack_name: 'second-rate-stack' }); vi.spyOn(DockerController, 'getInstance').mockReturnValue({ - getRunningContainers: vi.fn().mockResolvedValue([]), + getAllContainers: vi.fn().mockResolvedValue([]), } as unknown as ReturnType); const service = resetAutoHealSingleton(); @@ -195,7 +381,7 @@ describe('AutoHealService.evaluate', () => { const db = DatabaseService.getInstance(); const policy = makePolicy(db, { stack_name: 'cleanup-stack' }); vi.spyOn(DockerController, 'getInstance').mockReturnValue({ - getRunningContainers: vi.fn().mockResolvedValue([]), + getAllContainers: vi.fn().mockResolvedValue([]), } as unknown as ReturnType); const service = resetAutoHealSingleton(); @@ -214,6 +400,120 @@ describe('AutoHealService.evaluate', () => { expect(internals.historyTimestamps.size).toBe(0); }); + it('refreshes proxied remote leases from a paid controlling instance', async () => { + const db = DatabaseService.getInstance(); + db.addNode({ + name: 'lease-remote', + type: 'remote', + compose_dir: '', + is_default: false, + api_url: 'http://remote:1852', + api_token: 'tok', + }); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + vi.spyOn(LicenseService.getInstance(), 'getProxyHeaders').mockReturnValue( + { tier: 'paid', variant: 'admiral' } as ReturnType['getProxyHeaders']>, + ); + vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ + apiUrl: 'http://remote:1852', + apiToken: 'tok', + }); + const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ ok: true } as Response); + + const service = resetAutoHealSingleton(); + await (service as unknown as { refreshRemoteLeases: () => Promise }).refreshRemoteLeases(); + + expect(fetchSpy).toHaveBeenCalledTimes(1); + const [url, opts] = fetchSpy.mock.calls[0]; + expect(String(url)).toContain('/api/auto-heal/policies'); + expect((opts as RequestInit).headers).toMatchObject({ 'x-sencho-tier': 'paid' }); + }); + + it('does not refresh remote leases when the controlling instance is not paid', async () => { + const db = DatabaseService.getInstance(); + db.addNode({ + name: 'lease-remote-community', + type: 'remote', + compose_dir: '', + is_default: false, + api_url: 'http://remote2:1852', + api_token: 'tok2', + }); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ ok: true } as Response); + + const service = resetAutoHealSingleton(); + await (service as unknown as { refreshRemoteLeases: () => Promise }).refreshRemoteLeases(); + + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('keeps refreshing other remotes when one node is unreachable', async () => { + const db = DatabaseService.getInstance(); + db.addNode({ + name: 'good-remote', + type: 'remote', + compose_dir: '', + is_default: false, + api_url: 'http://good-host:1852', + api_token: 'tok', + }); + db.addNode({ + name: 'unreachable-remote', + type: 'remote', + compose_dir: '', + is_default: false, + api_url: 'http://bad-host:1852', + api_token: 'tok', + }); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + vi.spyOn(LicenseService.getInstance(), 'getProxyHeaders').mockReturnValue( + { tier: 'paid', variant: 'admiral' } as ReturnType['getProxyHeaders']>, + ); + const fetchSpy = vi.spyOn(global, 'fetch').mockImplementation(((input: unknown) => { + const url = String(input); + if (url.includes('bad-host')) return Promise.reject(new Error('ECONNREFUSED')); + return Promise.resolve({ ok: true } as Response); + }) as typeof fetch); + + const service = resetAutoHealSingleton(); + // One node rejecting must not throw or block the others. + await expect( + (service as unknown as { refreshRemoteLeases: () => Promise }).refreshRemoteLeases(), + ).resolves.toBeUndefined(); + + const calledUrls = fetchSpy.mock.calls.map(c => String(c[0])); + expect(calledUrls.some(u => u.includes('good-host'))).toBe(true); + expect(calledUrls.some(u => u.includes('bad-host'))).toBe(true); + }); + + it('warns after repeated lease refreshes find no reachable proxy target', async () => { + const db = DatabaseService.getInstance(); + db.addNode({ + name: 'no-target-remote', + type: 'remote', + compose_dir: '', + is_default: false, + api_url: '', + api_token: '', + }); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + vi.spyOn(LicenseService.getInstance(), 'getProxyHeaders').mockReturnValue( + { tier: 'paid', variant: 'admiral' } as ReturnType['getProxyHeaders']>, + ); + vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue(null); + const fetchSpy = vi.spyOn(global, 'fetch'); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + + const svc = resetAutoHealSingleton() as unknown as { refreshRemoteLeases: () => Promise }; + await svc.refreshRemoteLeases(); + await svc.refreshRemoteLeases(); + expect(warnSpy).not.toHaveBeenCalled(); + await svc.refreshRemoteLeases(); // third consecutive failure crosses the threshold + expect(warnSpy).toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + it('does not create duplicate timers when start is called twice', () => { vi.useFakeTimers(); try { @@ -224,9 +524,14 @@ describe('AutoHealService.evaluate', () => { service.start(); service.start(); + // One deferred-first-tick timeout, regardless of how many start() calls. expect(setTimeoutSpy).toHaveBeenCalledTimes(1); - vi.advanceTimersByTime(10_000); + // The lease-refresh interval is created immediately; the eval interval + // only after the initial delay fires. Calling start() twice must not + // duplicate either. expect(setIntervalSpy).toHaveBeenCalledTimes(1); + vi.advanceTimersByTime(10_000); + expect(setIntervalSpy).toHaveBeenCalledTimes(2); service.stop(); } finally { vi.useRealTimers(); diff --git a/backend/src/__tests__/AutoHealService.test.ts b/backend/src/__tests__/AutoHealService.test.ts index cd555aef..0025f317 100644 --- a/backend/src/__tests__/AutoHealService.test.ts +++ b/backend/src/__tests__/AutoHealService.test.ts @@ -3,7 +3,8 @@ * * shouldHeal is private; accessed via type cast (service as any) to avoid * exposing it in production API surface. All tests are pure (no I/O, no - * timers) - they exercise the decision function directly. + * timers) - they exercise the decision function directly against a normalized + * HealSignal. */ import { describe, it, expect, beforeEach } from 'vitest'; import { INTENTIONAL_KILL_WINDOW_MS } from '../services/ContainerLifecycleClassifier'; @@ -35,98 +36,108 @@ describe('AutoHealService.shouldHeal', () => { updated_at: Date.now(), }; - const baseState = { - id: 'container123', - name: 'mystack-web-1', - stackName: 'mystack', - healthStatus: 'unhealthy' as const, - unhealthySince: Date.now() - 6 * 60_000, // 6 minutes ago (past 5 min threshold) + // Unhealthy past the 5-min threshold, no recent kill. + const baseSignal = { + reason: 'unhealthy' as const, + downSince: Date.now() - 6 * 60_000, lastKillAt: undefined, }; it('returns heal:true when all conditions are met', () => { - const result = service.shouldHeal(baseState, basePolicy, 'container123', Date.now()); + const result = service.shouldHeal(baseSignal, basePolicy, 'container123', Date.now()); expect(result.heal).toBe(true); + expect(result.reason).toBe('unhealthy'); }); - it('returns heal:false when healthStatus is not unhealthy', () => { + it('returns heal:true with reason "crashed" for a crashed trigger', () => { const result = service.shouldHeal( - { ...baseState, healthStatus: 'healthy' }, + { reason: 'crashed', downSince: Date.now() - 6 * 60_000, lastKillAt: undefined }, + basePolicy, + 'container123', + Date.now(), + ); + expect(result.heal).toBe(true); + expect(result.reason).toBe('crashed'); + }); + + it('returns heal:false when there is no heal-worthy reason', () => { + const result = service.shouldHeal( + { reason: undefined, downSince: undefined }, basePolicy, 'container123', Date.now(), ); expect(result.heal).toBe(false); + expect(result.skipReason).toBe('not_unhealthy'); }); - it('returns heal:false when healthStatus is undefined', () => { + it('returns heal:false when downSince is undefined', () => { const result = service.shouldHeal( - { ...baseState, healthStatus: undefined }, - basePolicy, - 'container123', - Date.now(), - ); - expect(result.heal).toBe(false); - }); - - it('returns heal:false when state is undefined', () => { - const result = service.shouldHeal(undefined, basePolicy, 'container123', Date.now()); - expect(result.heal).toBe(false); - }); - - it('returns heal:false when unhealthySince is undefined', () => { - const result = service.shouldHeal( - { ...baseState, unhealthySince: undefined }, + { ...baseSignal, downSince: undefined }, basePolicy, 'container123', Date.now(), ); expect(result.heal).toBe(false); + expect(result.skipReason).toBe('not_unhealthy'); }); it('returns heal:false when duration threshold is not yet met', () => { // Only 2 minutes, threshold is 5 - const state = { ...baseState, unhealthySince: Date.now() - 2 * 60_000 }; - const result = service.shouldHeal(state, basePolicy, 'container123', Date.now()); + const signal = { ...baseSignal, downSince: Date.now() - 2 * 60_000 }; + const result = service.shouldHeal(signal, basePolicy, 'container123', Date.now()); expect(result.heal).toBe(false); expect(result.skipReason).toBe('duration_not_met'); }); it('returns skipped_user_action when lastKillAt is within the window', () => { // 30s ago, well within the 60s INTENTIONAL_KILL_WINDOW_MS - const state = { ...baseState, lastKillAt: Date.now() - 30_000 }; - const result = service.shouldHeal(state, basePolicy, 'container123', Date.now()); + const signal = { ...baseSignal, lastKillAt: Date.now() - 30_000 }; + const result = service.shouldHeal(signal, basePolicy, 'container123', Date.now()); + expect(result.heal).toBe(false); + expect(result.skipReason).toBe('skipped_user_action'); + }); + + it('suppresses a crashed trigger when lastKillAt is within the window', () => { + // An operator stop that Docker reports with a non-zero exit must not be + // resurrected: the kill-window is the backstop behind the crash classifier. + const signal = { + reason: 'crashed' as const, + downSince: Date.now() - 6 * 60_000, + lastKillAt: Date.now() - 30_000, + }; + const result = service.shouldHeal(signal, basePolicy, 'container123', Date.now()); expect(result.heal).toBe(false); expect(result.skipReason).toBe('skipped_user_action'); }); it('does not suppress when lastKillAt is outside the intentional kill window', () => { - const state = { - ...baseState, + const signal = { + ...baseSignal, lastKillAt: Date.now() - (INTENTIONAL_KILL_WINDOW_MS + 5_000), }; - const result = service.shouldHeal(state, basePolicy, 'container123', Date.now()); + const result = service.shouldHeal(signal, basePolicy, 'container123', Date.now()); expect(result.heal).toBe(true); }); it('returns skipped_cooldown when last_fired_at is within cooldown period', () => { // Fired 5 min ago, cooldown is 10 min const policy = { ...basePolicy, last_fired_at: Date.now() - 5 * 60_000, cooldown_mins: 10 }; - const result = service.shouldHeal(baseState, policy, 'container123', Date.now()); + const result = service.shouldHeal(baseSignal, policy, 'container123', Date.now()); expect(result.heal).toBe(false); expect(result.skipReason).toBe('skipped_cooldown'); }); it('does not apply cooldown when last_fired_at is 0', () => { const policy = { ...basePolicy, last_fired_at: 0 }; - const result = service.shouldHeal(baseState, policy, 'container123', Date.now()); + const result = service.shouldHeal(baseSignal, policy, 'container123', Date.now()); expect(result.heal).toBe(true); }); it('does not apply cooldown when last_fired_at exceeds the cooldown window', () => { // Fired 15 min ago, cooldown is 10 min const policy = { ...basePolicy, last_fired_at: Date.now() - 15 * 60_000, cooldown_mins: 10 }; - const result = service.shouldHeal(baseState, policy, 'container123', Date.now()); + const result = service.shouldHeal(baseSignal, policy, 'container123', Date.now()); expect(result.heal).toBe(true); }); @@ -135,7 +146,7 @@ describe('AutoHealService.shouldHeal', () => { // Pre-populate with 3 entries within the last hour (policy max is 3) const map = (service as any).restartTimestamps as Map; map.set('container123', [now - 10_000, now - 20_000, now - 30_000]); - const result = service.shouldHeal(baseState, basePolicy, 'container123', now); + const result = service.shouldHeal(baseSignal, basePolicy, 'container123', now); expect(result.heal).toBe(false); expect(result.skipReason).toBe('skipped_rate_limit'); }); @@ -145,7 +156,7 @@ describe('AutoHealService.shouldHeal', () => { const map = (service as any).restartTimestamps as Map; // All entries are >1 hour old, so they fall outside the rate-limit window map.set('container123', [now - 70 * 60_000, now - 80 * 60_000, now - 90 * 60_000]); - const result = service.shouldHeal(baseState, basePolicy, 'container123', now); + const result = service.shouldHeal(baseSignal, basePolicy, 'container123', now); expect(result.heal).toBe(true); }); @@ -154,17 +165,7 @@ describe('AutoHealService.shouldHeal', () => { const map = (service as any).restartTimestamps as Map; // 2 old (outside window) + 1 recent = 1 active restart; max is 3, so still allowed map.set('container123', [now - 70 * 60_000, now - 80 * 60_000, now - 5_000]); - const result = service.shouldHeal(baseState, basePolicy, 'container123', now); + const result = service.shouldHeal(baseSignal, basePolicy, 'container123', now); expect(result.heal).toBe(true); }); - - it('returns not_unhealthy as skipReason when container is healthy', () => { - const result = service.shouldHeal( - { ...baseState, healthStatus: 'healthy' }, - basePolicy, - 'container123', - Date.now(), - ); - expect(result.skipReason).toBe('not_unhealthy'); - }); }); diff --git a/backend/src/__tests__/docker-event-service.test.ts b/backend/src/__tests__/docker-event-service.test.ts index e405ee80..652516ce 100644 --- a/backend/src/__tests__/docker-event-service.test.ts +++ b/backend/src/__tests__/docker-event-service.test.ts @@ -142,6 +142,57 @@ describe('DockerEventService - die classification', () => { ); }); + it('stamps a crash signal and clears it on a later clean exit', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + // Crash: non-zero exit, no prior kill. + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c-clean', Attributes: { exitCode: '1', name: 'web' } }, + }); + await vi.advanceTimersByTimeAsync(600); + expect(service.getContainerState('c-clean')?.crashedAt).toBeTypeOf('number'); + + // A subsequent clean exit (code 0) must wipe the stale crash signal. + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c-clean', Attributes: { exitCode: '0', name: 'web' } }, + }); + await vi.advanceTimersByTimeAsync(600); + expect(service.getContainerState('c-clean')?.crashedAt).toBeUndefined(); + }); + + it('does not stamp a crash for a die that was superseded by a start', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + // Establish tracked state so the later start is recorded. + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'c-race', Attributes: { name: 'web' } }, + }); + // Crash, then restart strictly later but still within the 500ms grace + // window before the die is classified. + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'c-race', Attributes: { exitCode: '1', name: 'web' } }, + }); + await vi.advanceTimersByTimeAsync(100); + stream.push({ + Type: 'container', + Action: 'start', + Actor: { ID: 'c-race', Attributes: { name: 'web' } }, + }); + await vi.advanceTimersByTimeAsync(600); + + expect(service.getContainerState('c-race')?.crashedAt).toBeUndefined(); + }); + it('keeps stack and container routing for non-self compose crashes', async () => { service = new DockerEventService(1, 'local'); await service.start(); diff --git a/backend/src/services/AutoHealService.ts b/backend/src/services/AutoHealService.ts index 83a7d414..85b81e3e 100644 --- a/backend/src/services/AutoHealService.ts +++ b/backend/src/services/AutoHealService.ts @@ -4,7 +4,11 @@ import DockerController from './DockerController'; import { DockerEventManager } from './DockerEventManager'; import { ContainerHealthSnapshot } from './DockerEventService'; import { LicenseService } from './LicenseService'; +import { NodeRegistry } from './NodeRegistry'; import { NotificationService } from './NotificationService'; +import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './license-headers'; +import { isDebugEnabled } from '../utils/debug'; +import { getErrorMessage } from '../utils/errors'; // Dockerode listContainers shape (subset used here) type ContainerInfo = { @@ -15,19 +19,58 @@ type ContainerInfo = { Status?: string; }; +/** Why a container is eligible for healing. */ +type HealReason = 'unhealthy' | 'crashed'; + +/** + * Reasons shouldHeal declines to act. The `skipped_*` members are the subset + * recorded to history (the matching values exist in AutoHealHistoryEntry['action']); + * `not_unhealthy` and `duration_not_met` are internal-only and filtered out before + * any history write. + */ +type SkipReason = + | 'not_unhealthy' + | 'duration_not_met' + | 'skipped_user_action' + | 'skipped_cooldown' + | 'skipped_rate_limit'; + +/** + * Normalized heal trigger for a single container. `reason`/`downSince` are set + * only when there is something to act on; the safety rails (kill window, + * cooldown, hourly cap) are evaluated separately in shouldHeal. + */ +interface HealSignal { + reason?: HealReason; + /** When the heal-worthy condition began (unhealthy-since or crashed-at). */ + downSince?: number; + /** Last operator kill/stop, used to suppress healing right after a manual action. */ + lastKillAt?: number; +} + const EVAL_INTERVAL_MS = 30_000; const INITIAL_DELAY_MS = 10_000; const RATE_LIMIT_WINDOW_MS = 60 * 60_000; // 1 hour const HISTORY_THROTTLE_MS = 5 * 60_000; +// Refresh proxied remotes' entitlement well under the 5-minute lease the remote +// route grants, so a remote node's policies keep evaluating between operator visits. +const LEASE_REFRESH_INTERVAL_MS = 2 * 60_000; +const LEASE_REFRESH_TIMEOUT_MS = 10_000; +// Consecutive failed lease refreshes for one node before we surface a WARN. At the +// 2-minute cadence this is ~6 minutes of a node being unreachable, by which point +// its entitlement lease has lapsed and its auto-heal has stopped. +const LEASE_REFRESH_FAILURE_WARN_THRESHOLD = 3; export class AutoHealService { private static instance: AutoHealService; private intervalId: NodeJS.Timeout | null = null; private initialTimer: NodeJS.Timeout | null = null; + private leaseRefreshTimer: NodeJS.Timeout | null = null; private isProcessing = false; private restartTimestamps = new Map(); private observedUnhealthySince = new Map(); private historyTimestamps = new Map(); + private leaseRefreshFailures = new Map(); private constructor() {} @@ -44,6 +87,12 @@ export class AutoHealService { void this.evaluate(); this.intervalId = setInterval(() => void this.evaluate(), EVAL_INTERVAL_MS); }, INITIAL_DELAY_MS); + // Keep proxied remotes' auto-heal entitlement alive without depending on + // operator UI traffic (which would otherwise let the lease lapse). + this.leaseRefreshTimer = setInterval( + () => void this.refreshRemoteLeases(), + LEASE_REFRESH_INTERVAL_MS, + ); } stop(): void { @@ -55,6 +104,82 @@ export class AutoHealService { clearInterval(this.intervalId); this.intervalId = null; } + if (this.leaseRefreshTimer) { + clearInterval(this.leaseRefreshTimer); + this.leaseRefreshTimer = null; + } + } + + /** + * From a paid controlling instance, ping each enrolled remote node's auto-heal + * list endpoint so the remote renews its proxy entitlement lease. Without this, + * a Community-tier remote stops evaluating its policies a few minutes after the + * operator last opened the Auto-Heal sheet. Best-effort and per-node isolated: + * a single unreachable node never blocks the others or throws. + */ + private async refreshRemoteLeases(): Promise { + if (LicenseService.getInstance().getTier() !== 'paid') return; + const remotes = DatabaseService.getInstance().getNodes().filter(n => n.type === 'remote'); + if (remotes.length === 0) return; + + const registry = NodeRegistry.getInstance(); + const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); + // Drop failure counters for nodes that are no longer enrolled. + const remoteIds = new Set(remotes.map(n => n.id)); + for (const id of this.leaseRefreshFailures.keys()) { + if (!remoteIds.has(id)) this.leaseRefreshFailures.delete(id); + } + + await Promise.allSettled(remotes.map(async (node) => { + if (node.id === undefined) return; + const nodeId = node.id; + const target = registry.getProxyTarget(nodeId); + if (!target) { + // No reachable proxy target (e.g. a pilot-agent tunnel that is down + // or a remote missing its URL/token): count it so a persistently + // untargetable node is surfaced rather than silently skipped. + this.noteLeaseRefreshFailure(nodeId, 'no proxy target'); + return; + } + const baseUrl = target.apiUrl.replace(/\/$/, ''); + try { + const res = await fetch(`${baseUrl}/api/auto-heal/policies`, { + method: 'GET', + headers: { + 'Authorization': `Bearer ${target.apiToken}`, + [PROXY_TIER_HEADER]: proxyHeaders.tier, + [PROXY_VARIANT_HEADER]: proxyHeaders.variant ?? '', + }, + signal: AbortSignal.timeout(LEASE_REFRESH_TIMEOUT_MS), + }); + if (res.ok) { + this.leaseRefreshFailures.delete(nodeId); + } else { + this.noteLeaseRefreshFailure(nodeId, `HTTP ${res.status}`); + } + } catch (err) { + this.noteLeaseRefreshFailure(nodeId, getErrorMessage(err, 'unknown')); + } + })); + } + + /** + * Record a failed lease refresh for a node and surface a one-time WARN once it + * has failed enough times in a row that its auto-heal has almost certainly + * stopped, so a revoked token or a long-unreachable node is not silently lost. + */ + private noteLeaseRefreshFailure(nodeId: number, detail: string): void { + const count = (this.leaseRefreshFailures.get(nodeId) ?? 0) + 1; + this.leaseRefreshFailures.set(nodeId, count); + if (count === LEASE_REFRESH_FAILURE_WARN_THRESHOLD) { + console.warn( + `[AutoHeal] Could not refresh auto-heal entitlement for node ${nodeId} ` + + `after ${count} consecutive attempts (${detail}). Auto-heal on that node ` + + `will stop until it is reachable again.`, + ); + } else if (isDebugEnabled()) { + console.log(`[AutoHeal:diag] lease refresh for node ${nodeId} failed (attempt ${count}): ${detail}`); + } } async evaluate(): Promise { @@ -67,12 +192,18 @@ export class AutoHealService { // Evaluate only on local nodes (remote nodes self-monitor via their own instance) const nodes = db.getNodes().filter(n => n.type === 'local'); const now = Date.now(); + if (isDebugEnabled()) { + console.log(`[AutoHeal:diag] evaluate: ${nodes.length} local node(s), localPaid=${localPaid}`); + } for (const node of nodes) { const policies = db.getAutoHealPolicies(undefined, node.id).filter(p => p.enabled === 1 && (localPaid || p.proxy_entitled_until > now) ); this.pruneInactivePolicyHistory(node.id, policies); if (policies.length === 0) continue; + if (isDebugEnabled()) { + console.log(`[AutoHeal:diag] node ${node.id}: ${policies.length} active policy(ies)`); + } await this.evaluateForNode(node.id, policies); } } catch (err) { @@ -85,7 +216,9 @@ export class AutoHealService { private async evaluateForNode(nodeId: number, policies: AutoHealPolicy[]): Promise { let containers: ContainerInfo[]; try { - containers = await DockerController.getInstance(nodeId).getRunningContainers(); + // all:true so exited/crashed containers are visible too, not just + // running-but-unhealthy ones. + containers = await DockerController.getInstance(nodeId).getAllContainers(); } catch (err) { const now = Date.now(); const errorMsg = err instanceof Error ? err.message : String(err); @@ -134,6 +267,11 @@ export class AutoHealService { } this.pruneInactivePolicyHistory(nodeId, policies); + // Containers already restarted in this pass, so overlapping policies (an + // all-services policy plus a service-specific one) can't double-restart + // the same container and halve its hourly cap. + const healedThisPass = new Set(); + for (const policy of policies) { if (policy.id === undefined) { console.warn('[AutoHeal] skipping policy without id:', policy.stack_name); @@ -152,8 +290,8 @@ export class AutoHealService { const containerName = container.Names?.[0]?.replace(/^\//, '') ?? container.Id.slice(0, 12); const serviceOverride = container.Labels?.['com.docker.compose.service'] ?? null; - const state = this.getEffectiveState(nodeId, container, eventSvc?.getContainerState(container.Id), now); - const decision = this.shouldHeal(state, policy, this.containerKey(nodeId, container.Id), now); + const signal = this.getHealSignal(nodeId, container, eventSvc?.getContainerState(container.Id), now); + const decision = this.shouldHeal(signal, policy, this.containerKey(nodeId, container.Id), now); if (!decision.heal) { if ( @@ -167,7 +305,7 @@ export class AutoHealService { service_name: policy.service_name ?? serviceOverride, container_name: containerName, container_id: container.Id, - action: decision.skipReason as AutoHealHistoryEntry['action'], + action: decision.skipReason, reason: this.skipReasonText(decision.skipReason), success: 0, error: null, @@ -177,24 +315,48 @@ export class AutoHealService { continue; } + if (healedThisPass.has(container.Id)) continue; + healedThisPass.add(container.Id); + + if (isDebugEnabled()) { + const downForSec = signal.downSince ? Math.round((now - signal.downSince) / 1000) : 0; + console.log(`[AutoHeal:diag] healing ${containerName} on node ${nodeId}: reason=${decision.reason} downFor=${downForSec}s`); + } + await this.executeHeal( policy, nodeId, container.Id, containerName, policy.service_name ?? serviceOverride, + decision.reason ?? 'unhealthy', ); } } } - private getEffectiveState( + private getHealSignal( nodeId: number, container: ContainerInfo, eventState: ContainerHealthSnapshot | undefined, now: number, - ): ContainerHealthSnapshot | undefined { + ): HealSignal { const key = this.containerKey(nodeId, container.Id); + + // Stopped: heal only when the event stream classified the exit as a crash + // or OOM kill. crashedAt is never stamped for operator stops or clean + // exits, so intentionally-stopped containers are not resurrected. Checked + // before any health-text parsing so an exited container can never fall into + // the healthcheck path. + const rawState = (container.State ?? '').toLowerCase(); + if (rawState === 'exited' || rawState === 'dead') { + this.observedUnhealthySince.delete(key); + if (eventState?.crashedAt) { + return { reason: 'crashed', downSince: eventState.crashedAt, lastKillAt: eventState.lastKillAt }; + } + return { lastKillAt: eventState?.lastKillAt }; + } + const statusText = `${container.State ?? ''} ${container.Status ?? ''}`.toLowerCase(); const dockerHealth = statusText.includes('unhealthy') ? 'unhealthy' @@ -204,71 +366,66 @@ export class AutoHealService { ? 'starting' : undefined; + // Running but failing its healthcheck. if (dockerHealth === 'unhealthy') { - const unhealthySince = eventState?.healthStatus === 'unhealthy' && eventState.unhealthySince + const since = eventState?.healthStatus === 'unhealthy' && eventState.unhealthySince ? eventState.unhealthySince : this.observedUnhealthySince.get(key) ?? now; - this.observedUnhealthySince.set(key, unhealthySince); - return { - id: container.Id, - name: eventState?.name ?? container.Names?.[0]?.replace(/^\//, ''), - stackName: eventState?.stackName ?? container.Labels?.['com.docker.compose.project'], - healthStatus: 'unhealthy', - unhealthySince, - lastKillAt: eventState?.lastKillAt, - }; + this.observedUnhealthySince.set(key, since); + return { reason: 'unhealthy', downSince: since, lastKillAt: eventState?.lastKillAt }; } + // Running and healthy (or warming up): clear unhealthy tracking, nothing to heal. if (dockerHealth === 'healthy' || dockerHealth === 'starting') { this.observedUnhealthySince.delete(key); - return { - id: container.Id, - name: eventState?.name ?? container.Names?.[0]?.replace(/^\//, ''), - stackName: eventState?.stackName ?? container.Labels?.['com.docker.compose.project'], - healthStatus: dockerHealth, - lastKillAt: eventState?.lastKillAt, - }; + return { lastKillAt: eventState?.lastKillAt }; } - return eventState; + // Running without a Docker healthcheck (or freshly 'created'): fall back to + // any health the event stream recorded, otherwise no trigger. + if (eventState?.healthStatus === 'unhealthy' && eventState.unhealthySince) { + this.observedUnhealthySince.set(key, eventState.unhealthySince); + return { reason: 'unhealthy', downSince: eventState.unhealthySince, lastKillAt: eventState.lastKillAt }; + } + this.observedUnhealthySince.delete(key); + return { lastKillAt: eventState?.lastKillAt }; } private shouldHeal( - state: ContainerHealthSnapshot | undefined, + signal: HealSignal, policy: AutoHealPolicy, - containerId: string, + containerKey: string, now: number, - ): { heal: boolean; skipReason?: string } { - // No state tracked yet, or container is not unhealthy - if (!state || state.healthStatus !== 'unhealthy' || !state.unhealthySince) { + ): { heal: boolean; skipReason?: SkipReason; reason?: HealReason } { + // No heal-worthy condition (healthy, clean exit, or unclassified stop) + if (!signal.reason || !signal.downSince) { return { heal: false, skipReason: 'not_unhealthy' }; } // Duration threshold not yet met - const unhealthyMs = now - state.unhealthySince; - if (unhealthyMs < policy.unhealthy_duration_mins * 60_000) { + if (now - signal.downSince < policy.unhealthy_duration_mins * 60_000) { return { heal: false, skipReason: 'duration_not_met' }; } // Suppress if user recently killed the container - if (state.lastKillAt !== undefined && now - state.lastKillAt < INTENTIONAL_KILL_WINDOW_MS) { + if (signal.lastKillAt !== undefined && now - signal.lastKillAt < INTENTIONAL_KILL_WINDOW_MS) { return { heal: false, skipReason: 'skipped_user_action' }; } - // Cooldown: respect last_fired_at + // Cooldown: respect last_fired_at (set on every attempt, success or failure) if (policy.last_fired_at > 0 && now - policy.last_fired_at < policy.cooldown_mins * 60_000) { return { heal: false, skipReason: 'skipped_cooldown' }; } - // Rate limit: max restarts per hour - const recentRestarts = (this.restartTimestamps.get(containerId) ?? []).filter( + // Rate limit: max restart attempts per hour + const recentRestarts = (this.restartTimestamps.get(containerKey) ?? []).filter( t => now - t < RATE_LIMIT_WINDOW_MS, ); if (recentRestarts.length >= policy.max_restarts_per_hour) { return { heal: false, skipReason: 'skipped_rate_limit' }; } - return { heal: true }; + return { heal: true, reason: signal.reason }; } private async executeHeal( @@ -277,6 +434,7 @@ export class AutoHealService { containerId: string, containerName: string, serviceName: string | null, + reason: HealReason, ): Promise { const db = DatabaseService.getInstance(); const now = Date.now(); @@ -288,25 +446,31 @@ export class AutoHealService { container_id: containerId, timestamp: now, }; + const condition = reason === 'crashed' ? 'crashed and stayed down' : 'unhealthy'; + + // Stamp the attempt up front so the cooldown and hourly cap apply to a + // failed restart too. Otherwise a container that keeps failing to restart + // would be retried on every poll (every 30s) until auto-disable. + db.updateAutoHealPolicy(policy.id!, { last_fired_at: now }); + const restartKey = this.containerKey(nodeId, containerId); + const timestamps = (this.restartTimestamps.get(restartKey) ?? []).filter( + t => now - t < RATE_LIMIT_WINDOW_MS, + ); + timestamps.push(now); + this.restartTimestamps.set(restartKey, timestamps); try { await DockerController.getInstance(nodeId).restartContainer(containerId); + if (isDebugEnabled()) { + console.log(`[AutoHeal:diag] restart of ${containerName} on node ${nodeId} took ${Date.now() - now}ms`); + } db.resetConsecutiveFailures(policy.id!); - db.updateAutoHealPolicy(policy.id!, { last_fired_at: now }); - - const restartKey = this.containerKey(nodeId, containerId); - const timestamps = this.restartTimestamps.get(restartKey) ?? []; - timestamps.push(now); - this.restartTimestamps.set( - restartKey, - timestamps.filter(t => now - t < RATE_LIMIT_WINDOW_MS), - ); db.recordAutoHealHistory({ ...baseEntry, action: 'restarted', - reason: `Container unhealthy for ${policy.unhealthy_duration_mins} minute(s); auto-restarted.`, + reason: `Container ${condition} for ${policy.unhealthy_duration_mins} minute(s); auto-restarted.`, success: 1, error: null, }); @@ -326,7 +490,7 @@ export class AutoHealService { .dispatchAlert( 'info', 'autoheal_triggered', - `Auto-Heal: Restarted ${containerName} on stack ${policy.stack_name} after being unhealthy for ${policy.unhealthy_duration_mins} minute(s).`, + `Auto-Heal: Restarted ${containerName} on stack ${policy.stack_name} after being ${condition} for ${policy.unhealthy_duration_mins} minute(s).`, { stackName: policy.stack_name, containerName, actor: 'system:autoheal' }, ) .catch(err => console.error('[AutoHeal] notification dispatch failed:', err)); @@ -417,7 +581,7 @@ export class AutoHealService { return `${nodeId}:${containerId}`; } - private skipReasonText(reason: string): string { + private skipReasonText(reason: SkipReason): string { switch (reason) { case 'skipped_user_action': return 'Skipped: recent user action detected on this container.'; diff --git a/backend/src/services/DockerEventService.ts b/backend/src/services/DockerEventService.ts index 81f74cff..98bba059 100644 --- a/backend/src/services/DockerEventService.ts +++ b/backend/src/services/DockerEventService.ts @@ -33,6 +33,13 @@ export interface ContainerHealthSnapshot { healthStatus?: 'healthy' | 'unhealthy' | 'starting'; unhealthySince?: number; lastKillAt?: number; + /** + * Timestamp (ms) of the last exit classified as a crash or OOM kill, cleared + * when the container next starts. Set independently of the crash-alert toggle + * so Auto-Heal can distinguish a crash (heal-worthy) from an operator stop or + * clean exit (which are never stamped here). + */ + crashedAt?: number; } /** Grace window after a `die` before classifying, to absorb out-of-order kill events. */ @@ -85,6 +92,8 @@ interface InternalContainerState extends ContainerLifecycleState { lastActivityAt: number; healthStatus?: 'healthy' | 'unhealthy' | 'starting'; unhealthySince?: number; + crashedAt?: number; + lastStartAt?: number; } interface DockerEventPayload { @@ -427,12 +436,15 @@ export class DockerEventService { } private onDie(id: string, event: DockerEventPayload): void { + // Capture the die time at arrival, not inside the deferred classifier, so a + // start that races in during the grace window is correctly seen as later. + const dieAt = this.eventTimeMs(event); // Defer classification to absorb out-of-order kill events. const existing = this.pendingDieTimers.get(id); if (existing) clearTimeout(existing); const timer = setTimeout(() => { this.pendingDieTimers.delete(id); - void this.classifyDie(id, event); + void this.classifyDie(id, event, dieAt); }, DIE_GRACE_WINDOW_MS); this.pendingDieTimers.set(id, timer); } @@ -478,8 +490,10 @@ export class DockerEventService { state.lastKillAt = undefined; state.oomPending = undefined; state.lastCrashAlertAt = undefined; + state.crashedAt = undefined; state.unhealthySince = undefined; state.healthStatus = 'starting'; + state.lastStartAt = Date.now(); state.lastActivityAt = Date.now(); } @@ -492,7 +506,7 @@ export class DockerEventService { } } - private async classifyDie(id: string, event: DockerEventPayload): Promise { + private async classifyDie(id: string, event: DockerEventPayload, dieAt: number): Promise { const state = this.getOrCreateState(id, event); const exitCodeStr = event.Actor?.Attributes?.exitCode; const parsedExit = exitCodeStr !== undefined ? parseInt(exitCodeStr, 10) : undefined; @@ -500,7 +514,7 @@ export class DockerEventService { const now = Date.now(); let classification = classifyDie( - { at: this.eventTimeMs(event), exitCode }, + { at: dieAt, exitCode }, { lastKillAt: state.lastKillAt, oomPending: state.oomPending }, ); @@ -508,7 +522,25 @@ export class DockerEventService { state.oomPending = undefined; state.lastActivityAt = now; - if (classification === 'intentional' || classification === 'clean') return; + // A clean or intentional exit clears any prior crash signal, so a stale + // crash cannot outlive a later graceful stop and be mistaken for a fresh one. + if (classification === 'intentional' || classification === 'clean') { + state.crashedAt = undefined; + return; + } + + // If the container started again strictly after this die occurred, the die + // is superseded (the container recovered). Do not stamp a crash signal or + // alert for it; the classification is deferred 500ms, so an immediate + // restart can race ahead of this handler. A start at the same instant is + // treated as preceding a genuine re-crash, not superseding it. + if (state.lastStartAt !== undefined && state.lastStartAt > dieAt) return; + + // Stamp the heal signal for Auto-Heal before the alert dedup/toggle gates + // below. This must be independent of whether a crash alert is dispatched, + // so Auto-Heal still sees the crash when crash alerts are disabled or + // rate-suppressed. Cleared on the next `start` or a later clean exit. + state.crashedAt = now; // Dedup early: crashloops repeatedly reach this point with exit 137, // and the OOM fallback below issues a Docker inspect. Skipping the @@ -678,6 +710,10 @@ export class DockerEventService { if (this.containerState.size === 0) return; const cutoff = Date.now() - STATE_STALE_AFTER_MS; for (const [id, state] of this.containerState) { + // Keep state for a container with an unresolved crash so Auto-Heal can + // still act on it past the default stale window (policy thresholds run + // up to 24h). It is cleared on `start` and removed on `destroy`. + if (state.crashedAt !== undefined) continue; if (state.lastActivityAt < cutoff) { this.containerState.delete(id); } @@ -759,6 +795,7 @@ export class DockerEventService { healthStatus: s.healthStatus, unhealthySince: s.unhealthySince, lastKillAt: s.lastKillAt, + crashedAt: s.crashedAt, }; } } diff --git a/docs/features/auto-heal-policies.mdx b/docs/features/auto-heal-policies.mdx index dd7e8199..4f601d83 100644 --- a/docs/features/auto-heal-policies.mdx +++ b/docs/features/auto-heal-policies.mdx @@ -1,6 +1,6 @@ --- title: "Auto-Heal Policies" -description: "Restart containers that fail their Docker healthcheck and stay unhealthy, with per-policy thresholds and a built-in safety rail set." +description: "Restart containers that fail their Docker healthcheck or crash, with per-policy thresholds and a built-in safety rail set." --- @@ -9,16 +9,27 @@ description: "Restart containers that fail their Docker healthcheck and stay unh ## Overview -Auto-Heal Policies watch each container's Docker healthcheck and restart it when it has been continuously unhealthy for longer than you allow. Policies are scoped to a stack and can target every container in the stack or a single Compose service. Each policy runs with its own thresholds and four built-in safety rails so a persistently broken container cannot be restarted in a tight loop. +Auto-Heal Policies restart a container when it stays broken for longer than you allow. A policy acts on two conditions: + +- The container reports Docker's `unhealthy` status continuously past your threshold. +- The container crashes (exits with a non-zero code) and stays down past your threshold. + +Policies are scoped to a stack and can target every container in the stack or a single Compose service. Each policy runs with its own thresholds and four built-in safety rails so a persistently broken container cannot be restarted in a tight loop. + +A container that you stop yourself, or that exits cleanly (a one-shot job finishing with exit code 0), is never restarted. Auto-Heal only acts on failures, not on intentional shutdowns. Policies live next to your stack-level alert rules in the stack's **Monitor** sheet, which has two tabs: **Alerts** and **Auto-heal**. ## Prerequisites -- Containers must declare a `HEALTHCHECK` in the Dockerfile or a `healthcheck` block in `docker-compose.yml`. Auto-Heal only acts on containers that report a Docker health status; a container that fails to start or exits with a non-zero code without ever reaching `unhealthy` is not in scope. +- For **healthcheck-based** healing, containers must declare a `HEALTHCHECK` in the Dockerfile or a `healthcheck` block in `docker-compose.yml` so they report a Docker health status. **Crash-based** healing needs no healthcheck: any container that exits with a non-zero code qualifies. - You must be signed in as an admin. - A Skipper or Admiral license. + + Crash healing acts on crashes Sencho observes while it is running. A container that crashed before Sencho started is left for you to inspect rather than restarted automatically. + + ## Workflow 1. In the sidebar, right-click the stack you want to protect (or focus the stack and press **H**). @@ -37,7 +48,7 @@ You can add as many policies to a stack as you need. Each policy is evaluated in | Field | Meaning | |-------|---------| | **Service** | The Compose service this policy applies to. Leave on **All services** for a stack-wide policy, or pick a specific service to limit the policy to that service's containers. | -| **Unhealthy for (minutes)** | How long a container must be continuously reporting `unhealthy` before Auto-Heal restarts it. | +| **Unhealthy for (minutes)** | How long a container must stay broken before Auto-Heal restarts it. This applies to both conditions: continuously reporting `unhealthy`, or staying down after a crash. | | **Cooldown (minutes)** | After a restart fires, the policy pauses evaluation for this many minutes so the container has time to come back up. | | **Max restarts / hr** | The most times this policy will restart a given container in a rolling one-hour window. Once the cap is reached, further restarts are skipped until the window clears. | | **Auto-disable after (failures)** | If the restart call itself fails this many times in a row, the policy disables itself so it stops looping on a broken setup. The counter resets after any successful restart. | @@ -55,9 +66,9 @@ Each policy has four safety rails built in. They run before any restart. **Hourly cap.** The **Max restarts / hr** value is enforced as a rolling one-hour window per container. When the cap is reached, evaluations continue but restarts are skipped until the window clears. -**Recent operator action.** If you or another operator manually stops or restarts the container, the policy suppresses evaluation for that container for 60 seconds. This avoids fighting an operator who is intentionally power-cycling a service. +**Recent operator action.** If you or another operator manually stops or restarts the container, the policy suppresses evaluation for that container for 60 seconds. This avoids fighting an operator who is intentionally power-cycling a service. A clean stop is also never seen as a crash, so a service you take down stays down. -**Auto-disable.** If the restart call itself fails (for example, the Docker daemon is not reachable, or the container is in a state that cannot be restarted), the policy increments a consecutive-failure counter. When the counter reaches the configured **Auto-disable after (failures)** value, the policy disables itself and emits a notification. A successful restart resets the counter to zero. +**Auto-disable.** If the restart call itself fails (for example, the Docker daemon is not reachable, or the container is in a state that cannot be restarted), the policy increments a consecutive-failure counter. Failed attempts are spaced by the cooldown, so a broken setup is retried on the cooldown interval rather than on every evaluation tick. When the counter reaches the configured **Auto-disable after (failures)** value, the policy disables itself and emits a notification. A successful restart resets the counter to zero. ## Managing policies @@ -117,7 +128,7 @@ The dashboard's **Configuration status** card surfaces an **Auto-heal policies** - Auto-Heal only acts on Docker's `unhealthy` state. A container that crashes (non-zero exit) or never starts will not trigger a policy. Use `docker inspect ` and check `State.Health.Status` to confirm the container actually reaches `unhealthy`. + Auto-Heal acts on two conditions: a container that stays `unhealthy`, and a container that crashes (non-zero exit) and stays down. It does **not** act on a container that exited cleanly (exit code 0), one you stopped yourself, or a crash that happened before Sencho was running. For the healthcheck path, use `docker inspect ` and check `State.Health.Status` to confirm the container actually reaches `unhealthy`; for the crash path, check `State.ExitCode` is non-zero. Evaluation runs every 30 seconds, so there is up to a 30-second delay between the **Unhealthy for (minutes)** threshold being crossed and the restart firing. The 60-second operator-action window also suppresses evaluation right after a manual stop or restart. @@ -131,6 +142,6 @@ The dashboard's **Configuration status** card surfaces an **Auto-heal policies** - The tab is hidden for Community-tier installs. Upgrade to Skipper to unlock auto-heal alongside the rest of Sencho's automation surface. + Auto-Heal Policies require a Skipper or Admiral license and an admin sign-in. If the **Auto-heal** tab is not present on the **Monitor** sheet, confirm your license tier under **Settings → License** and that you are signed in as an admin.