feat(auto-heal): restart crashed containers and harden the heal loop (#1258)

* feat(auto-heal): restart crashed containers and harden the heal loop

Auto-Heal now restarts containers that crash (non-zero exit) and stay down
past the policy threshold, in addition to those that fail their Docker
healthcheck. Crash detection reuses the container event classifier so a
container that exits cleanly or that an operator stopped is never restarted;
only classified crashes set the heal signal.

Also hardens the existing loop:
- A paid controlling instance refreshes proxied remotes' entitlement on a
  background interval so a remote node's policies keep evaluating between
  operator visits instead of lapsing a few minutes after the sheet was last
  opened. A node that stays unreachable surfaces a warning.
- Overlapping policies (all-services plus a service-specific one) restart a
  given container at most once per evaluation pass, so the hourly cap holds.
- A failed restart now counts toward the cooldown and hourly cap, so a broken
  setup is retried on the cooldown interval rather than every poll.
- Diagnostic logging behind developer mode for evaluation, heal decisions,
  timing, and lease refresh.

* docs(auto-heal): document crash healing and refresh troubleshooting

Cover the two heal conditions (unhealthy and crashed), note that clean exits
and operator stops are never restarted and that crash healing acts on crashes
observed while Sencho is running, and update the troubleshooting and tab
visibility entries accordingly.

* fix(auto-heal): close stale crash-signal race and harden lease refresh

A crash signal could outlive the crash it described. The exit classifier is
deferred 500ms, so an immediate restart could let it stamp the crash marker
after the container was already running, and a later clean or operator-initiated
exit did not clear it; the next poll could then restart a container that had
exited cleanly. Now a clean or intentional exit always clears the marker, a die
that a start has superseded is not stamped, and the die's own time is captured at
arrival rather than at the deferred classification so the supersede check is
accurate.

Also:
- Crash state survives the event service's idle-prune window, so crash healing
  works for any configured threshold rather than only short ones.
- An exited or dead container is matched before any health-text parsing, so it
  can never fall into the healthcheck path.
- A remote with no reachable proxy target counts toward the lease-refresh
  failure warning instead of being silently skipped.

backend/src/__tests__/AutoHealService.evaluate.test.ts

@@ -8,6 +8,7 @@ let DockerController: typeof import('../services/DockerController').default;
 let DockerEventManager: typeof import('../services/DockerEventManager').DockerEventManager;
 let LicenseService: typeof import('../services/LicenseService').LicenseService;
 let NotificationService: typeof import('../services/NotificationService').NotificationService;
+let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
 
 function makePolicy(
     db: import('../services/DatabaseService').DatabaseService,
@@ -46,6 +47,7 @@ beforeAll(async () => {
     ({ DockerEventManager } = await import('../services/DockerEventManager'));
     ({ LicenseService } = await import('../services/LicenseService'));
     ({ NotificationService } = await import('../services/NotificationService'));
+    ({ NodeRegistry } = await import('../services/NodeRegistry'));
 });
 
 beforeEach(() => {
@@ -77,9 +79,9 @@ describe('AutoHealService.evaluate', () => {
         const db = DatabaseService.getInstance();
         makePolicy(db, { proxy_entitled_until: Date.now() + 60_000 });
         vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
-        const getRunningContainers = vi.fn().mockResolvedValue([]);
+        const getAllContainers = vi.fn().mockResolvedValue([]);
         const getInstance = vi.spyOn(DockerController, 'getInstance').mockReturnValue({
-            getRunningContainers,
+            getAllContainers,
         } as unknown as ReturnType<typeof DockerController.getInstance>);
 
         await resetAutoHealSingleton().evaluate();
@@ -93,7 +95,7 @@ describe('AutoHealService.evaluate', () => {
         vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
         const restartContainer = vi.fn().mockResolvedValue(undefined);
         vi.spyOn(DockerController, 'getInstance').mockReturnValue({
-            getRunningContainers: vi.fn().mockResolvedValue([{
+            getAllContainers: vi.fn().mockResolvedValue([{
                 Id: 'container-1',
                 Names: ['/heal-stack-web-1'],
                 Labels: {
@@ -118,6 +120,190 @@ describe('AutoHealService.evaluate', () => {
         expect(restartContainer).toHaveBeenCalledWith('container-1');
         const history = db.getAutoHealHistory(policy.id!);
         expect(history[0]).toMatchObject({ action: 'restarted', success: 1 });
+        expect(history[0].reason).toContain('unhealthy');
+    });
+
+    it('heals a crashed container that stayed down past the threshold', async () => {
+        const db = DatabaseService.getInstance();
+        const policy = makePolicy(db);
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+        const restartContainer = vi.fn().mockResolvedValue(undefined);
+        vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+            getAllContainers: vi.fn().mockResolvedValue([{
+                Id: 'crash-1',
+                Names: ['/heal-stack-worker-1'],
+                Labels: {
+                    'com.docker.compose.project': 'heal-stack',
+                    'com.docker.compose.service': 'worker',
+                },
+                State: 'exited',
+                Status: 'Exited (1) 2 minutes ago',
+            }]),
+            restartContainer,
+        } as unknown as ReturnType<typeof DockerController.getInstance>);
+        vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
+            getContainerState: () => ({ id: 'crash-1', crashedAt: Date.now() - 2 * 60_000 }),
+        } as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
+
+        await resetAutoHealSingleton().evaluate();
+
+        expect(restartContainer).toHaveBeenCalledWith('crash-1');
+        const history = db.getAutoHealHistory(policy.id!);
+        expect(history[0]).toMatchObject({ action: 'restarted', success: 1 });
+        expect(history[0].reason).toContain('crashed');
+    });
+
+    it('heals a container in the "dead" state when it crashed', async () => {
+        const db = DatabaseService.getInstance();
+        makePolicy(db);
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+        const restartContainer = vi.fn().mockResolvedValue(undefined);
+        vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+            getAllContainers: vi.fn().mockResolvedValue([{
+                Id: 'dead-1',
+                Names: ['/heal-stack-worker-1'],
+                Labels: {
+                    'com.docker.compose.project': 'heal-stack',
+                    'com.docker.compose.service': 'worker',
+                },
+                State: 'dead',
+                Status: 'Dead',
+            }]),
+            restartContainer,
+        } as unknown as ReturnType<typeof DockerController.getInstance>);
+        vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
+            getContainerState: () => ({ id: 'dead-1', crashedAt: Date.now() - 2 * 60_000 }),
+        } as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
+
+        await resetAutoHealSingleton().evaluate();
+
+        expect(restartContainer).toHaveBeenCalledWith('dead-1');
+    });
+
+    it('does not heal a recovered container that still carries a stale crash signal', async () => {
+        // Container crashed earlier (crashedAt set) but is running again now; the
+        // live running state must win so we do not restart a healthy container.
+        const db = DatabaseService.getInstance();
+        makePolicy(db);
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+        const restartContainer = vi.fn().mockResolvedValue(undefined);
+        vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+            getAllContainers: vi.fn().mockResolvedValue([{
+                Id: 'recovered-1',
+                Names: ['/heal-stack-worker-1'],
+                Labels: {
+                    'com.docker.compose.project': 'heal-stack',
+                    'com.docker.compose.service': 'worker',
+                },
+                State: 'running',
+                Status: 'Up 30 seconds',
+            }]),
+            restartContainer,
+        } as unknown as ReturnType<typeof DockerController.getInstance>);
+        vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
+            getContainerState: () => ({ id: 'recovered-1', crashedAt: Date.now() - 5 * 60_000 }),
+        } as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
+
+        await resetAutoHealSingleton().evaluate();
+
+        expect(restartContainer).not.toHaveBeenCalled();
+    });
+
+    it('does not heal an exited container with no crash signal (operator stop / clean exit)', async () => {
+        const db = DatabaseService.getInstance();
+        makePolicy(db);
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+        const restartContainer = vi.fn().mockResolvedValue(undefined);
+        vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+            getAllContainers: vi.fn().mockResolvedValue([{
+                Id: 'stopped-1',
+                Names: ['/heal-stack-worker-1'],
+                Labels: {
+                    'com.docker.compose.project': 'heal-stack',
+                    'com.docker.compose.service': 'worker',
+                },
+                State: 'exited',
+                Status: 'Exited (0) 2 minutes ago',
+            }]),
+            restartContainer,
+        } as unknown as ReturnType<typeof DockerController.getInstance>);
+        vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
+            getContainerState: () => ({ id: 'stopped-1' }), // no crashedAt: not a crash
+        } as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
+
+        await resetAutoHealSingleton().evaluate();
+
+        expect(restartContainer).not.toHaveBeenCalled();
+    });
+
+    it('restarts a container at most once per pass when policies overlap', async () => {
+        const db = DatabaseService.getInstance();
+        makePolicy(db); // all services
+        makePolicy(db, { service_name: 'worker' }); // service-specific, same container
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+        const restartContainer = vi.fn().mockResolvedValue(undefined);
+        vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+            getAllContainers: vi.fn().mockResolvedValue([{
+                Id: 'dup-1',
+                Names: ['/heal-stack-worker-1'],
+                Labels: {
+                    'com.docker.compose.project': 'heal-stack',
+                    'com.docker.compose.service': 'worker',
+                },
+                State: 'running',
+                Status: 'Up 5 minutes (unhealthy)',
+            }]),
+            restartContainer,
+        } as unknown as ReturnType<typeof DockerController.getInstance>);
+        vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
+            getContainerState: () => undefined,
+        } as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
+
+        const service = resetAutoHealSingleton();
+        (service as unknown as { observedUnhealthySince: Map<string, number> }).observedUnhealthySince
+            .set('1:dup-1', Date.now() - 2 * 60_000);
+
+        await service.evaluate();
+
+        expect(restartContainer).toHaveBeenCalledTimes(1);
+    });
+
+    it('counts a failed restart toward cooldown and the hourly cap', async () => {
+        const db = DatabaseService.getInstance();
+        const policy = makePolicy(db);
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+        const restartContainer = vi.fn().mockRejectedValue(new Error('no such container'));
+        vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+            getAllContainers: vi.fn().mockResolvedValue([{
+                Id: 'fail-1',
+                Names: ['/heal-stack-web-1'],
+                Labels: {
+                    'com.docker.compose.project': 'heal-stack',
+                    'com.docker.compose.service': 'web',
+                },
+                State: 'running',
+                Status: 'Up 1 minute (unhealthy)',
+            }]),
+            restartContainer,
+        } as unknown as ReturnType<typeof DockerController.getInstance>);
+        vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
+            getContainerState: () => undefined,
+        } as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
+
+        const service = resetAutoHealSingleton();
+        (service as unknown as { observedUnhealthySince: Map<string, number> }).observedUnhealthySince
+            .set('1:fail-1', Date.now() - 2 * 60_000);
+
+        await service.evaluate();
+
+        expect(restartContainer).toHaveBeenCalledTimes(1);
+        const updated = db.getAutoHealPolicy(policy.id!);
+        expect(updated!.last_fired_at).toBeGreaterThan(0);
+        expect(updated!.consecutive_failures).toBe(1);
+        const ts = (service as unknown as { restartTimestamps: Map<string, number[]> }).restartTimestamps.get('1:fail-1');
+        expect(ts).toHaveLength(1);
+        const history = db.getAutoHealHistory(policy.id!);
+        expect(history[0]).toMatchObject({ action: 'failed', success: 0 });
     });
 
     it('records Docker unavailable history once per throttle window when listing containers fails', async () => {
@@ -125,7 +311,7 @@ describe('AutoHealService.evaluate', () => {
         const policy = makePolicy(db);
         vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
         vi.spyOn(DockerController, 'getInstance').mockReturnValue({
-            getRunningContainers: vi.fn().mockRejectedValue(new Error('permission denied')),
+            getAllContainers: vi.fn().mockRejectedValue(new Error('permission denied')),
         } as unknown as ReturnType<typeof DockerController.getInstance>);
 
         const service = resetAutoHealSingleton();
@@ -153,9 +339,9 @@ describe('AutoHealService.evaluate', () => {
         });
         makePolicy(db, { node_id: secondNodeId, stack_name: 'second-stack' });
         vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
-        const getRunningContainers = vi.fn().mockResolvedValue([]);
+        const getAllContainers = vi.fn().mockResolvedValue([]);
         const getInstance = vi.spyOn(DockerController, 'getInstance').mockReturnValue({
-            getRunningContainers,
+            getAllContainers,
         } as unknown as ReturnType<typeof DockerController.getInstance>);
 
         await resetAutoHealSingleton().evaluate();
@@ -176,7 +362,7 @@ describe('AutoHealService.evaluate', () => {
         });
         const policy = makePolicy(db, { node_id: secondNodeId, stack_name: 'second-rate-stack' });
         vi.spyOn(DockerController, 'getInstance').mockReturnValue({
-            getRunningContainers: vi.fn().mockResolvedValue([]),
+            getAllContainers: vi.fn().mockResolvedValue([]),
         } as unknown as ReturnType<typeof DockerController.getInstance>);
 
         const service = resetAutoHealSingleton();
@@ -195,7 +381,7 @@ describe('AutoHealService.evaluate', () => {
         const db = DatabaseService.getInstance();
         const policy = makePolicy(db, { stack_name: 'cleanup-stack' });
         vi.spyOn(DockerController, 'getInstance').mockReturnValue({
-            getRunningContainers: vi.fn().mockResolvedValue([]),
+            getAllContainers: vi.fn().mockResolvedValue([]),
         } as unknown as ReturnType<typeof DockerController.getInstance>);
 
         const service = resetAutoHealSingleton();
@@ -214,6 +400,120 @@ describe('AutoHealService.evaluate', () => {
         expect(internals.historyTimestamps.size).toBe(0);
     });
 
+    it('refreshes proxied remote leases from a paid controlling instance', async () => {
+        const db = DatabaseService.getInstance();
+        db.addNode({
+            name: 'lease-remote',
+            type: 'remote',
+            compose_dir: '',
+            is_default: false,
+            api_url: 'http://remote:1852',
+            api_token: 'tok',
+        });
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+        vi.spyOn(LicenseService.getInstance(), 'getProxyHeaders').mockReturnValue(
+            { tier: 'paid', variant: 'admiral' } as ReturnType<ReturnType<typeof LicenseService.getInstance>['getProxyHeaders']>,
+        );
+        vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
+            apiUrl: 'http://remote:1852',
+            apiToken: 'tok',
+        });
+        const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ ok: true } as Response);
+
+        const service = resetAutoHealSingleton();
+        await (service as unknown as { refreshRemoteLeases: () => Promise<void> }).refreshRemoteLeases();
+
+        expect(fetchSpy).toHaveBeenCalledTimes(1);
+        const [url, opts] = fetchSpy.mock.calls[0];
+        expect(String(url)).toContain('/api/auto-heal/policies');
+        expect((opts as RequestInit).headers).toMatchObject({ 'x-sencho-tier': 'paid' });
+    });
+
+    it('does not refresh remote leases when the controlling instance is not paid', async () => {
+        const db = DatabaseService.getInstance();
+        db.addNode({
+            name: 'lease-remote-community',
+            type: 'remote',
+            compose_dir: '',
+            is_default: false,
+            api_url: 'http://remote2:1852',
+            api_token: 'tok2',
+        });
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
+        const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ ok: true } as Response);
+
+        const service = resetAutoHealSingleton();
+        await (service as unknown as { refreshRemoteLeases: () => Promise<void> }).refreshRemoteLeases();
+
+        expect(fetchSpy).not.toHaveBeenCalled();
+    });
+
+    it('keeps refreshing other remotes when one node is unreachable', async () => {
+        const db = DatabaseService.getInstance();
+        db.addNode({
+            name: 'good-remote',
+            type: 'remote',
+            compose_dir: '',
+            is_default: false,
+            api_url: 'http://good-host:1852',
+            api_token: 'tok',
+        });
+        db.addNode({
+            name: 'unreachable-remote',
+            type: 'remote',
+            compose_dir: '',
+            is_default: false,
+            api_url: 'http://bad-host:1852',
+            api_token: 'tok',
+        });
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+        vi.spyOn(LicenseService.getInstance(), 'getProxyHeaders').mockReturnValue(
+            { tier: 'paid', variant: 'admiral' } as ReturnType<ReturnType<typeof LicenseService.getInstance>['getProxyHeaders']>,
+        );
+        const fetchSpy = vi.spyOn(global, 'fetch').mockImplementation(((input: unknown) => {
+            const url = String(input);
+            if (url.includes('bad-host')) return Promise.reject(new Error('ECONNREFUSED'));
+            return Promise.resolve({ ok: true } as Response);
+        }) as typeof fetch);
+
+        const service = resetAutoHealSingleton();
+        // One node rejecting must not throw or block the others.
+        await expect(
+            (service as unknown as { refreshRemoteLeases: () => Promise<void> }).refreshRemoteLeases(),
+        ).resolves.toBeUndefined();
+
+        const calledUrls = fetchSpy.mock.calls.map(c => String(c[0]));
+        expect(calledUrls.some(u => u.includes('good-host'))).toBe(true);
+        expect(calledUrls.some(u => u.includes('bad-host'))).toBe(true);
+    });
+
+    it('warns after repeated lease refreshes find no reachable proxy target', async () => {
+        const db = DatabaseService.getInstance();
+        db.addNode({
+            name: 'no-target-remote',
+            type: 'remote',
+            compose_dir: '',
+            is_default: false,
+            api_url: '',
+            api_token: '',
+        });
+        vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+        vi.spyOn(LicenseService.getInstance(), 'getProxyHeaders').mockReturnValue(
+            { tier: 'paid', variant: 'admiral' } as ReturnType<ReturnType<typeof LicenseService.getInstance>['getProxyHeaders']>,
+        );
+        vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue(null);
+        const fetchSpy = vi.spyOn(global, 'fetch');
+        const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+
+        const svc = resetAutoHealSingleton() as unknown as { refreshRemoteLeases: () => Promise<void> };
+        await svc.refreshRemoteLeases();
+        await svc.refreshRemoteLeases();
+        expect(warnSpy).not.toHaveBeenCalled();
+        await svc.refreshRemoteLeases(); // third consecutive failure crosses the threshold
+        expect(warnSpy).toHaveBeenCalled();
+        expect(fetchSpy).not.toHaveBeenCalled();
+    });
+
     it('does not create duplicate timers when start is called twice', () => {
         vi.useFakeTimers();
         try {
@@ -224,9 +524,14 @@ describe('AutoHealService.evaluate', () => {
             service.start();
             service.start();
 
+            // One deferred-first-tick timeout, regardless of how many start() calls.
             expect(setTimeoutSpy).toHaveBeenCalledTimes(1);
-            vi.advanceTimersByTime(10_000);
+            // The lease-refresh interval is created immediately; the eval interval
+            // only after the initial delay fires. Calling start() twice must not
+            // duplicate either.
             expect(setIntervalSpy).toHaveBeenCalledTimes(1);
+            vi.advanceTimersByTime(10_000);
+            expect(setIntervalSpy).toHaveBeenCalledTimes(2);
             service.stop();
         } finally {
             vi.useRealTimers();

backend/src/__tests__/AutoHealService.test.ts

@@ -3,7 +3,8 @@
  *
  * shouldHeal is private; accessed via type cast (service as any) to avoid
  * exposing it in production API surface. All tests are pure (no I/O, no
- * timers) - they exercise the decision function directly.
+ * timers) - they exercise the decision function directly against a normalized
+ * HealSignal.
  */
 import { describe, it, expect, beforeEach } from 'vitest';
 import { INTENTIONAL_KILL_WINDOW_MS } from '../services/ContainerLifecycleClassifier';
@@ -35,98 +36,108 @@ describe('AutoHealService.shouldHeal', () => {
         updated_at: Date.now(),
     };
 
-    const baseState = {
-        id: 'container123',
-        name: 'mystack-web-1',
-        stackName: 'mystack',
-        healthStatus: 'unhealthy' as const,
-        unhealthySince: Date.now() - 6 * 60_000, // 6 minutes ago (past 5 min threshold)
+    // Unhealthy past the 5-min threshold, no recent kill.
+    const baseSignal = {
+        reason: 'unhealthy' as const,
+        downSince: Date.now() - 6 * 60_000,
         lastKillAt: undefined,
     };
 
     it('returns heal:true when all conditions are met', () => {
-        const result = service.shouldHeal(baseState, basePolicy, 'container123', Date.now());
+        const result = service.shouldHeal(baseSignal, basePolicy, 'container123', Date.now());
         expect(result.heal).toBe(true);
+        expect(result.reason).toBe('unhealthy');
     });
 
-    it('returns heal:false when healthStatus is not unhealthy', () => {
+    it('returns heal:true with reason "crashed" for a crashed trigger', () => {
         const result = service.shouldHeal(
-            { ...baseState, healthStatus: 'healthy' },
+            { reason: 'crashed', downSince: Date.now() - 6 * 60_000, lastKillAt: undefined },
+            basePolicy,
+            'container123',
+            Date.now(),
+        );
+        expect(result.heal).toBe(true);
+        expect(result.reason).toBe('crashed');
+    });
+
+    it('returns heal:false when there is no heal-worthy reason', () => {
+        const result = service.shouldHeal(
+            { reason: undefined, downSince: undefined },
             basePolicy,
             'container123',
             Date.now(),
         );
         expect(result.heal).toBe(false);
+        expect(result.skipReason).toBe('not_unhealthy');
     });
 
-    it('returns heal:false when healthStatus is undefined', () => {
+    it('returns heal:false when downSince is undefined', () => {
         const result = service.shouldHeal(
-            { ...baseState, healthStatus: undefined },
-            basePolicy,
-            'container123',
-            Date.now(),
-        );
-        expect(result.heal).toBe(false);
-    });
-
-    it('returns heal:false when state is undefined', () => {
-        const result = service.shouldHeal(undefined, basePolicy, 'container123', Date.now());
-        expect(result.heal).toBe(false);
-    });
-
-    it('returns heal:false when unhealthySince is undefined', () => {
-        const result = service.shouldHeal(
-            { ...baseState, unhealthySince: undefined },
+            { ...baseSignal, downSince: undefined },
             basePolicy,
             'container123',
             Date.now(),
         );
         expect(result.heal).toBe(false);
+        expect(result.skipReason).toBe('not_unhealthy');
     });
 
     it('returns heal:false when duration threshold is not yet met', () => {
         // Only 2 minutes, threshold is 5
-        const state = { ...baseState, unhealthySince: Date.now() - 2 * 60_000 };
-        const result = service.shouldHeal(state, basePolicy, 'container123', Date.now());
+        const signal = { ...baseSignal, downSince: Date.now() - 2 * 60_000 };
+        const result = service.shouldHeal(signal, basePolicy, 'container123', Date.now());
         expect(result.heal).toBe(false);
         expect(result.skipReason).toBe('duration_not_met');
     });
 
     it('returns skipped_user_action when lastKillAt is within the window', () => {
         // 30s ago, well within the 60s INTENTIONAL_KILL_WINDOW_MS
-        const state = { ...baseState, lastKillAt: Date.now() - 30_000 };
-        const result = service.shouldHeal(state, basePolicy, 'container123', Date.now());
+        const signal = { ...baseSignal, lastKillAt: Date.now() - 30_000 };
+        const result = service.shouldHeal(signal, basePolicy, 'container123', Date.now());
+        expect(result.heal).toBe(false);
+        expect(result.skipReason).toBe('skipped_user_action');
+    });
+
+    it('suppresses a crashed trigger when lastKillAt is within the window', () => {
+        // An operator stop that Docker reports with a non-zero exit must not be
+        // resurrected: the kill-window is the backstop behind the crash classifier.
+        const signal = {
+            reason: 'crashed' as const,
+            downSince: Date.now() - 6 * 60_000,
+            lastKillAt: Date.now() - 30_000,
+        };
+        const result = service.shouldHeal(signal, basePolicy, 'container123', Date.now());
         expect(result.heal).toBe(false);
         expect(result.skipReason).toBe('skipped_user_action');
     });
 
     it('does not suppress when lastKillAt is outside the intentional kill window', () => {
-        const state = {
-            ...baseState,
+        const signal = {
+            ...baseSignal,
             lastKillAt: Date.now() - (INTENTIONAL_KILL_WINDOW_MS + 5_000),
         };
-        const result = service.shouldHeal(state, basePolicy, 'container123', Date.now());
+        const result = service.shouldHeal(signal, basePolicy, 'container123', Date.now());
         expect(result.heal).toBe(true);
     });
 
     it('returns skipped_cooldown when last_fired_at is within cooldown period', () => {
         // Fired 5 min ago, cooldown is 10 min
         const policy = { ...basePolicy, last_fired_at: Date.now() - 5 * 60_000, cooldown_mins: 10 };
-        const result = service.shouldHeal(baseState, policy, 'container123', Date.now());
+        const result = service.shouldHeal(baseSignal, policy, 'container123', Date.now());
         expect(result.heal).toBe(false);
         expect(result.skipReason).toBe('skipped_cooldown');
     });
 
     it('does not apply cooldown when last_fired_at is 0', () => {
         const policy = { ...basePolicy, last_fired_at: 0 };
-        const result = service.shouldHeal(baseState, policy, 'container123', Date.now());
+        const result = service.shouldHeal(baseSignal, policy, 'container123', Date.now());
         expect(result.heal).toBe(true);
     });
 
     it('does not apply cooldown when last_fired_at exceeds the cooldown window', () => {
         // Fired 15 min ago, cooldown is 10 min
         const policy = { ...basePolicy, last_fired_at: Date.now() - 15 * 60_000, cooldown_mins: 10 };
-        const result = service.shouldHeal(baseState, policy, 'container123', Date.now());
+        const result = service.shouldHeal(baseSignal, policy, 'container123', Date.now());
         expect(result.heal).toBe(true);
     });
 
@@ -135,7 +146,7 @@ describe('AutoHealService.shouldHeal', () => {
         // Pre-populate with 3 entries within the last hour (policy max is 3)
         const map = (service as any).restartTimestamps as Map<string, number[]>;
         map.set('container123', [now - 10_000, now - 20_000, now - 30_000]);
-        const result = service.shouldHeal(baseState, basePolicy, 'container123', now);
+        const result = service.shouldHeal(baseSignal, basePolicy, 'container123', now);
         expect(result.heal).toBe(false);
         expect(result.skipReason).toBe('skipped_rate_limit');
     });
@@ -145,7 +156,7 @@ describe('AutoHealService.shouldHeal', () => {
         const map = (service as any).restartTimestamps as Map<string, number[]>;
         // All entries are >1 hour old, so they fall outside the rate-limit window
         map.set('container123', [now - 70 * 60_000, now - 80 * 60_000, now - 90 * 60_000]);
-        const result = service.shouldHeal(baseState, basePolicy, 'container123', now);
+        const result = service.shouldHeal(baseSignal, basePolicy, 'container123', now);
         expect(result.heal).toBe(true);
     });
 
@@ -154,17 +165,7 @@ describe('AutoHealService.shouldHeal', () => {
         const map = (service as any).restartTimestamps as Map<string, number[]>;
         // 2 old (outside window) + 1 recent = 1 active restart; max is 3, so still allowed
         map.set('container123', [now - 70 * 60_000, now - 80 * 60_000, now - 5_000]);
-        const result = service.shouldHeal(baseState, basePolicy, 'container123', now);
+        const result = service.shouldHeal(baseSignal, basePolicy, 'container123', now);
         expect(result.heal).toBe(true);
     });
-
-    it('returns not_unhealthy as skipReason when container is healthy', () => {
-        const result = service.shouldHeal(
-            { ...baseState, healthStatus: 'healthy' },
-            basePolicy,
-            'container123',
-            Date.now(),
-        );
-        expect(result.skipReason).toBe('not_unhealthy');
-    });
 });

backend/src/__tests__/docker-event-service.test.ts

@@ -142,6 +142,57 @@ describe('DockerEventService - die classification', () => {
         );
     });
 
+    it('stamps a crash signal and clears it on a later clean exit', async () => {
+        service = new DockerEventService(1, 'local');
+        await service.start();
+
+        // Crash: non-zero exit, no prior kill.
+        stream.push({
+            Type: 'container',
+            Action: 'die',
+            Actor: { ID: 'c-clean', Attributes: { exitCode: '1', name: 'web' } },
+        });
+        await vi.advanceTimersByTimeAsync(600);
+        expect(service.getContainerState('c-clean')?.crashedAt).toBeTypeOf('number');
+
+        // A subsequent clean exit (code 0) must wipe the stale crash signal.
+        stream.push({
+            Type: 'container',
+            Action: 'die',
+            Actor: { ID: 'c-clean', Attributes: { exitCode: '0', name: 'web' } },
+        });
+        await vi.advanceTimersByTimeAsync(600);
+        expect(service.getContainerState('c-clean')?.crashedAt).toBeUndefined();
+    });
+
+    it('does not stamp a crash for a die that was superseded by a start', async () => {
+        service = new DockerEventService(1, 'local');
+        await service.start();
+
+        // Establish tracked state so the later start is recorded.
+        stream.push({
+            Type: 'container',
+            Action: 'health_status: healthy',
+            Actor: { ID: 'c-race', Attributes: { name: 'web' } },
+        });
+        // Crash, then restart strictly later but still within the 500ms grace
+        // window before the die is classified.
+        stream.push({
+            Type: 'container',
+            Action: 'die',
+            Actor: { ID: 'c-race', Attributes: { exitCode: '1', name: 'web' } },
+        });
+        await vi.advanceTimersByTimeAsync(100);
+        stream.push({
+            Type: 'container',
+            Action: 'start',
+            Actor: { ID: 'c-race', Attributes: { name: 'web' } },
+        });
+        await vi.advanceTimersByTimeAsync(600);
+
+        expect(service.getContainerState('c-race')?.crashedAt).toBeUndefined();
+    });
+
     it('keeps stack and container routing for non-self compose crashes', async () => {
         service = new DockerEventService(1, 'local');
         await service.start();