fix: harden auto-heal policies (#1042)

* fix: harden auto-heal policies

* fix: resolve auto-heal lint failure

backend/src/services/AutoHealService.ts

@@ -3,6 +3,7 @@ import { DatabaseService, AutoHealPolicy, AutoHealHistoryEntry } from './Databas
 import DockerController from './DockerController';
 import { DockerEventManager } from './DockerEventManager';
 import { ContainerHealthSnapshot } from './DockerEventService';
+import { LicenseService } from './LicenseService';
 import { NotificationService } from './NotificationService';
 
 // Dockerode listContainers shape (subset used here)
@@ -10,11 +11,14 @@ type ContainerInfo = {
     Id: string;
     Names?: string[];
     Labels?: Record<string, string>;
+    State?: string;
+    Status?: string;
 };
 
 const EVAL_INTERVAL_MS = 30_000;
 const INITIAL_DELAY_MS = 10_000;
 const RATE_LIMIT_WINDOW_MS = 60 * 60_000; // 1 hour
+const HISTORY_THROTTLE_MS = 5 * 60_000;
 
 export class AutoHealService {
     private static instance: AutoHealService;
@@ -22,6 +26,8 @@ export class AutoHealService {
     private initialTimer: NodeJS.Timeout | null = null;
     private isProcessing = false;
     private restartTimestamps = new Map<string, number[]>();
+    private observedUnhealthySince = new Map<string, number>();
+    private historyTimestamps = new Map<string, number>();
 
     private constructor() {}
 
@@ -33,6 +39,7 @@ export class AutoHealService {
     }
 
     start(): void {
+        if (this.initialTimer || this.intervalId) return;
         this.initialTimer = setTimeout(() => {
             void this.evaluate();
             this.intervalId = setInterval(() => void this.evaluate(), EVAL_INTERVAL_MS);
@@ -54,13 +61,18 @@ export class AutoHealService {
         if (this.isProcessing) return;
         this.isProcessing = true;
         try {
+            const localPaid = LicenseService.getInstance().getTier() === 'paid';
             const db = DatabaseService.getInstance();
-            const policies = db.getAutoHealPolicies().filter(p => p.enabled === 1);
-            if (policies.length === 0) return;
 
             // Evaluate only on local nodes (remote nodes self-monitor via their own instance)
             const nodes = db.getNodes().filter(n => n.type === 'local');
+            const now = Date.now();
             for (const node of nodes) {
+                const policies = db.getAutoHealPolicies(undefined, node.id).filter(p =>
+                    p.enabled === 1 && (localPaid || p.proxy_entitled_until > now)
+                );
+                this.pruneInactivePolicyHistory(node.id, policies);
+                if (policies.length === 0) continue;
                 await this.evaluateForNode(node.id, policies);
             }
         } catch (err) {
@@ -75,27 +87,52 @@ export class AutoHealService {
         try {
             containers = await DockerController.getInstance(nodeId).getRunningContainers();
         } catch (err) {
+            const now = Date.now();
+            const errorMsg = err instanceof Error ? err.message : String(err);
             console.error(
                 `[AutoHeal] failed to list containers on node ${nodeId}:`,
-                err instanceof Error ? err.message : err,
+                errorMsg,
             );
+            for (const policy of policies) {
+                if (policy.id === undefined) continue;
+                this.recordThrottledHistory(policy, {
+                    policy_id: policy.id!,
+                    stack_name: policy.stack_name,
+                    service_name: policy.service_name,
+                    container_name: `node-${nodeId}`,
+                    container_id: `node-${nodeId}`,
+                    action: 'docker_unavailable',
+                    reason: 'Skipped: Docker daemon is unavailable for this node.',
+                    success: 0,
+                    error: errorMsg,
+                    timestamp: now,
+                }, nodeId);
+            }
             return;
         }
 
-        const db = DatabaseService.getInstance();
         const eventSvc = DockerEventManager.getInstance().getService(nodeId);
         const now = Date.now();
 
         // Prune stale entries for containers no longer running on this node
         const liveIds = new Set(containers.map(c => c.Id));
-        for (const [cid, timestamps] of this.restartTimestamps.entries()) {
+        const liveKeys = new Set(containers.map(c => this.containerKey(nodeId, c.Id)));
+        for (const [key, timestamps] of this.restartTimestamps.entries()) {
+            if (!key.startsWith(`${nodeId}:`)) continue;
+            const containerId = key.slice(String(nodeId).length + 1);
             const recent = timestamps.filter(t => now - t < RATE_LIMIT_WINDOW_MS);
-            if (recent.length === 0 || !liveIds.has(cid)) {
-                this.restartTimestamps.delete(cid);
+            if (recent.length === 0 || !liveIds.has(containerId)) {
+                this.restartTimestamps.delete(key);
             } else {
-                this.restartTimestamps.set(cid, recent);
+                this.restartTimestamps.set(key, recent);
             }
         }
+        for (const key of this.observedUnhealthySince.keys()) {
+            if (key.startsWith(`${nodeId}:`) && !liveKeys.has(key)) {
+                this.observedUnhealthySince.delete(key);
+            }
+        }
+        this.pruneInactivePolicyHistory(nodeId, policies);
 
         for (const policy of policies) {
             if (policy.id === undefined) {
@@ -115,8 +152,8 @@ export class AutoHealService {
                 const containerName =
                     container.Names?.[0]?.replace(/^\//, '') ?? container.Id.slice(0, 12);
                 const serviceOverride = container.Labels?.['com.docker.compose.service'] ?? null;
-                const state = eventSvc?.getContainerState(container.Id);
-                const decision = this.shouldHeal(state, policy, container.Id, now);
+                const state = this.getEffectiveState(nodeId, container, eventSvc?.getContainerState(container.Id), now);
+                const decision = this.shouldHeal(state, policy, this.containerKey(nodeId, container.Id), now);
 
                 if (!decision.heal) {
                     if (
@@ -124,7 +161,7 @@ export class AutoHealService {
                         decision.skipReason !== 'not_unhealthy' &&
                         decision.skipReason !== 'duration_not_met'
                     ) {
-                        db.recordAutoHealHistory({
+                        this.recordThrottledHistory(policy, {
                             policy_id: policy.id!,
                             stack_name: policy.stack_name,
                             service_name: policy.service_name ?? serviceOverride,
@@ -135,7 +172,7 @@ export class AutoHealService {
                             success: 0,
                             error: null,
                             timestamp: now,
-                        });
+                        }, nodeId, container.Id);
                     }
                     continue;
                 }
@@ -151,6 +188,51 @@ export class AutoHealService {
         }
     }
 
+    private getEffectiveState(
+        nodeId: number,
+        container: ContainerInfo,
+        eventState: ContainerHealthSnapshot | undefined,
+        now: number,
+    ): ContainerHealthSnapshot | undefined {
+        const key = this.containerKey(nodeId, container.Id);
+        const statusText = `${container.State ?? ''} ${container.Status ?? ''}`.toLowerCase();
+        const dockerHealth = statusText.includes('unhealthy')
+            ? 'unhealthy'
+            : statusText.includes('healthy')
+                ? 'healthy'
+                : statusText.includes('starting')
+                    ? 'starting'
+                    : undefined;
+
+        if (dockerHealth === 'unhealthy') {
+            const unhealthySince = eventState?.healthStatus === 'unhealthy' && eventState.unhealthySince
+                ? eventState.unhealthySince
+                : this.observedUnhealthySince.get(key) ?? now;
+            this.observedUnhealthySince.set(key, unhealthySince);
+            return {
+                id: container.Id,
+                name: eventState?.name ?? container.Names?.[0]?.replace(/^\//, ''),
+                stackName: eventState?.stackName ?? container.Labels?.['com.docker.compose.project'],
+                healthStatus: 'unhealthy',
+                unhealthySince,
+                lastKillAt: eventState?.lastKillAt,
+            };
+        }
+
+        if (dockerHealth === 'healthy' || dockerHealth === 'starting') {
+            this.observedUnhealthySince.delete(key);
+            return {
+                id: container.Id,
+                name: eventState?.name ?? container.Names?.[0]?.replace(/^\//, ''),
+                stackName: eventState?.stackName ?? container.Labels?.['com.docker.compose.project'],
+                healthStatus: dockerHealth,
+                lastKillAt: eventState?.lastKillAt,
+            };
+        }
+
+        return eventState;
+    }
+
     private shouldHeal(
         state: ContainerHealthSnapshot | undefined,
         policy: AutoHealPolicy,
@@ -213,10 +295,11 @@ export class AutoHealService {
             db.resetConsecutiveFailures(policy.id!);
             db.updateAutoHealPolicy(policy.id!, { last_fired_at: now });
 
-            const timestamps = this.restartTimestamps.get(containerId) ?? [];
+            const restartKey = this.containerKey(nodeId, containerId);
+            const timestamps = this.restartTimestamps.get(restartKey) ?? [];
             timestamps.push(now);
             this.restartTimestamps.set(
-                containerId,
+                restartKey,
                 timestamps.filter(t => now - t < RATE_LIMIT_WINDOW_MS),
             );
 
@@ -232,7 +315,7 @@ export class AutoHealService {
                 timestamp: now,
                 username: 'system',
                 method: 'POST',
-                path: '/api/auto-heal/execute',
+                path: '/system/auto-heal',
                 status_code: 200,
                 node_id: nodeId,
                 ip_address: '127.0.0.1',
@@ -306,6 +389,34 @@ export class AutoHealService {
             .catch(e => console.error('[AutoHeal] notification dispatch failed:', e));
     }
 
+    private recordThrottledHistory(
+        policy: AutoHealPolicy,
+        entry: Omit<AutoHealHistoryEntry, 'id'>,
+        nodeId: number,
+        containerId = entry.container_id,
+    ): void {
+        if (policy.id === undefined) return;
+        const key = `${nodeId}:${policy.id}:${containerId}:${entry.action}`;
+        const lastRecorded = this.historyTimestamps.get(key) ?? 0;
+        if (entry.timestamp - lastRecorded < HISTORY_THROTTLE_MS) return;
+        this.historyTimestamps.set(key, entry.timestamp);
+        DatabaseService.getInstance().recordAutoHealHistory(entry);
+    }
+
+    private pruneInactivePolicyHistory(nodeId: number, policies: AutoHealPolicy[]): void {
+        const activePolicyIds = new Set(policies.map(p => p.id).filter((id): id is number => id !== undefined));
+        for (const key of this.historyTimestamps.keys()) {
+            const [keyNodeId, policyId] = key.split(':');
+            if (keyNodeId === String(nodeId) && !activePolicyIds.has(Number(policyId))) {
+                this.historyTimestamps.delete(key);
+            }
+        }
+    }
+
+    private containerKey(nodeId: number, containerId: string): string {
+        return `${nodeId}:${containerId}`;
+    }
+
     private skipReasonText(reason: string): string {
         switch (reason) {
             case 'skipped_user_action':

backend/src/services/DatabaseService.ts

@@ -31,6 +31,8 @@ export type NodeMode = 'proxy' | 'pilot_agent';
 
 export interface AutoHealPolicy {
     id?: number;
+    node_id: number;
+    proxy_entitled_until: number;
     stack_name: string;
     service_name: string | null;
     unhealthy_duration_mins: number;
@@ -51,7 +53,7 @@ export interface AutoHealHistoryEntry {
     service_name: string | null;
     container_name: string;
     container_id: string;
-    action: 'restarted' | 'skipped_user_action' | 'skipped_cooldown' | 'skipped_rate_limit' | 'failed' | 'policy_auto_disabled';
+    action: 'restarted' | 'skipped_user_action' | 'skipped_cooldown' | 'skipped_rate_limit' | 'failed' | 'policy_auto_disabled' | 'docker_unavailable';
     reason: string;
     success: number;
     error: string | null;
@@ -650,6 +652,7 @@ export class DatabaseService {
         this.migrateAddNodeLastContact();
         this.migrateAddNodeCordonFields();
         this.migrateAddBlueprintPinnedNode();
+        this.migrateAutoHealNodeId();
 
         // Reset the cache once at end of constructor in case any migration
         // populated it via getGlobalSettings() and a subsequent migration
@@ -1074,6 +1077,8 @@ export class DatabaseService {
 
       CREATE TABLE IF NOT EXISTS auto_heal_policies (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
+        node_id INTEGER NOT NULL DEFAULT 1,
+        proxy_entitled_until INTEGER NOT NULL DEFAULT 0,
         stack_name TEXT NOT NULL,
         service_name TEXT,
         unhealthy_duration_mins INTEGER NOT NULL,
@@ -1372,11 +1377,14 @@ export class DatabaseService {
         this.db.prepare('CREATE INDEX IF NOT EXISTS idx_notification_routes_node_priority ON notification_routes(node_id, enabled, priority)').run();
     }
 
-    private tryAddColumn(table: string, col: string, def: string): void {
+    private tryAddColumn(table: string, col: string, def: string): boolean {
         try {
             this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run();
-        } catch {
-            /* column already present */
+            return true;
+        } catch (err) {
+            const message = err instanceof Error ? err.message.toLowerCase() : String(err).toLowerCase();
+            if (!message.includes('duplicate column name')) throw err;
+            return false;
         }
     }
 
@@ -1552,6 +1560,24 @@ export class DatabaseService {
         this.tryAddColumn('blueprints', 'pinned_node_id', 'INTEGER');
     }
 
+    private migrateAutoHealNodeId(): void {
+        const markerKey = 'migration_auto_heal_node_scope_v1';
+        const markerDone = this.getGlobalSettings()[markerKey] === '1';
+        this.tryAddColumn('auto_heal_policies', 'node_id', 'INTEGER NOT NULL DEFAULT 1');
+        this.tryAddColumn('auto_heal_policies', 'proxy_entitled_until', 'INTEGER NOT NULL DEFAULT 0');
+        if (!markerDone) {
+            const defaultNode = this.getDefaultNode();
+            if (defaultNode?.id) {
+                this.db.transaction(() => {
+                    this.db.prepare('UPDATE auto_heal_policies SET node_id = ? WHERE node_id IS NULL OR node_id = 1').run(defaultNode.id);
+                    this.updateGlobalSetting(markerKey, '1');
+                })();
+            } else {
+                this.updateGlobalSetting(markerKey, '1');
+            }
+        }
+    }
+
     // --- Sencho Mesh ---
 
     public listMeshStacks(nodeId?: number): Array<{ id: number; node_id: number; stack_name: string; created_at: number; created_by: string | null }> {
@@ -1786,10 +1812,16 @@ export class DatabaseService {
 
     // --- Auto-Heal Policies ---
 
-    public getAutoHealPolicies(stackName?: string): AutoHealPolicy[] {
+    public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] {
+        if (stackName && nodeId !== undefined) {
+            return this.db.prepare('SELECT * FROM auto_heal_policies WHERE stack_name = ? AND node_id = ?').all(stackName, nodeId) as AutoHealPolicy[];
+        }
         if (stackName) {
             return this.db.prepare('SELECT * FROM auto_heal_policies WHERE stack_name = ?').all(stackName) as AutoHealPolicy[];
         }
+        if (nodeId !== undefined) {
+            return this.db.prepare('SELECT * FROM auto_heal_policies WHERE node_id = ?').all(nodeId) as AutoHealPolicy[];
+        }
         return this.db.prepare('SELECT * FROM auto_heal_policies').all() as AutoHealPolicy[];
     }
 
@@ -1799,9 +1831,11 @@ export class DatabaseService {
 
     public addAutoHealPolicy(policy: Omit<AutoHealPolicy, 'id'>): AutoHealPolicy {
         const stmt = this.db.prepare(
-            'INSERT INTO auto_heal_policies (stack_name, service_name, unhealthy_duration_mins, cooldown_mins, max_restarts_per_hour, auto_disable_after_failures, enabled, consecutive_failures, last_fired_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
+            'INSERT INTO auto_heal_policies (node_id, proxy_entitled_until, stack_name, service_name, unhealthy_duration_mins, cooldown_mins, max_restarts_per_hour, auto_disable_after_failures, enabled, consecutive_failures, last_fired_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
         );
         const result = stmt.run(
+            policy.node_id,
+            policy.proxy_entitled_until,
             policy.stack_name,
             policy.service_name ?? null,
             policy.unhealthy_duration_mins,
@@ -1821,7 +1855,7 @@ export class DatabaseService {
         const ALLOWED_KEYS = new Set([
             'service_name', 'unhealthy_duration_mins', 'cooldown_mins',
             'max_restarts_per_hour', 'auto_disable_after_failures',
-            'enabled', 'consecutive_failures', 'last_fired_at',
+            'enabled', 'consecutive_failures', 'last_fired_at', 'proxy_entitled_until',
         ]);
         const entries = Object.entries(patch).filter(([k, v]) => ALLOWED_KEYS.has(k) && v !== undefined);
         if (entries.length === 0) return;
@@ -1852,6 +1886,7 @@ export class DatabaseService {
             entry.error ?? null,
             entry.timestamp
         );
+        this.pruneAutoHealHistory(entry.policy_id);
     }
 
     public getAutoHealHistory(policyId: number, limit = 50): AutoHealHistoryEntry[] {
@@ -1860,6 +1895,21 @@ export class DatabaseService {
         ).all(policyId, limit) as AutoHealHistoryEntry[];
     }
 
+    public pruneAutoHealHistory(policyId: number, maxRows = 500, maxAgeMs = 30 * 24 * 60 * 60_000): void {
+        const cutoff = Date.now() - maxAgeMs;
+        this.db.prepare('DELETE FROM auto_heal_history WHERE policy_id = ? AND timestamp < ?').run(policyId, cutoff);
+        this.db.prepare(`
+            DELETE FROM auto_heal_history
+            WHERE policy_id = ?
+              AND id NOT IN (
+                SELECT id FROM auto_heal_history
+                WHERE policy_id = ?
+                ORDER BY timestamp DESC, id DESC
+                LIMIT ?
+              )
+        `).run(policyId, policyId, maxRows);
+    }
+
     public incrementConsecutiveFailures(policyId: number): void {
         this.db.prepare('UPDATE auto_heal_policies SET consecutive_failures = consecutive_failures + 1, updated_at = ? WHERE id = ?').run(Date.now(), policyId);
     }