diff --git a/backend/src/__tests__/AutoHealService.evaluate.test.ts b/backend/src/__tests__/AutoHealService.evaluate.test.ts new file mode 100644 index 00000000..e0a72b07 --- /dev/null +++ b/backend/src/__tests__/AutoHealService.evaluate.test.ts @@ -0,0 +1,235 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let AutoHealService: typeof import('../services/AutoHealService').AutoHealService; +let DockerController: typeof import('../services/DockerController').default; +let DockerEventManager: typeof import('../services/DockerEventManager').DockerEventManager; +let LicenseService: typeof import('../services/LicenseService').LicenseService; +let NotificationService: typeof import('../services/NotificationService').NotificationService; + +function makePolicy( + db: import('../services/DatabaseService').DatabaseService, + overrides: Partial = {}, +) { + const nodeId = db.getDefaultNode()?.id ?? 1; + const now = Date.now(); + return db.addAutoHealPolicy({ + node_id: nodeId, + proxy_entitled_until: 0, + stack_name: 'heal-stack', + service_name: null, + unhealthy_duration_mins: 1, + cooldown_mins: 5, + max_restarts_per_hour: 3, + auto_disable_after_failures: 2, + enabled: 1, + consecutive_failures: 0, + last_fired_at: 0, + created_at: now, + updated_at: now, + ...overrides, + }); +} + +function resetAutoHealSingleton() { + (AutoHealService as unknown as { instance?: unknown }).instance = undefined; + return AutoHealService.getInstance(); +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ AutoHealService } = await import('../services/AutoHealService')); + ({ default: DockerController } = await import('../services/DockerController')); + ({ DockerEventManager } = await import('../services/DockerEventManager')); + ({ LicenseService } = await import('../services/LicenseService')); + ({ NotificationService } = await import('../services/NotificationService')); +}); + +beforeEach(() => { + vi.restoreAllMocks(); + const db = DatabaseService.getInstance(); + db.getDb().prepare('DELETE FROM auto_heal_history').run(); + db.getDb().prepare('DELETE FROM auto_heal_policies').run(); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(); +}); + +afterAll(() => { + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +describe('AutoHealService.evaluate', () => { + it('does not evaluate existing policies on Community tier', async () => { + const db = DatabaseService.getInstance(); + makePolicy(db); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + const dockerSpy = vi.spyOn(DockerController, 'getInstance'); + + await resetAutoHealSingleton().evaluate(); + + expect(dockerSpy).not.toHaveBeenCalled(); + }); + + it('evaluates trusted proxy-entitled policies on a Community runtime node', async () => { + const db = DatabaseService.getInstance(); + makePolicy(db, { proxy_entitled_until: Date.now() + 60_000 }); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + const getRunningContainers = vi.fn().mockResolvedValue([]); + const getInstance = vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getRunningContainers, + } as unknown as ReturnType); + + await resetAutoHealSingleton().evaluate(); + + expect(getInstance).toHaveBeenCalled(); + }); + + it('heals a currently unhealthy container after the observed threshold elapses without event state', async () => { + const db = DatabaseService.getInstance(); + const policy = makePolicy(db); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const restartContainer = vi.fn().mockResolvedValue(undefined); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getRunningContainers: vi.fn().mockResolvedValue([{ + Id: 'container-1', + Names: ['/heal-stack-web-1'], + Labels: { + 'com.docker.compose.project': 'heal-stack', + 'com.docker.compose.service': 'web', + }, + State: 'running', + Status: 'Up 5 minutes (unhealthy)', + }]), + restartContainer, + } as unknown as ReturnType); + vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({ + getContainerState: () => undefined, + } as unknown as ReturnType['getService']>); + + const service = resetAutoHealSingleton(); + (service as unknown as { observedUnhealthySince: Map }).observedUnhealthySince + .set(`1:container-1`, Date.now() - 2 * 60_000); + + await service.evaluate(); + + expect(restartContainer).toHaveBeenCalledWith('container-1'); + const history = db.getAutoHealHistory(policy.id!); + expect(history[0]).toMatchObject({ action: 'restarted', success: 1 }); + }); + + it('records Docker unavailable history once per throttle window when listing containers fails', async () => { + const db = DatabaseService.getInstance(); + const policy = makePolicy(db); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getRunningContainers: vi.fn().mockRejectedValue(new Error('permission denied')), + } as unknown as ReturnType); + + const service = resetAutoHealSingleton(); + await service.evaluate(); + await service.evaluate(); + + const history = db.getAutoHealHistory(policy.id!); + expect(history).toHaveLength(1); + expect(history[0]).toMatchObject({ + action: 'docker_unavailable', + success: 0, + error: 'permission denied', + }); + }); + + it('evaluates only policies scoped to each local node', async () => { + const db = DatabaseService.getInstance(); + const secondNodeId = db.addNode({ + name: 'second-local', + type: 'local', + compose_dir: process.env.COMPOSE_DIR ?? '', + is_default: false, + api_url: '', + api_token: '', + }); + makePolicy(db, { node_id: secondNodeId, stack_name: 'second-stack' }); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const getRunningContainers = vi.fn().mockResolvedValue([]); + const getInstance = vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getRunningContainers, + } as unknown as ReturnType); + + await resetAutoHealSingleton().evaluate(); + + expect(getInstance).toHaveBeenCalledTimes(1); + expect(getInstance).toHaveBeenCalledWith(secondNodeId); + }); + + it('keeps restart rate-limit state isolated by node while pruning', async () => { + const db = DatabaseService.getInstance(); + const secondNodeId = db.addNode({ + name: 'rate-limit-second-local', + type: 'local', + compose_dir: process.env.COMPOSE_DIR ?? '', + is_default: false, + api_url: '', + api_token: '', + }); + const policy = makePolicy(db, { node_id: secondNodeId, stack_name: 'second-rate-stack' }); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getRunningContainers: vi.fn().mockResolvedValue([]), + } as unknown as ReturnType); + + const service = resetAutoHealSingleton(); + const restartTimestamps = (service as unknown as { restartTimestamps: Map }).restartTimestamps; + const nodeOneKey = '1:node-one-container'; + restartTimestamps.set(nodeOneKey, [Date.now()]); + + await (service as unknown as { + evaluateForNode: (nodeId: number, policies: import('../services/DatabaseService').AutoHealPolicy[]) => Promise; + }).evaluateForNode(secondNodeId, [policy]); + + expect(restartTimestamps.has(nodeOneKey)).toBe(true); + }); + + it('prunes observed health and history throttle entries for removed containers and deleted policies', async () => { + const db = DatabaseService.getInstance(); + const policy = makePolicy(db, { stack_name: 'cleanup-stack' }); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getRunningContainers: vi.fn().mockResolvedValue([]), + } as unknown as ReturnType); + + const service = resetAutoHealSingleton(); + const internals = service as unknown as { + observedUnhealthySince: Map; + historyTimestamps: Map; + evaluateForNode: (nodeId: number, policies: import('../services/DatabaseService').AutoHealPolicy[]) => Promise; + }; + internals.observedUnhealthySince.set('1:removed-container', Date.now()); + internals.historyTimestamps.set(`1:${policy.id}:removed-container:skipped_cooldown`, Date.now()); + db.deleteAutoHealPolicy(policy.id!); + + await internals.evaluateForNode(1, []); + + expect(internals.observedUnhealthySince.has('1:removed-container')).toBe(false); + expect(internals.historyTimestamps.size).toBe(0); + }); + + it('does not create duplicate timers when start is called twice', () => { + vi.useFakeTimers(); + try { + const service = resetAutoHealSingleton(); + const setTimeoutSpy = vi.spyOn(global, 'setTimeout'); + const setIntervalSpy = vi.spyOn(global, 'setInterval'); + + service.start(); + service.start(); + + expect(setTimeoutSpy).toHaveBeenCalledTimes(1); + vi.advanceTimersByTime(10_000); + expect(setIntervalSpy).toHaveBeenCalledTimes(1); + service.stop(); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/backend/src/__tests__/AutoHealService.test.ts b/backend/src/__tests__/AutoHealService.test.ts index 23a3ee90..cd555aef 100644 --- a/backend/src/__tests__/AutoHealService.test.ts +++ b/backend/src/__tests__/AutoHealService.test.ts @@ -20,6 +20,8 @@ describe('AutoHealService.shouldHeal', () => { const basePolicy = { id: 1, + node_id: 1, + proxy_entitled_until: 0, stack_name: 'mystack', service_name: null, unhealthy_duration_mins: 5, diff --git a/backend/src/__tests__/DatabaseService.autoHeal.test.ts b/backend/src/__tests__/DatabaseService.autoHeal.test.ts index eb2db88d..151f9c86 100644 --- a/backend/src/__tests__/DatabaseService.autoHeal.test.ts +++ b/backend/src/__tests__/DatabaseService.autoHeal.test.ts @@ -13,6 +13,8 @@ let db: any; const makePolicy = (overrides: Record = {}) => { const now = Date.now(); return { + node_id: 1, + proxy_entitled_until: 0, stack_name: 'teststack', service_name: null, unhealthy_duration_mins: 5, @@ -59,6 +61,8 @@ describe('DatabaseService - auto-heal policy CRUD', () => { const fetched = db.getAutoHealPolicy(created.id); expect(fetched).toBeDefined(); expect(fetched.stack_name).toBe('roundtrip-stack'); + expect(fetched.node_id).toBe(1); + expect(fetched.proxy_entitled_until).toBe(0); expect(fetched.service_name).toBe('web'); expect(fetched.unhealthy_duration_mins).toBe(3); expect(fetched.cooldown_mins).toBe(15); @@ -94,6 +98,51 @@ describe('DatabaseService - auto-heal policy CRUD', () => { expect(yPolicies.every((p: any) => p.stack_name === 'filter-stack-y')).toBe(true); }); + it('getAutoHealPolicies with nodeId filter returns only matching rows', () => { + db.addAutoHealPolicy(makePolicy({ stack_name: 'node-filter-a', node_id: 1 })); + db.addAutoHealPolicy(makePolicy({ stack_name: 'node-filter-b', node_id: 2 })); + + const nodeOnePolicies = db.getAutoHealPolicies(undefined, 1); + const nodeTwoPolicies = db.getAutoHealPolicies(undefined, 2); + + expect(nodeOnePolicies.every((p: any) => p.node_id === 1)).toBe(true); + expect(nodeTwoPolicies.every((p: any) => p.node_id === 2)).toBe(true); + }); + + it('auto-heal node migration does not rewrite already-scoped node 1 policies', () => { + const created = db.addAutoHealPolicy(makePolicy({ stack_name: 'migration-node-one', node_id: 1 })); + db.addNode({ + name: 'new-default-node', + type: 'local', + compose_dir: process.env.COMPOSE_DIR ?? '', + is_default: true, + api_url: '', + api_token: '', + }); + + (db as any).migrateAutoHealNodeId(); + + expect(db.getAutoHealPolicy(created.id).node_id).toBe(1); + }); + + it('auto-heal node migration resumes backfill when the completion marker is missing', () => { + db.updateGlobalSetting('migration_auto_heal_node_scope_v1', ''); + const created = db.addAutoHealPolicy(makePolicy({ stack_name: 'migration-partial', node_id: 1 })); + const newDefaultId = db.addNode({ + name: 'partial-new-default-node', + type: 'local', + compose_dir: process.env.COMPOSE_DIR ?? '', + is_default: true, + api_url: '', + api_token: '', + }); + + (db as any).migrateAutoHealNodeId(); + + expect(db.getAutoHealPolicy(created.id).node_id).toBe(newDefaultId); + expect(db.getGlobalSettings().migration_auto_heal_node_scope_v1).toBe('1'); + }); + it('updateAutoHealPolicy partial update changes only specified fields', () => { const created = db.addAutoHealPolicy(makePolicy({ stack_name: 'update-stack', cooldown_mins: 10 })); @@ -212,6 +261,32 @@ describe('DatabaseService - auto-heal history', () => { expect(limited[1].reason).toBe('entry-8'); expect(limited[2].reason).toBe('entry-7'); }); + + it('recordAutoHealHistory prunes old rows beyond the retained window', () => { + const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'history-prune-stack' })); + const policyId: number = policy.id; + const baseTs = Date.now(); + + for (let i = 0; i < 505; i++) { + db.recordAutoHealHistory({ + policy_id: policyId, + stack_name: 'history-prune-stack', + service_name: null, + container_name: 'history-prune-stack-web-1', + container_id: 'hist-prune', + action: 'skipped_cooldown', + reason: `entry-${i}`, + success: 0, + error: null, + timestamp: baseTs + i, + }); + } + + const retained = db.getAutoHealHistory(policyId, 600); + expect(retained.length).toBe(500); + expect(retained[0].reason).toBe('entry-504'); + expect(retained[499].reason).toBe('entry-5'); + }); }); describe('DatabaseService - consecutive failure counters', () => { diff --git a/backend/src/__tests__/auto-heal-routes.test.ts b/backend/src/__tests__/auto-heal-routes.test.ts new file mode 100644 index 00000000..f164c2c5 --- /dev/null +++ b/backend/src/__tests__/auto-heal-routes.test.ts @@ -0,0 +1,233 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import crypto from 'crypto'; +import bcrypt from 'bcrypt'; +import { setupTestDb, cleanupTestDb, TEST_JWT_SECRET, TEST_USERNAME } from './helpers/setupTestDb'; +import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from '../services/license-headers'; + +let tmpDir: string; +let app: import('express').Express; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let LicenseService: typeof import('../services/LicenseService').LicenseService; + +function userToken(username: string): string { + const user = DatabaseService.getInstance().getUserByUsername(username); + if (!user) throw new Error(`missing test user ${username}`); + return jwt.sign({ username, role: user.role, tv: user.token_version }, TEST_JWT_SECRET, { expiresIn: '5m' }); +} + +function createApiToken(scope: 'read-only' | 'deploy-only' | 'full-admin'): string { + const rawToken = jwt.sign({ scope: 'api_token', jti: crypto.randomUUID() }, TEST_JWT_SECRET, { expiresIn: '5m' }); + const tokenHash = crypto.createHash('sha256').update(rawToken).digest('hex'); + const admin = DatabaseService.getInstance().getUserByUsername(TEST_USERNAME); + if (!admin?.id) throw new Error('missing seeded admin'); + DatabaseService.getInstance().addApiToken({ + token_hash: tokenHash, + name: `auto-heal-${scope}-${Date.now()}`, + scope, + user_id: admin.id, + created_at: Date.now(), + expires_at: null, + }); + return rawToken; +} + +function makePolicy(nodeId: number, stackName = 'route-stack') { + const now = Date.now(); + return DatabaseService.getInstance().addAutoHealPolicy({ + node_id: nodeId, + proxy_entitled_until: 0, + stack_name: stackName, + service_name: null, + unhealthy_duration_mins: 5, + cooldown_mins: 5, + max_restarts_per_hour: 3, + auto_disable_after_failures: 5, + enabled: 1, + consecutive_failures: 0, + last_fired_at: 0, + created_at: now, + updated_at: now, + }); +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ LicenseService } = await import('../services/LicenseService')); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral'); + + const viewerHash = await bcrypt.hash('password123', 1); + DatabaseService.getInstance().addUser({ username: 'route-viewer', password_hash: viewerHash, role: 'viewer' }); + + ({ app } = await import('../index')); +}); + +beforeEach(() => { + DatabaseService.getInstance().getDb().prepare('DELETE FROM auto_heal_history').run(); + DatabaseService.getInstance().getDb().prepare('DELETE FROM auto_heal_policies').run(); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral'); +}); + +afterAll(() => { + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +describe('/api/auto-heal routes', () => { + it('rejects Community tier access', async () => { + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + + const res = await request(app) + .get('/api/auto-heal/policies') + .set('Authorization', `Bearer ${userToken(TEST_USERNAME)}`); + + expect(res.status).toBe(403); + expect(res.body.code).toBe('PAID_REQUIRED'); + }); + + it('marks trusted proxy-created policies with a lease on a Community runtime node', async () => { + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + const proxyToken = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '5m' }); + + const res = await request(app) + .post('/api/auto-heal/policies') + .set('Authorization', `Bearer ${proxyToken}`) + .set(PROXY_TIER_HEADER, 'paid') + .set(PROXY_VARIANT_HEADER, 'admiral') + .send({ + stack_name: 'proxy-runtime-stack', + unhealthy_duration_mins: 5, + }); + + expect(res.status).toBe(201); + expect(res.body.proxy_entitled_until).toBeGreaterThan(Date.now()); + }); + + it('allows admins to create node-scoped policies on paid tier', async () => { + const res = await request(app) + .post('/api/auto-heal/policies') + .set('Authorization', `Bearer ${userToken(TEST_USERNAME)}`) + .send({ + stack_name: 'route-stack', + service_name: null, + unhealthy_duration_mins: 5, + }); + + expect(res.status).toBe(201); + expect(res.body.node_id).toBe(DatabaseService.getInstance().getDefaultNode()?.id); + expect(res.body.proxy_entitled_until).toBe(0); + }); + + it('rejects non-admin policy mutation', async () => { + const res = await request(app) + .post('/api/auto-heal/policies') + .set('Authorization', `Bearer ${userToken('route-viewer')}`) + .send({ + stack_name: 'route-stack', + unhealthy_duration_mins: 5, + }); + + expect(res.status).toBe(403); + expect(res.body.code).toBe('ADMIN_REQUIRED'); + }); + + it('lists only policies for the active node', async () => { + const defaultNodeId = DatabaseService.getInstance().getDefaultNode()?.id ?? 1; + const secondNodeId = DatabaseService.getInstance().addNode({ + name: 'route-second-local', + type: 'local', + compose_dir: process.env.COMPOSE_DIR ?? '', + is_default: false, + api_url: '', + api_token: '', + }); + makePolicy(defaultNodeId, 'same-stack'); + makePolicy(secondNodeId, 'same-stack'); + + const defaultRes = await request(app) + .get('/api/auto-heal/policies?stackName=same-stack') + .set('Authorization', `Bearer ${userToken(TEST_USERNAME)}`); + const secondRes = await request(app) + .get('/api/auto-heal/policies?stackName=same-stack') + .set('Authorization', `Bearer ${userToken(TEST_USERNAME)}`) + .set('x-node-id', String(secondNodeId)); + + expect(defaultRes.status).toBe(200); + expect(defaultRes.body).toHaveLength(1); + expect(defaultRes.body[0].node_id).toBe(defaultNodeId); + expect(secondRes.status).toBe(200); + expect(secondRes.body).toHaveLength(1); + expect(secondRes.body[0].node_id).toBe(secondNodeId); + }); + + it('rejects history access for a policy owned by a different node', async () => { + const secondNodeId = DatabaseService.getInstance().addNode({ + name: 'history-second-local', + type: 'local', + compose_dir: process.env.COMPOSE_DIR ?? '', + is_default: false, + api_url: '', + api_token: '', + }); + const policy = makePolicy(secondNodeId, 'history-stack'); + + const res = await request(app) + .get(`/api/auto-heal/policies/${policy.id}/history`) + .set('Authorization', `Bearer ${userToken(TEST_USERNAME)}`); + + expect(res.status).toBe(404); + }); + + it('persists enabled toggles through the patch route', async () => { + const defaultNodeId = DatabaseService.getInstance().getDefaultNode()?.id ?? 1; + const policy = makePolicy(defaultNodeId, 'toggle-stack'); + + const res = await request(app) + .patch(`/api/auto-heal/policies/${policy.id}`) + .set('Authorization', `Bearer ${userToken(TEST_USERNAME)}`) + .send({ enabled: 0 }); + + expect(res.status).toBe(200); + expect(res.body.enabled).toBe(0); + expect(DatabaseService.getInstance().getAutoHealPolicy(policy.id!)?.enabled).toBe(0); + }); + + it('allows read-only API tokens to list but not mutate policies', async () => { + const token = createApiToken('read-only'); + + const getRes = await request(app) + .get('/api/auto-heal/policies') + .set('Authorization', `Bearer ${token}`); + const postRes = await request(app) + .post('/api/auto-heal/policies') + .set('Authorization', `Bearer ${token}`) + .send({ stack_name: 'api-token-stack', unhealthy_duration_mins: 5 }); + + expect(getRes.status).toBe(200); + expect(postRes.status).toBe(403); + expect(postRes.body.code).toBe('SCOPE_DENIED'); + }); + + it('returns 503 for disconnected remote auto-heal requests before local routes run', async () => { + const remoteNodeId = DatabaseService.getInstance().addNode({ + name: 'disconnected-remote', + type: 'remote', + compose_dir: '', + is_default: false, + api_url: '', + api_token: '', + }); + + const res = await request(app) + .get('/api/auto-heal/policies') + .set('Authorization', `Bearer ${userToken(TEST_USERNAME)}`) + .set('x-node-id', String(remoteNodeId)); + + expect(res.status).toBe(503); + expect(res.body.error).toContain('has no API URL or token'); + }); +}); diff --git a/backend/src/routes/autoHeal.ts b/backend/src/routes/autoHeal.ts index d86fcee2..a65abaa5 100644 --- a/backend/src/routes/autoHeal.ts +++ b/backend/src/routes/autoHeal.ts @@ -14,15 +14,36 @@ const AutoHealPolicyCreateSchema = z.object({ max_restarts_per_hour: z.coerce.number().int().min(1).max(60).default(3), auto_disable_after_failures: z.coerce.number().int().min(1).max(100).default(5), }); -const AutoHealPolicyUpdateSchema = AutoHealPolicyCreateSchema.partial().omit({ stack_name: true }); +const AutoHealPolicyUpdateSchema = AutoHealPolicyCreateSchema + .partial() + .omit({ stack_name: true }) + .extend({ + enabled: z.coerce.number().int().min(0).max(1).optional(), + }); export const autoHealRouter = Router(); +const PROXY_ENTITLEMENT_LEASE_MS = 5 * 60_000; + +function proxyEntitlementUntil(req: Request): number { + return req.proxyTier === 'paid' ? Date.now() + PROXY_ENTITLEMENT_LEASE_MS : 0; +} + autoHealRouter.get('/policies', authMiddleware, (req: Request, res: Response): void => { if (!requirePaid(req, res)) return; const stackName = typeof req.query.stackName === 'string' ? req.query.stackName : undefined; try { - res.json(DatabaseService.getInstance().getAutoHealPolicies(stackName)); + const db = DatabaseService.getInstance(); + const policies = db.getAutoHealPolicies(stackName, req.nodeId); + const leaseUntil = proxyEntitlementUntil(req); + if (leaseUntil > 0) { + for (const policy of policies) { + if (policy.id !== undefined) db.updateAutoHealPolicy(policy.id, { proxy_entitled_until: leaseUntil }); + } + res.json(db.getAutoHealPolicies(stackName, req.nodeId)); + return; + } + res.json(policies); } catch (err) { console.error('[AutoHeal] Failed to list policies:', getErrorMessage(err, 'unknown')); res.status(500).json({ error: 'Internal server error' }); @@ -41,6 +62,8 @@ autoHealRouter.post('/policies', authMiddleware, (req: Request, res: Response): const now = Date.now(); try { const policy = DatabaseService.getInstance().addAutoHealPolicy({ + node_id: req.nodeId, + proxy_entitled_until: proxyEntitlementUntil(req), stack_name, service_name: service_name ?? null, unhealthy_duration_mins, @@ -72,8 +95,9 @@ autoHealRouter.patch('/policies/:id', authMiddleware, (req: Request, res: Respon } try { const db = DatabaseService.getInstance(); - if (!db.getAutoHealPolicy(id)) { res.status(404).json({ error: 'Policy not found' }); return; } - db.updateAutoHealPolicy(id, parsed.data); + const policy = db.getAutoHealPolicy(id); + if (!policy || policy.node_id !== req.nodeId) { res.status(404).json({ error: 'Policy not found' }); return; } + db.updateAutoHealPolicy(id, { ...parsed.data, proxy_entitled_until: Math.max(policy.proxy_entitled_until, proxyEntitlementUntil(req)) }); res.json(db.getAutoHealPolicy(id)); } catch (err) { console.error('[AutoHeal] Failed to update policy:', getErrorMessage(err, 'unknown')); @@ -88,7 +112,8 @@ autoHealRouter.delete('/policies/:id', authMiddleware, (req: Request, res: Respo if (id === null) return; try { const db = DatabaseService.getInstance(); - if (!db.getAutoHealPolicy(id)) { res.status(404).json({ error: 'Policy not found' }); return; } + const policy = db.getAutoHealPolicy(id); + if (!policy || policy.node_id !== req.nodeId) { res.status(404).json({ error: 'Policy not found' }); return; } db.deleteAutoHealPolicy(id); res.json({ success: true }); } catch (err) { @@ -103,7 +128,10 @@ autoHealRouter.get('/policies/:id/history', authMiddleware, (req: Request, res: if (id === null) return; const limit = Math.min(parseInt(String(req.query.limit ?? '50'), 10) || 50, 100); try { - res.json(DatabaseService.getInstance().getAutoHealHistory(id, limit)); + const db = DatabaseService.getInstance(); + const policy = db.getAutoHealPolicy(id); + if (!policy || policy.node_id !== req.nodeId) { res.status(404).json({ error: 'Policy not found' }); return; } + res.json(db.getAutoHealHistory(id, limit)); } catch (err) { console.error('[AutoHeal] Failed to fetch history:', getErrorMessage(err, 'unknown')); res.status(500).json({ error: 'Internal server error' }); diff --git a/backend/src/routes/dashboard.ts b/backend/src/routes/dashboard.ts index 896aa5a3..74394326 100644 --- a/backend/src/routes/dashboard.ts +++ b/backend/src/routes/dashboard.ts @@ -65,7 +65,7 @@ export function buildLocalConfigurationStatus( const alertRules = db.getStackAlerts().length; const notifRoutes = db.getNotificationRoutes(); - const healPolicies = db.getAutoHealPolicies(); + const healPolicies = db.getAutoHealPolicies(undefined, nodeId); const autoUpdateMap = db.getStackAutoUpdateSettingsForNode(nodeId); const autoUpdateEnabled = Object.values(autoUpdateMap).filter(Boolean).length; const autoUpdateTotal = Object.keys(autoUpdateMap).length; diff --git a/backend/src/services/AutoHealService.ts b/backend/src/services/AutoHealService.ts index bf3b7a9f..934c8566 100644 --- a/backend/src/services/AutoHealService.ts +++ b/backend/src/services/AutoHealService.ts @@ -3,6 +3,7 @@ import { DatabaseService, AutoHealPolicy, AutoHealHistoryEntry } from './Databas import DockerController from './DockerController'; import { DockerEventManager } from './DockerEventManager'; import { ContainerHealthSnapshot } from './DockerEventService'; +import { LicenseService } from './LicenseService'; import { NotificationService } from './NotificationService'; // Dockerode listContainers shape (subset used here) @@ -10,11 +11,14 @@ type ContainerInfo = { Id: string; Names?: string[]; Labels?: Record; + State?: string; + Status?: string; }; const EVAL_INTERVAL_MS = 30_000; const INITIAL_DELAY_MS = 10_000; const RATE_LIMIT_WINDOW_MS = 60 * 60_000; // 1 hour +const HISTORY_THROTTLE_MS = 5 * 60_000; export class AutoHealService { private static instance: AutoHealService; @@ -22,6 +26,8 @@ export class AutoHealService { private initialTimer: NodeJS.Timeout | null = null; private isProcessing = false; private restartTimestamps = new Map(); + private observedUnhealthySince = new Map(); + private historyTimestamps = new Map(); private constructor() {} @@ -33,6 +39,7 @@ export class AutoHealService { } start(): void { + if (this.initialTimer || this.intervalId) return; this.initialTimer = setTimeout(() => { void this.evaluate(); this.intervalId = setInterval(() => void this.evaluate(), EVAL_INTERVAL_MS); @@ -54,13 +61,18 @@ export class AutoHealService { if (this.isProcessing) return; this.isProcessing = true; try { + const localPaid = LicenseService.getInstance().getTier() === 'paid'; const db = DatabaseService.getInstance(); - const policies = db.getAutoHealPolicies().filter(p => p.enabled === 1); - if (policies.length === 0) return; // Evaluate only on local nodes (remote nodes self-monitor via their own instance) const nodes = db.getNodes().filter(n => n.type === 'local'); + const now = Date.now(); for (const node of nodes) { + const policies = db.getAutoHealPolicies(undefined, node.id).filter(p => + p.enabled === 1 && (localPaid || p.proxy_entitled_until > now) + ); + this.pruneInactivePolicyHistory(node.id, policies); + if (policies.length === 0) continue; await this.evaluateForNode(node.id, policies); } } catch (err) { @@ -75,27 +87,52 @@ export class AutoHealService { try { containers = await DockerController.getInstance(nodeId).getRunningContainers(); } catch (err) { + const now = Date.now(); + const errorMsg = err instanceof Error ? err.message : String(err); console.error( `[AutoHeal] failed to list containers on node ${nodeId}:`, - err instanceof Error ? err.message : err, + errorMsg, ); + for (const policy of policies) { + if (policy.id === undefined) continue; + this.recordThrottledHistory(policy, { + policy_id: policy.id!, + stack_name: policy.stack_name, + service_name: policy.service_name, + container_name: `node-${nodeId}`, + container_id: `node-${nodeId}`, + action: 'docker_unavailable', + reason: 'Skipped: Docker daemon is unavailable for this node.', + success: 0, + error: errorMsg, + timestamp: now, + }, nodeId); + } return; } - const db = DatabaseService.getInstance(); const eventSvc = DockerEventManager.getInstance().getService(nodeId); const now = Date.now(); // Prune stale entries for containers no longer running on this node const liveIds = new Set(containers.map(c => c.Id)); - for (const [cid, timestamps] of this.restartTimestamps.entries()) { + const liveKeys = new Set(containers.map(c => this.containerKey(nodeId, c.Id))); + for (const [key, timestamps] of this.restartTimestamps.entries()) { + if (!key.startsWith(`${nodeId}:`)) continue; + const containerId = key.slice(String(nodeId).length + 1); const recent = timestamps.filter(t => now - t < RATE_LIMIT_WINDOW_MS); - if (recent.length === 0 || !liveIds.has(cid)) { - this.restartTimestamps.delete(cid); + if (recent.length === 0 || !liveIds.has(containerId)) { + this.restartTimestamps.delete(key); } else { - this.restartTimestamps.set(cid, recent); + this.restartTimestamps.set(key, recent); } } + for (const key of this.observedUnhealthySince.keys()) { + if (key.startsWith(`${nodeId}:`) && !liveKeys.has(key)) { + this.observedUnhealthySince.delete(key); + } + } + this.pruneInactivePolicyHistory(nodeId, policies); for (const policy of policies) { if (policy.id === undefined) { @@ -115,8 +152,8 @@ export class AutoHealService { const containerName = container.Names?.[0]?.replace(/^\//, '') ?? container.Id.slice(0, 12); const serviceOverride = container.Labels?.['com.docker.compose.service'] ?? null; - const state = eventSvc?.getContainerState(container.Id); - const decision = this.shouldHeal(state, policy, container.Id, now); + const state = this.getEffectiveState(nodeId, container, eventSvc?.getContainerState(container.Id), now); + const decision = this.shouldHeal(state, policy, this.containerKey(nodeId, container.Id), now); if (!decision.heal) { if ( @@ -124,7 +161,7 @@ export class AutoHealService { decision.skipReason !== 'not_unhealthy' && decision.skipReason !== 'duration_not_met' ) { - db.recordAutoHealHistory({ + this.recordThrottledHistory(policy, { policy_id: policy.id!, stack_name: policy.stack_name, service_name: policy.service_name ?? serviceOverride, @@ -135,7 +172,7 @@ export class AutoHealService { success: 0, error: null, timestamp: now, - }); + }, nodeId, container.Id); } continue; } @@ -151,6 +188,51 @@ export class AutoHealService { } } + private getEffectiveState( + nodeId: number, + container: ContainerInfo, + eventState: ContainerHealthSnapshot | undefined, + now: number, + ): ContainerHealthSnapshot | undefined { + const key = this.containerKey(nodeId, container.Id); + const statusText = `${container.State ?? ''} ${container.Status ?? ''}`.toLowerCase(); + const dockerHealth = statusText.includes('unhealthy') + ? 'unhealthy' + : statusText.includes('healthy') + ? 'healthy' + : statusText.includes('starting') + ? 'starting' + : undefined; + + if (dockerHealth === 'unhealthy') { + const unhealthySince = eventState?.healthStatus === 'unhealthy' && eventState.unhealthySince + ? eventState.unhealthySince + : this.observedUnhealthySince.get(key) ?? now; + this.observedUnhealthySince.set(key, unhealthySince); + return { + id: container.Id, + name: eventState?.name ?? container.Names?.[0]?.replace(/^\//, ''), + stackName: eventState?.stackName ?? container.Labels?.['com.docker.compose.project'], + healthStatus: 'unhealthy', + unhealthySince, + lastKillAt: eventState?.lastKillAt, + }; + } + + if (dockerHealth === 'healthy' || dockerHealth === 'starting') { + this.observedUnhealthySince.delete(key); + return { + id: container.Id, + name: eventState?.name ?? container.Names?.[0]?.replace(/^\//, ''), + stackName: eventState?.stackName ?? container.Labels?.['com.docker.compose.project'], + healthStatus: dockerHealth, + lastKillAt: eventState?.lastKillAt, + }; + } + + return eventState; + } + private shouldHeal( state: ContainerHealthSnapshot | undefined, policy: AutoHealPolicy, @@ -213,10 +295,11 @@ export class AutoHealService { db.resetConsecutiveFailures(policy.id!); db.updateAutoHealPolicy(policy.id!, { last_fired_at: now }); - const timestamps = this.restartTimestamps.get(containerId) ?? []; + const restartKey = this.containerKey(nodeId, containerId); + const timestamps = this.restartTimestamps.get(restartKey) ?? []; timestamps.push(now); this.restartTimestamps.set( - containerId, + restartKey, timestamps.filter(t => now - t < RATE_LIMIT_WINDOW_MS), ); @@ -232,7 +315,7 @@ export class AutoHealService { timestamp: now, username: 'system', method: 'POST', - path: '/api/auto-heal/execute', + path: '/system/auto-heal', status_code: 200, node_id: nodeId, ip_address: '127.0.0.1', @@ -306,6 +389,34 @@ export class AutoHealService { .catch(e => console.error('[AutoHeal] notification dispatch failed:', e)); } + private recordThrottledHistory( + policy: AutoHealPolicy, + entry: Omit, + nodeId: number, + containerId = entry.container_id, + ): void { + if (policy.id === undefined) return; + const key = `${nodeId}:${policy.id}:${containerId}:${entry.action}`; + const lastRecorded = this.historyTimestamps.get(key) ?? 0; + if (entry.timestamp - lastRecorded < HISTORY_THROTTLE_MS) return; + this.historyTimestamps.set(key, entry.timestamp); + DatabaseService.getInstance().recordAutoHealHistory(entry); + } + + private pruneInactivePolicyHistory(nodeId: number, policies: AutoHealPolicy[]): void { + const activePolicyIds = new Set(policies.map(p => p.id).filter((id): id is number => id !== undefined)); + for (const key of this.historyTimestamps.keys()) { + const [keyNodeId, policyId] = key.split(':'); + if (keyNodeId === String(nodeId) && !activePolicyIds.has(Number(policyId))) { + this.historyTimestamps.delete(key); + } + } + } + + private containerKey(nodeId: number, containerId: string): string { + return `${nodeId}:${containerId}`; + } + private skipReasonText(reason: string): string { switch (reason) { case 'skipped_user_action': diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index a1ab4878..6603c47d 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -31,6 +31,8 @@ export type NodeMode = 'proxy' | 'pilot_agent'; export interface AutoHealPolicy { id?: number; + node_id: number; + proxy_entitled_until: number; stack_name: string; service_name: string | null; unhealthy_duration_mins: number; @@ -51,7 +53,7 @@ export interface AutoHealHistoryEntry { service_name: string | null; container_name: string; container_id: string; - action: 'restarted' | 'skipped_user_action' | 'skipped_cooldown' | 'skipped_rate_limit' | 'failed' | 'policy_auto_disabled'; + action: 'restarted' | 'skipped_user_action' | 'skipped_cooldown' | 'skipped_rate_limit' | 'failed' | 'policy_auto_disabled' | 'docker_unavailable'; reason: string; success: number; error: string | null; @@ -650,6 +652,7 @@ export class DatabaseService { this.migrateAddNodeLastContact(); this.migrateAddNodeCordonFields(); this.migrateAddBlueprintPinnedNode(); + this.migrateAutoHealNodeId(); // Reset the cache once at end of constructor in case any migration // populated it via getGlobalSettings() and a subsequent migration @@ -1074,6 +1077,8 @@ export class DatabaseService { CREATE TABLE IF NOT EXISTS auto_heal_policies ( id INTEGER PRIMARY KEY AUTOINCREMENT, + node_id INTEGER NOT NULL DEFAULT 1, + proxy_entitled_until INTEGER NOT NULL DEFAULT 0, stack_name TEXT NOT NULL, service_name TEXT, unhealthy_duration_mins INTEGER NOT NULL, @@ -1372,11 +1377,14 @@ export class DatabaseService { this.db.prepare('CREATE INDEX IF NOT EXISTS idx_notification_routes_node_priority ON notification_routes(node_id, enabled, priority)').run(); } - private tryAddColumn(table: string, col: string, def: string): void { + private tryAddColumn(table: string, col: string, def: string): boolean { try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); - } catch { - /* column already present */ + return true; + } catch (err) { + const message = err instanceof Error ? err.message.toLowerCase() : String(err).toLowerCase(); + if (!message.includes('duplicate column name')) throw err; + return false; } } @@ -1552,6 +1560,24 @@ export class DatabaseService { this.tryAddColumn('blueprints', 'pinned_node_id', 'INTEGER'); } + private migrateAutoHealNodeId(): void { + const markerKey = 'migration_auto_heal_node_scope_v1'; + const markerDone = this.getGlobalSettings()[markerKey] === '1'; + this.tryAddColumn('auto_heal_policies', 'node_id', 'INTEGER NOT NULL DEFAULT 1'); + this.tryAddColumn('auto_heal_policies', 'proxy_entitled_until', 'INTEGER NOT NULL DEFAULT 0'); + if (!markerDone) { + const defaultNode = this.getDefaultNode(); + if (defaultNode?.id) { + this.db.transaction(() => { + this.db.prepare('UPDATE auto_heal_policies SET node_id = ? WHERE node_id IS NULL OR node_id = 1').run(defaultNode.id); + this.updateGlobalSetting(markerKey, '1'); + })(); + } else { + this.updateGlobalSetting(markerKey, '1'); + } + } + } + // --- Sencho Mesh --- public listMeshStacks(nodeId?: number): Array<{ id: number; node_id: number; stack_name: string; created_at: number; created_by: string | null }> { @@ -1786,10 +1812,16 @@ export class DatabaseService { // --- Auto-Heal Policies --- - public getAutoHealPolicies(stackName?: string): AutoHealPolicy[] { + public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] { + if (stackName && nodeId !== undefined) { + return this.db.prepare('SELECT * FROM auto_heal_policies WHERE stack_name = ? AND node_id = ?').all(stackName, nodeId) as AutoHealPolicy[]; + } if (stackName) { return this.db.prepare('SELECT * FROM auto_heal_policies WHERE stack_name = ?').all(stackName) as AutoHealPolicy[]; } + if (nodeId !== undefined) { + return this.db.prepare('SELECT * FROM auto_heal_policies WHERE node_id = ?').all(nodeId) as AutoHealPolicy[]; + } return this.db.prepare('SELECT * FROM auto_heal_policies').all() as AutoHealPolicy[]; } @@ -1799,9 +1831,11 @@ export class DatabaseService { public addAutoHealPolicy(policy: Omit): AutoHealPolicy { const stmt = this.db.prepare( - 'INSERT INTO auto_heal_policies (stack_name, service_name, unhealthy_duration_mins, cooldown_mins, max_restarts_per_hour, auto_disable_after_failures, enabled, consecutive_failures, last_fired_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' + 'INSERT INTO auto_heal_policies (node_id, proxy_entitled_until, stack_name, service_name, unhealthy_duration_mins, cooldown_mins, max_restarts_per_hour, auto_disable_after_failures, enabled, consecutive_failures, last_fired_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' ); const result = stmt.run( + policy.node_id, + policy.proxy_entitled_until, policy.stack_name, policy.service_name ?? null, policy.unhealthy_duration_mins, @@ -1821,7 +1855,7 @@ export class DatabaseService { const ALLOWED_KEYS = new Set([ 'service_name', 'unhealthy_duration_mins', 'cooldown_mins', 'max_restarts_per_hour', 'auto_disable_after_failures', - 'enabled', 'consecutive_failures', 'last_fired_at', + 'enabled', 'consecutive_failures', 'last_fired_at', 'proxy_entitled_until', ]); const entries = Object.entries(patch).filter(([k, v]) => ALLOWED_KEYS.has(k) && v !== undefined); if (entries.length === 0) return; @@ -1852,6 +1886,7 @@ export class DatabaseService { entry.error ?? null, entry.timestamp ); + this.pruneAutoHealHistory(entry.policy_id); } public getAutoHealHistory(policyId: number, limit = 50): AutoHealHistoryEntry[] { @@ -1860,6 +1895,21 @@ export class DatabaseService { ).all(policyId, limit) as AutoHealHistoryEntry[]; } + public pruneAutoHealHistory(policyId: number, maxRows = 500, maxAgeMs = 30 * 24 * 60 * 60_000): void { + const cutoff = Date.now() - maxAgeMs; + this.db.prepare('DELETE FROM auto_heal_history WHERE policy_id = ? AND timestamp < ?').run(policyId, cutoff); + this.db.prepare(` + DELETE FROM auto_heal_history + WHERE policy_id = ? + AND id NOT IN ( + SELECT id FROM auto_heal_history + WHERE policy_id = ? + ORDER BY timestamp DESC, id DESC + LIMIT ? + ) + `).run(policyId, policyId, maxRows); + } + public incrementConsecutiveFailures(policyId: number): void { this.db.prepare('UPDATE auto_heal_policies SET consecutive_failures = consecutive_failures + 1, updated_at = ? WHERE id = ?').run(Date.now(), policyId); } diff --git a/frontend/src/components/StackAlertSheet.tsx b/frontend/src/components/StackAlertSheet.tsx index c7fa5e8a..fb3581ca 100644 --- a/frontend/src/components/StackAlertSheet.tsx +++ b/frontend/src/components/StackAlertSheet.tsx @@ -35,6 +35,8 @@ interface StackAlert { interface AutoHealPolicy { id?: number; + node_id: number; + proxy_entitled_until: number; stack_name: string; service_name: string | null; unhealthy_duration_mins: number; @@ -55,7 +57,7 @@ interface AutoHealHistoryEntry { service_name: string | null; container_name: string; container_id: string; - action: 'restarted' | 'skipped_user_action' | 'skipped_cooldown' | 'skipped_rate_limit' | 'failed' | 'policy_auto_disabled'; + action: 'restarted' | 'skipped_user_action' | 'skipped_cooldown' | 'skipped_rate_limit' | 'failed' | 'policy_auto_disabled' | 'docker_unavailable'; reason: string; success: number; error: string | null; @@ -122,6 +124,7 @@ function actionLabel(action: AutoHealHistoryEntry['action']): string { case 'skipped_rate_limit': return 'Skipped (rate limit)'; case 'failed': return 'Failed'; case 'policy_auto_disabled': return 'Auto-disabled'; + case 'docker_unavailable': return 'Docker unavailable'; } }