import { randomUUID } from 'crypto'; import DockerController from './DockerController'; import { DatabaseService, type HealthGateRunRow } from './DatabaseService'; import { sanitizeForLog } from '../utils/safeLog'; import { getErrorMessage } from '../utils/errors'; import { withTimeout } from '../utils/withTimeout'; import type { HealthGateContainer, HealthGateReport } from './updateGuard/types'; const POLL_INTERVAL_MS = 5_000; // Per-observe ceiling so a hung Docker socket cannot leave a poll pending // forever. Above POLL_INTERVAL_MS so a slow-but-live probe is not cut short; a // timeout counts as a poll error and three in a row resolve the gate unknown. const OBSERVE_TIMEOUT_MS = 8_000; // A stack whose containers never appear gives up after this long. const EMPTY_GRACE_MS = 15_000; const DEFAULT_WINDOW_SECONDS = 90; const MIN_WINDOW_SECONDS = 15; const MAX_WINDOW_SECONDS = 600; // Backstop against runaway concurrency (a burst of webhook or scheduled // updates). Gates past the cap finalize immediately as unknown. const MAX_CONCURRENT_GATES = 25; interface ObservedContainer { id: string; name: string; startedAt: string | null; /** Docker's raw RestartCount at observation time. */ restartCount: number; /** Gate-maintained restart tally since the baseline; 0 on a fresh snapshot, * set by poll()'s accounting pass, at most one increment per poll. */ restarts: number; state: string; health: string | null; } interface ActiveGate { runId: string; nodeId: number; stackName: string; windowSeconds: number; startedAt: number; /** Self-scheduling poll timer, armed only between settled poll cycles. */ timer: ReturnType | null; /** Expected container set, keyed by name; null until the first non-empty poll. */ expected: Map | null; consecutivePollErrors: number; /** Names missing on the previous poll (two consecutive misses fail the gate). */ missingLastPoll: Set; /** Names in `restarting` state on the previous poll. */ restartingLastPoll: Set; /** * Set by finalize so a poll that was mid-await when this gate was superseded * or stopped can never overwrite the terminal verdict with a stale one. */ finalized: boolean; } /** * Post-update health gate: after a deploy/update succeeds, observe the * stack's containers for a configurable window and record a passed / failed / * unknown verdict plus an activity timeline event. Purely observational: it * never restarts, heals, or rolls anything back. AutoHeal needs no special * handling: the unhealthy or exited state that triggers it is seen by the * gate's own polls and fails the gate, and repeated restarts trip the * restart-loop check. * * begin() is the single shared post-success hook for every gated deploy and * update path; excluded paths (rollback, installs, reconciler loops) simply * never call it. */ export class HealthGateService { private static instance: HealthGateService; private readonly active = new Map(); private started = false; public static getInstance(): HealthGateService { if (!HealthGateService.instance) { HealthGateService.instance = new HealthGateService(); } return HealthGateService.instance; } /** Sweep runs left observing by a previous process, then accept begin() calls. */ public start(): void { this.started = true; try { const swept = DatabaseService.getInstance().markInterruptedHealthGateRuns( 'Sencho restarted during observation', Date.now(), ); if (swept > 0) { console.log(`[HealthGate] Marked ${swept} interrupted observation(s) as unknown`); } } catch (error) { console.error('[HealthGate] Startup sweep failed:', getErrorMessage(error, 'unknown')); } } /** Clear every poll timer and finalize in-flight gates as unknown. */ public stop(): void { this.started = false; for (const gate of [...this.active.values()]) { this.finalize(gate, 'unknown', 'shutdown during observation', []); } } /** * Begin observing a stack after a successful deploy/update. Returns the gate * run id for response correlation, or null when gating is disabled, the * service is not started, or recording fails internally. Inserts the row * synchronously so the caller can include the id in its response; * observation then runs on a timer. Never throws. * * Also records the `update_started` activity event for update triggers, so * every gated update path gets the timeline marker even when the gate * itself is disabled. */ public begin( nodeId: number, stackName: string, trigger: 'update' | 'deploy', actor: string | null, ): string | null { // Refuses work outside the start()/stop() lifecycle so a late call during // shutdown cannot leave a dangling poll timer. if (!this.started) return null; try { const db = DatabaseService.getInstance(); const settings = this.readSettings(); if (trigger === 'update') { this.recordActivity(nodeId, stackName, 'info', 'update_started', `${stackName} update started`, actor); } if (!settings.enabled) return null; // A newer operation supersedes an in-flight gate for the same stack. const key = `${nodeId}:${stackName}`; const existing = this.active.get(key); if (existing) { this.finalize(existing, 'unknown', 'superseded by a newer update', []); } const runId = randomUUID(); const startedAt = Date.now(); const row: HealthGateRunRow = { id: runId, node_id: nodeId, stack_name: stackName, trigger_action: trigger, status: 'observing', reason: null, window_seconds: settings.windowSeconds, containers_json: '[]', started_at: startedAt, ended_at: null, created_by: actor, }; if (this.active.size >= MAX_CONCURRENT_GATES) { db.insertHealthGateRun({ ...row, status: 'unknown', reason: 'too many concurrent observations', ended_at: startedAt }); return runId; } db.insertHealthGateRun(row); const gate: ActiveGate = { runId, nodeId, stackName, windowSeconds: settings.windowSeconds, startedAt, timer: null, expected: null, consecutivePollErrors: 0, missingLastPoll: new Set(), restartingLastPoll: new Set(), finalized: false, }; this.active.set(key, gate); this.scheduleNextPoll(gate); return runId; } catch (error) { // The gate is an observer; its failure must never fail the operation. console.error( '[HealthGate] begin (%s) failed for %s on node %d:', trigger, sanitizeForLog(stackName), nodeId, error, ); return null; } } /** A specific run by id, the latest run, or the never-run sentinel. */ public getReport(nodeId: number, stackName: string, gateId?: string): HealthGateReport { const db = DatabaseService.getInstance(); const row = gateId ? db.getHealthGateRun(nodeId, stackName, gateId) : db.getLatestHealthGateRun(nodeId, stackName); if (!row) { return { stack: stackName, id: null, status: 'never-run', trigger: null, reason: null, windowSeconds: null, startedAt: null, endedAt: null, containers: [], }; } let containers: HealthGateContainer[] = []; try { const parsed: unknown = JSON.parse(row.containers_json); if (Array.isArray(parsed)) containers = parsed as HealthGateContainer[]; } catch { // A corrupt blob only loses the per-container detail, never the verdict. console.warn('[HealthGate] Unreadable containers_json for run %s', sanitizeForLog(row.id)); } return { stack: stackName, id: row.id, status: row.status, trigger: row.trigger_action, reason: row.reason, windowSeconds: row.window_seconds, startedAt: row.started_at, endedAt: row.ended_at, containers, }; } /** * Orchestrate one poll cycle and arm the next. Polls are single-flight: the * next timer is scheduled only after this cycle fully settles (the finally * below), so a slow or timed-out observe can never overlap the following poll * or corrupt the restart/missing accounting that assumes one poll at a time. */ private async poll(gate: ActiveGate): Promise { const key = `${gate.nodeId}:${gate.stackName}`; // A late timer fire after supersede, stop, or finalize must do nothing. if (gate.finalized || this.active.get(key) !== gate) return; try { await this.runPollCycle(gate, key); } finally { if (!gate.finalized && this.active.get(key) === gate) { this.scheduleNextPoll(gate); } } } /** Arm the next poll. No-op once the gate is finalized. */ private scheduleNextPoll(gate: ActiveGate): void { if (gate.finalized) return; gate.timer = setTimeout(() => { void this.poll(gate); }, POLL_INTERVAL_MS); } private async runPollCycle(gate: ActiveGate, key: string): Promise { let observed: ObservedContainer[]; try { observed = await withTimeout( this.observeContainers(gate), OBSERVE_TIMEOUT_MS, 'health gate observe', ); } catch (error) { gate.consecutivePollErrors += 1; console.warn( '[HealthGate] poll error %d for %s:', gate.consecutivePollErrors, sanitizeForLog(gate.stackName), getErrorMessage(error, 'unknown'), ); if (gate.consecutivePollErrors >= 3) { this.finalize(gate, 'unknown', 'Docker became unreachable during observation', []); } return; } // The await above can straddle a supersede or stop; never act on a gate // that was finalized mid-flight. if (gate.finalized || this.active.get(key) !== gate) return; gate.consecutivePollErrors = 0; const elapsedMs = Date.now() - gate.startedAt; if (gate.expected === null) { if (observed.length > 0) { gate.expected = new Map(observed.map(c => [c.name, c])); } else if (elapsedMs >= EMPTY_GRACE_MS) { this.finalize(gate, 'unknown', 'no containers found to observe', []); } return; } const byName = new Map(observed.map(c => [c.name, c])); // First pass: restart accounting for every expected container still // present, so the summary below reflects the tallies the checks act on. A // restart counts when the container was replaced (new id), relaunched // (StartedAt moved), or Docker bumped its RestartCount; at most one // restart is tallied per poll regardless of how many occurred in the gap. for (const [name, baseline] of gate.expected) { const current = byName.get(name); if (!current) continue; const restarted = current.id !== baseline.id || current.restartCount > baseline.restartCount || (current.startedAt !== null && baseline.startedAt !== null && current.startedAt !== baseline.startedAt); current.restarts = baseline.restarts + (restarted ? 1 : 0); } const summary = this.summarize(gate.expected, byName); // Second pass: fail fast on a clearly bad state. for (const [name, baseline] of gate.expected) { const current = byName.get(name); if (!current) { if (gate.missingLastPoll.has(name)) { this.finalize(gate, 'failed', `container ${name} disappeared during observation`, summary); return; } gate.missingLastPoll.add(name); continue; } gate.missingLastPoll.delete(name); if (current.state === 'exited' && baseline.restarts === current.restarts) { // An exit with no restart attempt is terminal for the window. this.finalize(gate, 'failed', `container ${name} exited during observation`, summary); return; } if (current.health === 'unhealthy') { this.finalize(gate, 'failed', `container ${name} reported unhealthy`, summary); return; } if (current.restarts >= 2) { this.finalize(gate, 'failed', `container ${name} is restart looping`, summary); return; } if (current.state === 'restarting') { if (gate.restartingLastPoll.has(name)) { this.finalize(gate, 'failed', `container ${name} is stuck restarting`, summary); return; } gate.restartingLastPoll.add(name); } else { gate.restartingLastPoll.delete(name); } // Carry the running restart tally forward as the new baseline. gate.expected.set(name, current); } if (elapsedMs < gate.windowSeconds * 1000) return; // Window complete: pass requires everything running and healthy wherever a // healthcheck exists. A health state still 'starting' is not a pass. const stillStarting = observed.filter(c => c.health === 'starting'); if (stillStarting.length > 0) { this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary); return; } const notRunning = [...gate.expected.keys()].filter(name => byName.get(name)?.state !== 'running'); if (notRunning.length > 0) { this.finalize(gate, 'failed', `not running at the end of the window: ${notRunning.join(', ')}`, summary); return; } this.finalize(gate, 'passed', null, summary); } private async observeContainers(gate: ActiveGate): Promise { const docker = DockerController.getInstance(gate.nodeId).getDocker(); const listed = await docker.listContainers({ all: true, filters: { label: [`com.docker.compose.project=${gate.stackName}`] }, }); const observed = await Promise.all( listed.map(async (info): Promise => { try { const inspect = await docker.getContainer(info.Id).inspect(); return { id: info.Id, name: info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12), startedAt: inspect.State?.StartedAt ?? null, restartCount: typeof inspect.RestartCount === 'number' ? inspect.RestartCount : 0, restarts: 0, state: inspect.State?.Status ?? info.State ?? 'unknown', health: inspect.State?.Health?.Status ?? null, }; } catch (e: unknown) { // Removed between list and inspect; the missing-container logic will // see its absence on this or the next poll. if ((e as { statusCode?: number })?.statusCode === 404) return null; throw e; } }), ); return observed.filter((c): c is ObservedContainer => c !== null); } private summarize( expected: Map, current: Map, ): HealthGateContainer[] { return [...expected.values()].map(baseline => { const now = current.get(baseline.name); return { name: baseline.name, state: now?.state ?? 'missing', health: now?.health ?? null, restarts: now?.restarts ?? baseline.restarts, }; }); } private finalize( gate: ActiveGate, status: 'passed' | 'failed' | 'unknown', reason: string | null, containers: HealthGateContainer[], ): void { if (gate.finalized) return; gate.finalized = true; if (gate.timer) clearTimeout(gate.timer); const key = `${gate.nodeId}:${gate.stackName}`; if (this.active.get(key) === gate) this.active.delete(key); try { DatabaseService.getInstance().finalizeHealthGateRun( gate.runId, status, reason, Date.now(), JSON.stringify(containers), ); } catch (error) { // The verdict is lost from the DB (the startup sweep will later rewrite // the row as unknown), so log everything needed to reconstruct it. console.error( '[HealthGate] Failed to persist verdict %s (%s) for run %s, stack %s:', status, sanitizeForLog(reason ?? 'no reason'), gate.runId, sanitizeForLog(gate.stackName), getErrorMessage(error, 'unknown'), ); } if (status === 'passed') { this.recordActivity(gate.nodeId, gate.stackName, 'info', 'health_gate_passed', `${gate.stackName} health gate passed after ${gate.windowSeconds}s`, 'system'); } else if (status === 'failed') { this.recordActivity(gate.nodeId, gate.stackName, 'warning', 'health_gate_failed', `${gate.stackName} health gate failed: ${reason ?? 'unknown reason'}`, 'system'); } } private recordActivity( nodeId: number, stackName: string, level: 'info' | 'warning', category: 'update_started' | 'health_gate_passed' | 'health_gate_failed', message: string, actor: string | null, ): void { try { DatabaseService.getInstance().addNotificationHistory(nodeId, { level, category, message, timestamp: Date.now(), stack_name: stackName, actor_username: actor, }); } catch (error) { console.warn('[HealthGate] Failed to record activity for %s:', sanitizeForLog(stackName), getErrorMessage(error, 'unknown')); } } private readSettings(): { enabled: boolean; windowSeconds: number } { try { const settings = DatabaseService.getInstance().getGlobalSettings(); const windowRaw = parseInt(settings['health_gate_window_seconds'] ?? '', 10); const windowSeconds = Number.isFinite(windowRaw) ? Math.min(MAX_WINDOW_SECONDS, Math.max(MIN_WINDOW_SECONDS, windowRaw)) : DEFAULT_WINDOW_SECONDS; return { enabled: settings['health_gate_enabled'] !== '0', windowSeconds }; } catch (error) { // Safe default: observing is non-destructive, so a settings read // failure keeps the gate on with the default window. console.warn('[HealthGate] Settings read failed; using defaults:', getErrorMessage(error, 'unknown')); return { enabled: true, windowSeconds: DEFAULT_WINDOW_SECONDS }; } } }