diff --git a/backend/src/__tests__/failure-classifier.test.ts b/backend/src/__tests__/failure-classifier.test.ts new file mode 100644 index 00000000..d771738a --- /dev/null +++ b/backend/src/__tests__/failure-classifier.test.ts @@ -0,0 +1,154 @@ +import { describe, it, expect } from 'vitest'; +import { classifyFailure } from '../services/updateGuard/failureClassifier'; +import type { FailureReason } from '../services/updateGuard/types'; + +describe('classifyFailure', () => { + const cases: Array<{ name: string; message: string; reason: FailureReason }> = [ + { + name: 'container crash sentinel', + message: 'CONTAINER_CRASHED\nExit Code: 137\nContainer exited after deployment. Check container logs for details.', + reason: 'container_exited', + }, + { + name: 'idle stall sentinel', + message: 'STACK_STALLED_OUTPUT: no output for 600s', + reason: 'unknown', + }, + { + name: 'docker daemon down (message)', + message: 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?', + reason: 'node_unreachable', + }, + { + name: 'docker desktop connect error', + message: 'error during connect: this error may indicate that the docker daemon is not running', + reason: 'node_unreachable', + }, + { + name: 'required variable missing', + message: 'error while interpolating services.web.environment.TOKEN: required variable REQ_TOKEN is missing a value: must be provided', + reason: 'env_missing', + }, + { + name: 'variable not set (logfmt warning escalated)', + message: 'level=warning msg="The \\"DB_HOST\\" variable is not set. Defaulting to a blank string."', + reason: 'env_missing', + }, + { + name: 'env file not found', + message: "env file /compose/app/.env not found: stat /compose/app/.env: no such file or directory", + reason: 'env_missing', + }, + { + name: 'pull access denied', + message: 'Error response from daemon: pull access denied for ghost/missing, repository does not exist or may require docker login', + reason: 'image_pull_failed', + }, + { + name: 'manifest unknown', + message: 'manifest unknown: manifest unknown', + reason: 'image_pull_failed', + }, + { + name: 'registry rate limited', + message: 'toomanyrequests: You have reached your pull rate limit.', + reason: 'image_pull_failed', + }, + { + name: 'port already allocated', + message: 'Error response from daemon: driver failed programming external connectivity: Bind for 0.0.0.0:8080 failed: port is already allocated', + reason: 'port_conflict', + }, + { + name: 'address already in use', + message: 'listen tcp4 0.0.0.0:443: bind: address already in use', + reason: 'port_conflict', + }, + { + name: 'windows ports not available', + message: 'ports are not available: exposing port TCP 0.0.0.0:5432 -> 0.0.0.0:0', + reason: 'port_conflict', + }, + { + name: 'bind source missing', + message: 'Error response from daemon: invalid mount config for type "bind": bind source path does not exist: /srv/missing', + reason: 'bind_path_missing', + }, + { + name: 'permission denied', + message: 'open /compose/app/data: permission denied', + reason: 'permission_denied', + }, + { + name: 'unhealthy dependency', + message: 'dependency failed to start: container app-db-1 is unhealthy', + reason: 'healthcheck_failed', + }, + { + name: 'dependency exited', + message: 'dependency failed to start: container app-db-1 exited (1)', + reason: 'dependency_unavailable', + }, + { + name: 'external network missing', + message: 'network proxy_net declared as external, but could not be found', + reason: 'dependency_unavailable', + }, + { + name: 'yaml syntax error', + message: 'yaml: line 14: mapping values are not allowed in this context', + reason: 'compose_render_failed', + }, + { + name: 'undefined volume', + message: 'service "web" refers to undefined volume data: invalid compose project', + reason: 'compose_render_failed', + }, + { + name: 'unmatched output falls back to unknown', + message: 'something completely unexpected happened', + reason: 'unknown', + }, + ]; + + it.each(cases)('classifies $name as $reason', ({ message, reason }) => { + const result = classifyFailure(message); + expect(result.reason).toBe(reason); + expect(result.label.length).toBeGreaterThan(0); + expect(result.suggestion.length).toBeGreaterThan(0); + }); + + it('short-circuits to node_unreachable when the route flags a dead daemon', () => { + expect(classifyFailure('arbitrary text', { dockerUnavailable: true }).reason).toBe('node_unreachable'); + }); + + it('prefers env_missing over compose_render_failed when a render fails on a missing variable', () => { + const result = classifyFailure( + 'invalid compose project: required variable DB_PASS is missing a value', + ); + expect(result.reason).toBe('env_missing'); + }); + + it('prefers the crash sentinel over any other pattern in the same output', () => { + const result = classifyFailure( + 'CONTAINER_CRASHED\nExit Code: 1\nport is already allocated', + ); + expect(result.reason).toBe('container_exited'); + }); + + it('classifies the underlying cause of a rolled-back update from its message', () => { + // ComposeRollbackError copies the original error message, so the route can + // pass getErrorMessage(error) unchanged for wrapped failures. + const causeMessage = 'pull access denied for private/app'; + expect(classifyFailure(causeMessage).reason).toBe('image_pull_failed'); + }); + + it('always returns a classification (total over arbitrary input)', () => { + for (const message of ['', ' ', 'x'.repeat(10_000)]) { + const result = classifyFailure(message); + expect(result.reason).toBeDefined(); + expect(result.label).toBeDefined(); + expect(result.suggestion).toBeDefined(); + } + }); +}); diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index d832753b..c30c683a 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -1052,6 +1052,29 @@ describe('GitSourceService.apply', () => { .rejects.toMatchObject({ code: 'GIT_ERROR', message: expect.stringMatching(/pending commit has changed/i) }); }); + it('begins a deploy health gate after a successful apply-and-deploy', async () => { + const sha = 'eeee555eeee555eeee555eeee555eeee555eeee5'; + const svc = await seedPending('apply-deploy-gate', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const { HealthGateService } = await import('../services/HealthGateService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-git'); + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + + const result = await svc.apply('apply-deploy-gate', sha, { deploy: true }); + expect(result.deployed).toBe(true); + expect(deploySpy).toHaveBeenCalledWith('apply-deploy-gate'); + expect(beginSpy).toHaveBeenCalledWith(nodeId, 'apply-deploy-gate', 'deploy', 'system:git-source'); + + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + beginSpy.mockRestore(); + }); + it('returns deployError when the deploy step fails after writing to disk', async () => { const sha = 'cccc333cccc333cccc333cccc333cccc333cccc3'; const svc = await seedPending('apply-deploy-fail', 'services:\n x:\n image: alpine\n', sha); diff --git a/backend/src/__tests__/health-gate-service.test.ts b/backend/src/__tests__/health-gate-service.test.ts new file mode 100644 index 00000000..fdd0ac23 --- /dev/null +++ b/backend/src/__tests__/health-gate-service.test.ts @@ -0,0 +1,402 @@ +/** + * State-machine tests for HealthGateService with fake timers and an in-memory + * DatabaseService mock: verdicts, restart detection, supersede semantics, + * startup sweep, the disabled setting, the concurrency cap, and timer hygiene. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +interface StoredRun { + id: string; + node_id: number; + stack_name: string; + trigger_action: 'update' | 'deploy'; + status: 'observing' | 'passed' | 'failed' | 'unknown'; + reason: string | null; + window_seconds: number; + containers_json: string; + started_at: number; + ended_at: number | null; + created_by: string | null; +} + +const { state } = vi.hoisted(() => ({ + state: { + runs: new Map(), + activity: [] as Array<{ category?: string; message: string; level: string }>, + settings: {} as Record, + listContainers: vi.fn(), + inspect: vi.fn(), + }, +})); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getGlobalSettings: () => state.settings, + insertHealthGateRun: (run: StoredRun) => { state.runs.set(run.id, { ...run }); }, + finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string) => { + const run = state.runs.get(id); + if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson }); + }, + getHealthGateRun: (nodeId: number, stackName: string, id: string) => { + const run = state.runs.get(id); + return run && run.node_id === nodeId && run.stack_name === stackName ? { ...run } : undefined; + }, + getLatestHealthGateRun: (nodeId: number, stackName: string) => { + const matches = [...state.runs.values()] + .filter(r => r.node_id === nodeId && r.stack_name === stackName) + .sort((a, b) => b.started_at - a.started_at); + return matches[0] ? { ...matches[0] } : undefined; + }, + markInterruptedHealthGateRuns: (reason: string, endedAt: number) => { + let n = 0; + for (const run of state.runs.values()) { + if (run.status === 'observing') { + Object.assign(run, { status: 'unknown', reason, ended_at: endedAt }); + n++; + } + } + return n; + }, + addNotificationHistory: (_nodeId: number, item: { category?: string; message: string; level: string }) => { + state.activity.push(item); + return { ...item, id: state.activity.length, is_read: false }; + }, + }), + }, +})); + +vi.mock('../services/DockerController', () => ({ + default: { + getInstance: () => ({ + getDocker: () => ({ + listContainers: state.listContainers, + getContainer: (id: string) => ({ inspect: () => state.inspect(id) }), + }), + }), + }, +})); + +import { HealthGateService } from '../services/HealthGateService'; + +type ContainerFixture = { + id: string; + name: string; + state?: string; + health?: string | null; + restartCount?: number; + startedAt?: string; +}; + +/** Configure the docker mocks from a simple fixture list. */ +function setContainers(fixtures: ContainerFixture[]): void { + state.listContainers.mockResolvedValue(fixtures.map(f => ({ Id: f.id, Names: [`/${f.name}`], State: f.state ?? 'running' }))); + state.inspect.mockImplementation((id: string) => { + const f = fixtures.find(c => c.id === id); + if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 })); + return Promise.resolve({ + State: { + Status: f.state ?? 'running', + Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined, + StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z', + }, + RestartCount: f.restartCount ?? 0, + }); + }); +} + +const svc = () => HealthGateService.getInstance(); + +const latest = (stack = 'web') => svc().getReport(0, stack); + +async function ticks(n: number): Promise { + for (let i = 0; i < n; i++) { + await vi.advanceTimersByTimeAsync(5_000); + } +} + +beforeEach(() => { + vi.useFakeTimers(); + state.runs.clear(); + state.activity.length = 0; + state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' }; + state.listContainers.mockReset(); + state.inspect.mockReset(); + setContainers([{ id: 'aaa', name: 'web-app-1' }]); + svc().start(); +}); + +afterEach(() => { + svc().stop(); + expect(vi.getTimerCount()).toBe(0); + vi.useRealTimers(); +}); + +describe('HealthGateService verdicts', () => { + it('passes at the window end when containers stay running', async () => { + const id = svc().begin(0, 'web', 'update', 'tester'); + expect(id).toBeTruthy(); + await ticks(3); // 15s: still observing + expect(latest().status).toBe('observing'); + await ticks(4); // past the 30s window + expect(latest().status).toBe('passed'); + expect(state.activity.some(a => a.category === 'health_gate_passed')).toBe(true); + }); + + it('fails fast when a container exits', async () => { + svc().begin(0, 'web', 'update', 'tester'); + await ticks(1); // baseline + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]); + await ticks(1); + const report = latest(); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited'); + expect(state.activity.some(a => a.category === 'health_gate_failed')).toBe(true); + }); + + it('fails fast when a healthcheck reports unhealthy', async () => { + svc().begin(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', health: 'unhealthy' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('unhealthy'); + }); + + it('detects a restart loop via container replacement (new id)', async () => { + svc().begin(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'bbb', name: 'web-app-1' }]); + await ticks(1); // restart 1 observed; carried as new baseline + setContainers([{ id: 'ccc', name: 'web-app-1' }]); + await ticks(1); // restart 2: loop + const report = latest(); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('restart looping'); + // The persisted summary reflects the tally the verdict acted on. + expect(report.containers).toEqual([ + expect.objectContaining({ name: 'web-app-1', restarts: 2 }), + ]); + }); + + it('detects a restart loop via RestartCount and StartedAt movement', async () => { + svc().begin(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', restartCount: 1 }]); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', restartCount: 1, startedAt: '2026-06-10T00:05:00Z' }]); + await ticks(1); + const report = latest(); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('restart looping'); + expect(report.containers).toEqual([ + expect.objectContaining({ name: 'web-app-1', restarts: 2 }), + ]); + }); + + it('tolerates a one-poll disappearance but fails on two consecutive misses', async () => { + svc().begin(0, 'web', 'update', 'tester'); + await ticks(1); // baseline + setContainers([]); // one missed poll: tolerated + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1' }]); // back before the second miss + await ticks(5); // through the 30s window + expect(latest().status).toBe('passed'); + + const second = svc().begin(0, 'web', 'update', 'tester')!; + await ticks(1); + setContainers([]); + await ticks(2); // two consecutive misses: disappeared + const report = svc().getReport(0, 'web', second); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('disappeared'); + }); + + it('fails when a container is stuck restarting across consecutive polls', async () => { + svc().begin(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'restarting' }]); + await ticks(2); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('restarting'); + }); + + it('goes unknown after three consecutive docker errors', async () => { + svc().begin(0, 'web', 'update', 'tester'); + await ticks(1); + state.listContainers.mockRejectedValue(new Error('socket gone')); + await ticks(3); + expect(latest().status).toBe('unknown'); + expect(latest().reason).toContain('unreachable'); + }); + + it('resolves unknown when every docker observe hangs', async () => { + svc().begin(0, 'web', 'update', 'tester'); + // A wedged socket never settles. The per-observe timeout turns each poll + // into an error, and three in a row finalize the gate unknown instead of + // observing forever on a pending promise. + state.listContainers.mockImplementation(() => new Promise(() => {})); + // Each cycle is the 5s interval plus the 8s observe timeout; 45s covers + // three of them. + await vi.advanceTimersByTimeAsync(45_000); + expect(latest().status).toBe('unknown'); + expect(latest().reason).toContain('unreachable'); + }); + + it('recovers from a transient observe timeout instead of finalizing', async () => { + svc().begin(0, 'web', 'update', 'tester'); + // One observe wedges and times out (a single strike), then the socket + // recovers; the gate must keep observing, not give up at one error. + state.listContainers.mockImplementationOnce(() => new Promise(() => {})); + // 14s covers the first cycle's 5s wait plus 8s timeout. + await vi.advanceTimersByTimeAsync(14_000); + expect(latest().status).toBe('observing'); + // Later polls succeed and carry the gate to a pass at the window end. + await vi.advanceTimersByTimeAsync(50_000); + expect(latest().status).toBe('passed'); + }); + + it('runs polls single-flight: no second observe until the first settles', async () => { + svc().begin(0, 'web', 'update', 'tester'); + let release: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {}; + state.listContainers.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + // Advance past a second poll interval while the first observe is still + // pending. Self-scheduling means the next poll is armed only after the + // current cycle settles, so listContainers is entered exactly once. + await vi.advanceTimersByTimeAsync(7_000); + expect(state.listContainers).toHaveBeenCalledTimes(1); + // Let the first cycle finish; the next poll then runs and observes again. + release([{ Id: 'aaa', Names: ['/web-app-1'], State: 'running' }]); + await vi.advanceTimersByTimeAsync(5_000); + expect(state.listContainers.mock.calls.length).toBeGreaterThanOrEqual(2); + }); + + it('ends unknown when a healthcheck is still starting at the window end', async () => { + svc().begin(0, 'web', 'update', 'tester'); + setContainers([{ id: 'aaa', name: 'web-app-1', health: 'starting' }]); + await ticks(7); + expect(latest().status).toBe('unknown'); + expect(latest().reason).toContain('still starting'); + }); + + it('goes unknown when no containers ever appear', async () => { + setContainers([]); + svc().begin(0, 'web', 'update', 'tester'); + await ticks(4); + expect(latest().status).toBe('unknown'); + expect(latest().reason).toContain('no containers'); + }); +}); + +describe('HealthGateService lifecycle', () => { + it('never lets a poll that straddled a supersede overwrite the terminal verdict', async () => { + // A poll is mid-await on Docker when a newer update supersedes the gate; + // when the await resolves with healthy containers, the superseded run + // must keep its terminal unknown verdict. + const first = svc().begin(0, 'web', 'update', 'tester')!; + await ticks(2); // baseline established, healthy + + let releasePoll: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {}; + state.listContainers.mockImplementationOnce( + () => new Promise(resolve => { releasePoll = resolve; }), + ); + const straddlingPoll = vi.advanceTimersByTimeAsync(5_000); // poll now awaiting Docker + + const second = svc().begin(0, 'web', 'update', 'tester')!; + expect(svc().getReport(0, 'web', first).status).toBe('unknown'); + + releasePoll([{ Id: 'aaa', Names: ['/web-app-1'], State: 'running' }]); + await straddlingPoll; + + const superseded = svc().getReport(0, 'web', first); + expect(superseded.status).toBe('unknown'); + expect(superseded.reason).toContain('superseded'); + + await ticks(7); + expect(svc().getReport(0, 'web', second).status).toBe('passed'); + }); + + it('supersede finalizes the old run as unknown, clears its timer, and getRun still resolves it', async () => { + const first = svc().begin(0, 'web', 'update', 'tester')!; + await ticks(1); + const timersBefore = vi.getTimerCount(); + const second = svc().begin(0, 'web', 'update', 'tester')!; + expect(vi.getTimerCount()).toBe(timersBefore); // old interval cleared, new one added + + const superseded = svc().getReport(0, 'web', first); + expect(superseded.status).toBe('unknown'); + expect(superseded.reason).toContain('superseded'); + + await ticks(7); + expect(svc().getReport(0, 'web', second).status).toBe('passed'); + // The by-id read still returns the superseded run unchanged. + expect(svc().getReport(0, 'web', first).status).toBe('unknown'); + }); + + it('start() sweeps runs left observing by a previous process', () => { + state.runs.set('stale', { + id: 'stale', node_id: 0, stack_name: 'web', trigger_action: 'update', status: 'observing', + reason: null, window_seconds: 30, containers_json: '[]', started_at: 1, ended_at: null, created_by: null, + }); + svc().start(); + expect(state.runs.get('stale')!.status).toBe('unknown'); + expect(state.runs.get('stale')!.reason).toContain('restarted'); + }); + + it('no-ops when disabled but still records the update_started event', () => { + state.settings.health_gate_enabled = '0'; + const id = svc().begin(0, 'web', 'update', 'tester'); + expect(id).toBeNull(); + expect(state.runs.size).toBe(0); + expect(state.activity.some(a => a.category === 'update_started')).toBe(true); + expect(vi.getTimerCount()).toBe(0); + }); + + it('records update_started for update triggers but not deploy triggers', () => { + svc().begin(0, 'web', 'deploy', 'tester'); + expect(state.activity.some(a => a.category === 'update_started')).toBe(false); + svc().begin(0, 'web', 'update', 'tester'); + expect(state.activity.some(a => a.category === 'update_started')).toBe(true); + }); + + it('refuses to begin before start() so shutdown cannot leak timers', () => { + svc().stop(); + expect(svc().begin(0, 'web', 'update', 'tester')).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + svc().start(); + }); + + it('persists an immediate unknown past the concurrency cap', () => { + for (let i = 0; i < 25; i++) { + svc().begin(0, `stack-${i}`, 'update', 'tester'); + } + const overCap = svc().begin(0, 'one-too-many', 'update', 'tester')!; + const report = svc().getReport(0, 'one-too-many', overCap); + expect(report.status).toBe('unknown'); + expect(report.reason).toContain('concurrent'); + }); + + it('clamps the configured window into its valid range and falls back on garbage', () => { + state.settings.health_gate_window_seconds = '99999'; + const a = svc().begin(0, 'web', 'update', 'tester')!; + expect(svc().getReport(0, 'web', a).windowSeconds).toBe(600); + state.settings.health_gate_window_seconds = 'banana'; + const b = svc().begin(0, 'web', 'update', 'tester')!; + expect(svc().getReport(0, 'web', b).windowSeconds).toBe(90); + }); + + it('returns the never-run sentinel for a stack with no runs', () => { + const report = svc().getReport(0, 'nothing-here'); + expect(report.status).toBe('never-run'); + expect(report.id).toBeNull(); + }); + + it('stop() finalizes in-flight gates as unknown with zero timers left', async () => { + const id = svc().begin(0, 'web', 'update', 'tester')!; + await ticks(1); + svc().stop(); + expect(vi.getTimerCount()).toBe(0); + expect(svc().getReport(0, 'web', id).status).toBe('unknown'); + svc().start(); + }); +}); diff --git a/backend/src/__tests__/image-updates-routes.test.ts b/backend/src/__tests__/image-updates-routes.test.ts index 2689c9bb..e90aa3b2 100644 --- a/backend/src/__tests__/image-updates-routes.test.ts +++ b/backend/src/__tests__/image-updates-routes.test.ts @@ -207,4 +207,37 @@ describe('POST /api/auto-update/execute', () => { expect(res.status).toBe(200); expect(typeof res.body.result).toBe('string'); }); + + it('begins an update health gate after an auto-update applies', async () => { + // Target a single stack; the route works off the running containers, so + // stub the container probe, the update check, and the compose update, then + // assert the gate begins for the applied stack. + const { TEST_USERNAME } = await import('./helpers/setupTestDb'); + const DockerController = (await import('../services/DockerController')).default; + const { ImageUpdateService } = await import('../services/ImageUpdateService'); + const { ComposeService } = await import('../services/ComposeService'); + const { HealthGateService } = await import('../services/HealthGateService'); + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + + const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') + .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); + const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') + .mockResolvedValue({ hasUpdate: true } as never); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue(); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-au'); + try { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ target: 'auto-upd-gate' }); + expect(res.status).toBe(200); + expect(updateSpy).toHaveBeenCalledWith('auto-upd-gate', undefined, true); + expect(beginSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate', 'update', `auto-update:${TEST_USERNAME}`); + } finally { + containersSpy.mockRestore(); + checkSpy.mockRestore(); + updateSpy.mockRestore(); + beginSpy.mockRestore(); + } + }); }); diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index 2a53093b..6713010a 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -647,6 +647,32 @@ describe('SchedulerService - executeUpdate', () => { ); }); + it('begins a health gate after a scheduled update succeeds', async () => { + const { HealthGateService } = await import('../services/HealthGateService'); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-1'); + try { + mockGetScheduledTask.mockReturnValue({ + id: 83, + name: 'update-gated', + action: 'update', + cron_expression: '0 4 * * *', + enabled: true, + target_id: 'web-app', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]); + mockCheckImage.mockResolvedValue({ hasUpdate: true }); + + await SchedulerService.getInstance().triggerTask(83); + + expect(beginSpy).toHaveBeenCalledWith(1, 'web-app', 'update', 'system:scheduler'); + } finally { + beginSpy.mockRestore(); + } + }); + it('skips when all images up to date', async () => { mockGetScheduledTask.mockReturnValue({ id: 81, diff --git a/backend/src/__tests__/settings-routes.test.ts b/backend/src/__tests__/settings-routes.test.ts index 5c362cb4..2bbef2e4 100644 --- a/backend/src/__tests__/settings-routes.test.ts +++ b/backend/src/__tests__/settings-routes.test.ts @@ -236,6 +236,63 @@ describe('prune_on_update (auto-prune after updates)', () => { }); }); +describe('health gate settings', () => { + it('seeds enabled with a 90 second window in a fresh database', () => { + const settings = DatabaseService.getInstance().getGlobalSettings(); + expect(settings.health_gate_enabled).toBe('1'); + expect(settings.health_gate_window_seconds).toBe('90'); + }); + + it('is exposed through the settings GET projection', async () => { + const res = await request(app).get('/api/settings').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.health_gate_enabled).toBeDefined(); + expect(res.body.health_gate_window_seconds).toBeDefined(); + }); + + it('accepts a single-key toggle write and persists it', async () => { + const res = await request(app) + .post('/api/settings') + .set('Cookie', adminCookie) + .send({ key: 'health_gate_enabled', value: '0' }); + expect(res.status).toBe(200); + expect(DatabaseService.getInstance().getGlobalSettings().health_gate_enabled).toBe('0'); + DatabaseService.getInstance().updateGlobalSetting('health_gate_enabled', '1'); + }); + + it('accepts an in-range window via bulk PATCH alongside another key', async () => { + const res = await request(app) + .patch('/api/settings') + .set('Cookie', adminCookie) + .send({ health_gate_window_seconds: 120, health_gate_enabled: '1' }); + expect(res.status).toBe(200); + expect(DatabaseService.getInstance().getGlobalSettings().health_gate_window_seconds).toBe('120'); + DatabaseService.getInstance().updateGlobalSetting('health_gate_window_seconds', '90'); + }); + + it('rejects out-of-range windows and non-enum toggles', async () => { + const tooShort = await request(app) + .post('/api/settings') + .set('Cookie', adminCookie) + .send({ key: 'health_gate_window_seconds', value: '5' }); + expect(tooShort.status).toBe(400); + + const tooLong = await request(app) + .post('/api/settings') + .set('Cookie', adminCookie) + .send({ key: 'health_gate_window_seconds', value: '9000' }); + expect(tooLong.status).toBe(400); + + const badToggle = await request(app) + .post('/api/settings') + .set('Cookie', adminCookie) + .send({ key: 'health_gate_enabled', value: 'yes' }); + expect(badToggle.status).toBe(400); + + expect(DatabaseService.getInstance().getGlobalSettings().health_gate_window_seconds).toBe('90'); + }); +}); + describe('PATCH /api/settings (bulk update)', () => { it('rejects unauthenticated requests with 401', async () => { const res = await request(app).patch('/api/settings').send({ host_cpu_limit: 50 }); diff --git a/backend/src/__tests__/stacks-failure-notifications.test.ts b/backend/src/__tests__/stacks-failure-notifications.test.ts index 6088748f..4a225329 100644 --- a/backend/src/__tests__/stacks-failure-notifications.test.ts +++ b/backend/src/__tests__/stacks-failure-notifications.test.ts @@ -7,7 +7,7 @@ * ComposeService and DockerController are mocked so no real Docker daemon is * required. NotificationService.dispatchAlert is spied on to assert dispatch. */ -import { describe, it, expect, beforeAll, afterAll, vi, beforeEach } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, vi, beforeEach, afterEach } from 'vitest'; import request from 'supertest'; import jwt from 'jsonwebtoken'; import { setupTestDb, cleanupTestDb, loginAsTestAdmin, TEST_JWT_SECRET } from './helpers/setupTestDb'; @@ -242,6 +242,134 @@ describe('deploy_failure notification on /deploy error', () => { }); }); +describe('health gate begin call sites', () => { + let beginSpy: ReturnType; + + beforeEach(async () => { + const { HealthGateService } = await import('../services/HealthGateService'); + beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-123') as ReturnType; + }); + + afterEach(() => { + beginSpy.mockRestore(); + }); + + it('begins a gate after a manual deploy and returns its id', async () => { + mockDeployStack.mockResolvedValue(undefined); + const res = await request(app) + .post('/api/stacks/myapp/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + expect(res.status).toBe(200); + expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'myapp', 'deploy', 'testadmin'); + expect(res.body.healthGateId).toBe('gate-123'); + }); + + it('begins a gate after a manual update and returns its id', async () => { + mockUpdateStack.mockResolvedValue(undefined); + const res = await request(app) + .post('/api/stacks/myapp/update') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + expect(res.status).toBe(200); + expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'myapp', 'update', 'testadmin'); + expect(res.body.healthGateId).toBe('gate-123'); + }); + + it('begins a gate per stack in a bulk update and carries ids in the results', async () => { + mockUpdateStack.mockResolvedValue(undefined); + const res = await request(app) + .post('/api/stacks/bulk') + .set('Cookie', authCookie) + .send({ action: 'update', stackNames: ['myapp', 'webapp'] }); + expect(res.status).toBe(200); + expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'myapp', 'update', 'testadmin'); + expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'webapp', 'update', 'testadmin'); + const items = res.body.results as Array<{ stackName: string; ok: boolean; healthGateId?: string | null }>; + expect(items).toHaveLength(2); + for (const item of items) { + expect(item.ok).toBe(true); + expect(item.healthGateId).toBe('gate-123'); + } + }); + + it('does not begin a gate on a failed deploy', async () => { + mockDeployStack.mockRejectedValue(new Error('boom')); + await request(app).post('/api/stacks/myapp/deploy').set('Cookie', authCookie); + expect(beginSpy).not.toHaveBeenCalled(); + }); + + it('never begins a gate for the rollback recovery path', async () => { + mockDeployStack.mockResolvedValue(undefined); + const res = await request(app) + .post('/api/stacks/myapp/rollback') + .set('Cookie', authCookie); + expect(res.status).toBe(200); + expect(beginSpy).not.toHaveBeenCalled(); + }); +}); + +describe('failure classification on deploy/update error responses', () => { + it('classifies a failed deploy and includes failure in the body', async () => { + mockDeployStack.mockRejectedValue( + new Error('Bind for 0.0.0.0:8080 failed: port is already allocated'), + ); + + const res = await request(app) + .post('/api/stacks/myapp/deploy') + .set('Cookie', authCookie); + + expect(res.status).toBe(500); + expect(res.body.failure).toMatchObject({ reason: 'port_conflict' }); + expect(typeof res.body.failure.label).toBe('string'); + expect(typeof res.body.failure.suggestion).toBe('string'); + }); + + it('classifies a failed update and includes failure in the body', async () => { + mockUpdateStack.mockRejectedValue( + new Error('pull access denied for private/app, repository does not exist'), + ); + + const res = await request(app) + .post('/api/stacks/myapp/update') + .set('Cookie', authCookie); + + expect(res.status).toBe(500); + expect(res.body.failure).toMatchObject({ reason: 'image_pull_failed' }); + }); + + it('classifies the underlying cause when the update was rolled back', async () => { + mockUpdateStack.mockRejectedValue( + new ComposeRollbackError( + new Error('dependency failed to start: container app-db-1 is unhealthy'), + true, + true, + ), + ); + + const res = await request(app) + .post('/api/stacks/myapp/update') + .set('Cookie', authCookie); + + expect(res.status).toBe(500); + expect(res.body).toMatchObject({ + rolledBack: true, + failure: { reason: 'healthcheck_failed' }, + }); + }); + + it('falls back to unknown for unrecognized failures', async () => { + mockDeployStack.mockRejectedValue(new Error('weird one-off explosion')); + + const res = await request(app) + .post('/api/stacks/myapp/deploy') + .set('Cookie', authCookie); + + expect(res.status).toBe(500); + expect(res.body.failure.reason).toBe('unknown'); + }); +}); + describe('post-deploy scan opt-out', () => { it('does not trigger a post-deploy scan when skip_scan is true', async () => { mockDeployStack.mockResolvedValue(undefined); diff --git a/backend/src/__tests__/update-guard-readiness.test.ts b/backend/src/__tests__/update-guard-readiness.test.ts new file mode 100644 index 00000000..4091815e --- /dev/null +++ b/backend/src/__tests__/update-guard-readiness.test.ts @@ -0,0 +1,191 @@ +import { describe, it, expect } from 'vitest'; +import { + aggregateVerdict, + backupSlotSignal, + containersSignal, + diskSignal, + driftSignal, + healthchecksSignal, + preflightSignal, + updatePreviewSignal, +} from '../services/updateGuard/readiness'; +import type { ContainerProbe, ReadinessSignal } from '../services/updateGuard/types'; +import type { UpdatePreviewSummary } from '../services/UpdatePreviewService'; + +const NOW = 1_750_000_000_000; + +const probe = (over: Partial = {}): ContainerProbe => ({ + name: 'app-web-1', + state: 'running', + health: null, + exitCode: null, + hasHealthcheck: false, + restartPolicy: 'unless-stopped', + mounts: [], + ...over, +}); + +const summary = (over: Partial = {}): UpdatePreviewSummary => ({ + has_update: false, + primary_image: null, + current_tag: null, + next_tag: null, + semver_bump: 'none', + update_kind: 'none', + blocked: false, + blocked_reason: null, + ...over, +}); + +describe('preflightSignal', () => { + const cases = [ + { status: 'never-run', expected: 'unknown', affects: false }, + { status: 'blocker', expected: 'blocked', affects: true }, + { status: 'unrenderable', expected: 'attention', affects: true }, + { status: 'high', expected: 'attention', affects: true }, + { status: 'warning', expected: 'warning', affects: true }, + { status: 'pass', expected: 'ok', affects: true }, + { status: 'info', expected: 'ok', affects: true }, + ] as const; + + it.each(cases)('maps preflight status $status to $expected', ({ status, expected, affects }) => { + const signal = preflightSignal({ status }); + expect(signal.status).toBe(expected); + expect(signal.affectsVerdict).toBe(affects); + }); + + it('degrades a read failure to a non-verdict-affecting unknown', () => { + const signal = preflightSignal('error'); + expect(signal.status).toBe('unknown'); + expect(signal.affectsVerdict).toBe(false); + }); +}); + +describe('driftSignal', () => { + it('warns on open findings and is ok at zero', () => { + expect(driftSignal(2).status).toBe('warning'); + expect(driftSignal(0).status).toBe('ok'); + expect(driftSignal('error')).toMatchObject({ status: 'unknown', affectsVerdict: false }); + }); +}); + +describe('containersSignal', () => { + it('is ok when all containers run normally', () => { + expect(containersSignal([probe(), probe({ name: 'app-db-1' })]).status).toBe('ok'); + }); + + it('warns when the stack is not running', () => { + const signal = containersSignal([]); + expect(signal.status).toBe('warning'); + expect(signal.detail).toContain('not running'); + }); + + it('flags unhealthy, restarting, and crashed containers for review', () => { + expect(containersSignal([probe({ health: 'unhealthy' })]).status).toBe('attention'); + expect(containersSignal([probe({ state: 'restarting' })]).status).toBe('attention'); + expect(containersSignal([probe({ state: 'exited', exitCode: 1 })]).status).toBe('attention'); + }); + + it('does not flag a cleanly exited container', () => { + expect(containersSignal([probe(), probe({ state: 'exited', exitCode: 0 })]).status).toBe('ok'); + }); + + it('treats a docker error as a verdict-affecting unknown', () => { + expect(containersSignal('error')).toMatchObject({ status: 'unknown', affectsVerdict: true }); + }); +}); + +describe('healthchecksSignal', () => { + it('is informational and never affects the verdict', () => { + for (const input of [[probe()], [probe({ hasHealthcheck: true })], [], 'error'] as const) { + const signal = healthchecksSignal(input as ContainerProbe[] | 'error'); + expect(signal.status).toBe('ok'); + expect(signal.affectsVerdict).toBe(false); + } + }); + + it('states coverage and missing restart policies', () => { + const signal = healthchecksSignal([ + probe({ hasHealthcheck: true }), + probe({ name: 'app-db-1', restartPolicy: null }), + ]); + expect(signal.detail).toContain('1 of 2'); + expect(signal.detail).toContain('no restart policy'); + }); +}); + +describe('updatePreviewSignal', () => { + it('reflects a policy block as blocked', () => { + const signal = updatePreviewSignal(summary({ blocked: true, blocked_reason: 'Policy "prod" blocks critical CVEs' })); + expect(signal.status).toBe('blocked'); + expect(signal.detail).toContain('prod'); + }); + + it('flags a major bump for review', () => { + expect(updatePreviewSignal(summary({ has_update: true, semver_bump: 'major', current_tag: '1.9.0', next_tag: '2.0.0' })).status).toBe('attention'); + }); + + it('warns on an unclassifiable pending update', () => { + expect(updatePreviewSignal(summary({ has_update: true, semver_bump: 'unknown' })).status).toBe('warning'); + }); + + it('is ok for patch and digest updates and for no update', () => { + expect(updatePreviewSignal(summary({ has_update: true, semver_bump: 'patch', update_kind: 'tag' })).status).toBe('ok'); + expect(updatePreviewSignal(summary({ has_update: true, update_kind: 'digest' })).status).toBe('ok'); + expect(updatePreviewSignal(summary()).status).toBe('ok'); + }); + + it('degrades a preview failure to a non-verdict-affecting unknown', () => { + expect(updatePreviewSignal('error')).toMatchObject({ status: 'unknown', affectsVerdict: false }); + }); +}); + +describe('backupSlotSignal', () => { + it('is ok with an existing backup and warns without one', () => { + expect(backupSlotSignal({ exists: true, timestamp: NOW - 60_000 }, NOW).status).toBe('ok'); + expect(backupSlotSignal({ exists: false, timestamp: null }, NOW).status).toBe('warning'); + expect(backupSlotSignal('error', NOW)).toMatchObject({ status: 'unknown', affectsVerdict: false }); + }); +}); + +describe('diskSignal', () => { + it('grades disk pressure against the alert threshold', () => { + expect(diskSignal({ usePercent: 50, limitPercent: 90 }).status).toBe('ok'); + expect(diskSignal({ usePercent: 86, limitPercent: 90 }).status).toBe('warning'); + expect(diskSignal({ usePercent: 90, limitPercent: 90 }).status).toBe('attention'); + expect(diskSignal({ usePercent: 97, limitPercent: 90 }).status).toBe('attention'); + expect(diskSignal(null)).toMatchObject({ status: 'unknown', affectsVerdict: false }); + expect(diskSignal('error')).toMatchObject({ status: 'unknown', affectsVerdict: false }); + }); +}); + +describe('aggregateVerdict', () => { + const signal = (status: ReadinessSignal['status'], affectsVerdict = true): ReadinessSignal => ({ + id: 'drift', + status, + title: 't', + detail: 'd', + affectsVerdict, + }); + + it('orders blocked > attention > unknown > warning > ready', () => { + expect(aggregateVerdict([signal('ok'), signal('blocked'), signal('attention'), signal('unknown'), signal('warning')])).toBe('blocked'); + expect(aggregateVerdict([signal('ok'), signal('attention'), signal('unknown'), signal('warning')])).toBe('review_required'); + expect(aggregateVerdict([signal('ok'), signal('unknown'), signal('warning')])).toBe('unknown'); + expect(aggregateVerdict([signal('ok'), signal('warning')])).toBe('ready_with_warnings'); + expect(aggregateVerdict([signal('ok'), signal('ok')])).toBe('ready'); + }); + + it('ignores informational unknowns', () => { + expect(aggregateVerdict([signal('ok'), signal('unknown', false)])).toBe('ready'); + expect(aggregateVerdict([signal('warning'), signal('unknown', false)])).toBe('ready_with_warnings'); + }); + + it('reaches every verdict from realistic signal sets', () => { + expect(aggregateVerdict([preflightSignal({ status: 'blocker' }), driftSignal(0)])).toBe('blocked'); + expect(aggregateVerdict([preflightSignal({ status: 'high' }), driftSignal(0)])).toBe('review_required'); + expect(aggregateVerdict([containersSignal('error'), driftSignal(0)])).toBe('unknown'); + expect(aggregateVerdict([driftSignal(1), preflightSignal({ status: 'pass' })])).toBe('ready_with_warnings'); + expect(aggregateVerdict([driftSignal(0), preflightSignal({ status: 'pass' }), healthchecksSignal([probe()])])).toBe('ready'); + }); +}); diff --git a/backend/src/__tests__/update-guard-rollback.test.ts b/backend/src/__tests__/update-guard-rollback.test.ts new file mode 100644 index 00000000..6e14d128 --- /dev/null +++ b/backend/src/__tests__/update-guard-rollback.test.ts @@ -0,0 +1,219 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +// Mutable state the mocked NodeRegistry reads (same harness as +// filesystem-backup.test.ts) so FileSystemService can be constructed against +// temp directories. +const mockState = { composeDir: '' }; + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getComposeDir: () => mockState.composeDir, + getDefaultNodeId: () => 1, + }), + }, +})); + +import { + aggregateRollbackOverall, + buildRollbackItems, + type RollbackInputs, +} from '../services/updateGuard/readiness'; +import type { ContainerProbe } from '../services/updateGuard/types'; + +const NOW = 1_750_000_000_000; + +const baseInputs = (over: Partial = {}): RollbackInputs => ({ + backup: { exists: true, timestamp: NOW - 3_600_000 }, + envSummary: { exists: true, envPresent: true, keys: ['DB_HOST', 'DB_PASS'] }, + stackHasEnv: true, + rollbackTarget: { target: 'nginx:1.27.1' }, + lastDeployAt: NOW - 3_600_000, + containers: [{ + name: 'app-web-1', state: 'running', health: 'healthy', exitCode: null, + hasHealthcheck: true, restartPolicy: 'unless-stopped', mounts: ['volume app_data'], + }], + ...over, +}); + +const itemById = (inputs: RollbackInputs, id: string) => + buildRollbackItems(inputs, NOW).find(i => i.id === id)!; + +describe('buildRollbackItems', () => { + it('reports a full set of six items', () => { + const items = buildRollbackItems(baseInputs(), NOW); + expect(items.map(i => i.id)).toEqual([ + 'compose_source', 'env_keys', 'previous_images', 'last_deploy', 'healthchecks', 'volume_data', + ]); + }); + + it('marks the volume row not_covered unconditionally and names the mounts', () => { + const item = itemById(baseInputs(), 'volume_data'); + expect(item.state).toBe('not_covered'); + expect(item.detail).toContain('not included in file backups'); + expect(item.detail).toContain('volume app_data'); + + const noMounts = itemById(baseInputs({ containers: [] }), 'volume_data'); + expect(noMounts.state).toBe('not_covered'); + + const dockerDown = itemById(baseInputs({ containers: 'error' }), 'volume_data'); + expect(dockerDown.state).toBe('not_covered'); + }); + + it('exposes env coverage as names only', () => { + const item = itemById(baseInputs(), 'env_keys'); + expect(item.state).toBe('ready'); + expect(item.detail).toContain('2 variable names'); + expect(item.detail).not.toContain('DB_HOST'); + expect(item.detail).not.toContain('DB_PASS'); + }); + + it('treats a stack without an env file as covered', () => { + const item = itemById(baseInputs({ + envSummary: { exists: true, envPresent: false, keys: [] }, + stackHasEnv: false, + }), 'env_keys'); + expect(item.state).toBe('ready'); + expect(item.detail).toContain('no env file'); + }); + + it('flags an env file the backup predates', () => { + const item = itemById(baseInputs({ + envSummary: { exists: true, envPresent: false, keys: [] }, + stackHasEnv: true, + }), 'env_keys'); + expect(item.state).toBe('missing'); + }); + + it('marks the previous image unknown when no rollback target is known', () => { + expect(itemById(baseInputs({ rollbackTarget: { target: null } }), 'previous_images').state).toBe('unknown'); + expect(itemById(baseInputs({ rollbackTarget: 'error' }), 'previous_images').state).toBe('unknown'); + const known = itemById(baseInputs(), 'previous_images'); + expect(known.state).toBe('ready'); + expect(known.detail).toContain('nginx:1.27.1'); + }); + + it('does not mistake an image literally named error for a failed preview', () => { + const item = itemById(baseInputs({ rollbackTarget: { target: 'error' } }), 'previous_images'); + expect(item.state).toBe('ready'); + expect(item.detail).toContain('error'); + }); + + it('reports last deploy and healthcheck coverage', () => { + expect(itemById(baseInputs(), 'last_deploy').state).toBe('ready'); + expect(itemById(baseInputs({ lastDeployAt: null }), 'last_deploy').state).toBe('missing'); + expect(itemById(baseInputs(), 'healthchecks').state).toBe('ready'); + const none: ContainerProbe[] = [{ + name: 'a', state: 'running', health: null, exitCode: null, + hasHealthcheck: false, restartPolicy: null, mounts: [], + }]; + expect(itemById(baseInputs({ containers: none }), 'healthchecks').state).toBe('missing'); + }); +}); + +describe('aggregateRollbackOverall', () => { + it('is ready when compose, env, and previous image are all covered', () => { + expect(aggregateRollbackOverall(buildRollbackItems(baseInputs(), NOW))).toBe('ready'); + }); + + it('is not_ready without a backup slot', () => { + const items = buildRollbackItems(baseInputs({ + backup: { exists: false, timestamp: null }, + envSummary: { exists: false, envPresent: false, keys: [] }, + }), NOW); + expect(aggregateRollbackOverall(items)).toBe('not_ready'); + }); + + it('is partial when the previous image tag is unknown', () => { + const items = buildRollbackItems(baseInputs({ rollbackTarget: { target: null } }), NOW); + expect(aggregateRollbackOverall(items)).toBe('partial'); + }); + + it('is partial when env coverage is missing', () => { + const items = buildRollbackItems(baseInputs({ + envSummary: { exists: true, envPresent: false, keys: [] }, + stackHasEnv: true, + }), NOW); + expect(aggregateRollbackOverall(items)).toBe('partial'); + }); + + it('never gates on the volume or healthcheck disclosures', () => { + const items = buildRollbackItems(baseInputs({ containers: 'error' }), NOW); + expect(aggregateRollbackOverall(items)).toBe('ready'); + }); +}); + +describe('FileSystemService.getBackupEnvSummary', () => { + let tmpDir: string; + let composeDir: string; + let originalDataDir: string | undefined; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-envsum-')); + composeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-envsum-compose-')); + mockState.composeDir = composeDir; + originalDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = tmpDir; + }); + + afterEach(() => { + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + fs.rmSync(tmpDir, { recursive: true, force: true }); + fs.rmSync(composeDir, { recursive: true, force: true }); + }); + + async function getService() { + const { FileSystemService } = await import('../services/FileSystemService'); + return FileSystemService.getInstance(); + } + + function writeBackupEnv(stackName: string, content: string | null) { + const dir = path.join(tmpDir, 'backups', '1', stackName); + fs.mkdirSync(dir, { recursive: true }); + if (content !== null) fs.writeFileSync(path.join(dir, '.env'), content, 'utf-8'); + } + + it('returns key names only, never values', async () => { + writeBackupEnv('web', 'DB_HOST=db.internal\nDB_PASS=s3cret-value\n# comment\nEMPTY=\n INDENTED=ok\nnot a var line\n'); + const svc = await getService(); + const summary = await svc.getBackupEnvSummary('web'); + expect(summary).toEqual({ + exists: true, + envPresent: true, + keys: ['DB_HOST', 'DB_PASS', 'EMPTY', 'INDENTED'], + }); + expect(JSON.stringify(summary)).not.toContain('s3cret-value'); + expect(JSON.stringify(summary)).not.toContain('db.internal'); + }); + + it('reports a backup without an env file', async () => { + writeBackupEnv('web', null); + const svc = await getService(); + expect(await svc.getBackupEnvSummary('web')).toEqual({ exists: true, envPresent: false, keys: [] }); + }); + + it('reports a missing backup slot', async () => { + const svc = await getService(); + expect(await svc.getBackupEnvSummary('web')).toEqual({ exists: false, envPresent: false, keys: [] }); + }); + + it('rejects traversal-shaped stack names without touching the filesystem', async () => { + const svc = await getService(); + expect(await svc.getBackupEnvSummary('../../etc')).toEqual({ exists: false, envPresent: false, keys: [] }); + expect(await svc.getBackupEnvSummary('..')).toEqual({ exists: false, envPresent: false, keys: [] }); + }); + + it('propagates a non-ENOENT env read failure instead of reporting "no env in backup"', async () => { + // An unreadable .env (here: a directory, EISDIR) must throw so callers + // degrade the item to unknown rather than falsely claiming the backup + // contains no env file. + const dir = path.join(tmpDir, 'backups', '1', 'web'); + fs.mkdirSync(path.join(dir, '.env'), { recursive: true }); + const svc = await getService(); + await expect(svc.getBackupEnvSummary('web')).rejects.toThrow(); + }); +}); diff --git a/backend/src/__tests__/update-guard-routes.test.ts b/backend/src/__tests__/update-guard-routes.test.ts new file mode 100644 index 00000000..d2493527 --- /dev/null +++ b/backend/src/__tests__/update-guard-routes.test.ts @@ -0,0 +1,223 @@ +/** + * Route-level tests for the update guard endpoints: the readiness GETs on the + * stacks router and the fleet snapshot coverage lookup. UpdateGuardService and + * FileSystemService are mocked; the focus is auth, validation, route + * placement, and response shape. + */ +import { describe, it, expect, beforeAll, afterAll, vi, beforeEach } from 'vitest'; +import request from 'supertest'; +import bcrypt from 'bcrypt'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +const { + mockComputeUpdateReadiness, + mockComputeRollbackReadiness, +} = vi.hoisted(() => ({ + mockComputeUpdateReadiness: vi.fn(), + mockComputeRollbackReadiness: vi.fn(), +})); + +vi.mock('../services/UpdateGuardService', () => ({ + UpdateGuardService: { + getInstance: () => ({ + computeUpdateReadiness: mockComputeUpdateReadiness, + computeRollbackReadiness: mockComputeRollbackReadiness, + }), + }, +})); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getStacks: vi.fn().mockResolvedValue([]), + getBaseDir: () => '/tmp/compose', + hasComposeFile: vi.fn().mockResolvedValue(true), + }), + }, +})); + +let tmpDir: string; +let app: import('express').Express; +let authCookie: string; +let viewerCookie: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + authCookie = await loginAsTestAdmin(app); + + const { DatabaseService } = await import('../services/DatabaseService'); + const viewerHash = await bcrypt.hash('viewerpass', 1); + DatabaseService.getInstance().addUser({ username: 'guard-viewer', password_hash: viewerHash, role: 'viewer' }); + const viewerRes = await request(app).post('/api/auth/login').send({ username: 'guard-viewer', password: 'viewerpass' }); + const cookies = viewerRes.headers['set-cookie'] as string | string[]; + viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies; +}); + +afterAll(() => { + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + mockComputeUpdateReadiness.mockReset(); + mockComputeRollbackReadiness.mockReset(); +}); + +describe('GET /api/stacks/:stackName/update-readiness', () => { + it('requires authentication', async () => { + const res = await request(app).get('/api/stacks/web/update-readiness'); + expect(res.status).toBe(401); + expect(mockComputeUpdateReadiness).not.toHaveBeenCalled(); + }); + + it('returns the computed report', async () => { + const report = { stack: 'web', computedAt: 1, verdict: 'ready', signals: [] }; + mockComputeUpdateReadiness.mockResolvedValue(report); + const res = await request(app).get('/api/stacks/web/update-readiness').set('Cookie', authCookie); + expect(res.status).toBe(200); + expect(res.body).toEqual(report); + }); + + it('returns a clean 500 when the computation fails', async () => { + mockComputeUpdateReadiness.mockRejectedValue(new Error('docker exploded')); + const res = await request(app).get('/api/stacks/web/update-readiness').set('Cookie', authCookie); + expect(res.status).toBe(500); + expect(res.body).toEqual({ error: 'Failed to compute update readiness' }); + }); +}); + +describe('GET /api/stacks/:stackName/rollback-readiness', () => { + it('requires authentication', async () => { + const res = await request(app).get('/api/stacks/web/rollback-readiness'); + expect(res.status).toBe(401); + expect(mockComputeRollbackReadiness).not.toHaveBeenCalled(); + }); + + it('returns the computed report', async () => { + const report = { stack: 'web', computedAt: 1, overall: 'partial', items: [] }; + mockComputeRollbackReadiness.mockResolvedValue(report); + const res = await request(app).get('/api/stacks/web/rollback-readiness').set('Cookie', authCookie); + expect(res.status).toBe(200); + expect(res.body).toEqual(report); + }); + + it('returns a clean 500 when the computation fails', async () => { + mockComputeRollbackReadiness.mockRejectedValue(new Error('boom')); + const res = await request(app).get('/api/stacks/web/rollback-readiness').set('Cookie', authCookie); + expect(res.status).toBe(500); + expect(res.body).toEqual({ error: 'Failed to compute rollback readiness' }); + }); +}); + +describe('GET /api/stacks/:stackName/health-gate', () => { + const seedRun = async (id: string, status: 'observing' | 'passed' | 'failed' | 'unknown', startedAt: number, reason: string | null = null) => { + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + // The route resolves req.nodeId to the seeded default node, so the rows + // must carry that id, not a literal. + const defaultNodeId = db.getNodes().find(n => n.is_default)!.id; + db.insertHealthGateRun({ + id, node_id: defaultNodeId, stack_name: 'web', trigger_action: 'update', status, reason, + window_seconds: 90, containers_json: '[]', started_at: startedAt, ended_at: status === 'observing' ? null : startedAt + 1000, created_by: 'tester', + }); + }; + + it('requires authentication', async () => { + const res = await request(app).get('/api/stacks/web/health-gate'); + expect(res.status).toBe(401); + }); + + it('returns the never-run sentinel before any run exists', async () => { + const res = await request(app).get('/api/stacks/web/health-gate').set('Cookie', authCookie); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ stack: 'web', id: null, status: 'never-run' }); + }); + + it('returns the latest run without a gateId and a specific (superseded) run with one', async () => { + await seedRun('gate-old', 'unknown', 1_000, 'superseded by a newer update'); + await seedRun('gate-new', 'passed', 2_000); + + const latest = await request(app).get('/api/stacks/web/health-gate').set('Cookie', authCookie); + expect(latest.status).toBe(200); + expect(latest.body).toMatchObject({ id: 'gate-new', status: 'passed' }); + + const byId = await request(app).get('/api/stacks/web/health-gate?gateId=gate-old').set('Cookie', authCookie); + expect(byId.status).toBe(200); + expect(byId.body).toMatchObject({ id: 'gate-old', status: 'unknown', reason: 'superseded by a newer update' }); + }); + + it('returns never-run for an unknown gateId', async () => { + const res = await request(app).get('/api/stacks/web/health-gate?gateId=no-such-id').set('Cookie', authCookie); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ status: 'never-run' }); + }); +}); + +describe('GET /api/fleet/snapshots/coverage', () => { + it('requires authentication', async () => { + const res = await request(app).get('/api/fleet/snapshots/coverage?nodeId=0&stackName=web'); + expect(res.status).toBe(401); + }); + + it('hits the coverage handler, not the /snapshots/:id route', async () => { + const res = await request(app) + .get('/api/fleet/snapshots/coverage?nodeId=0&stackName=web') + .set('Cookie', authCookie); + // The /:id handler would have rejected "coverage" as a bad snapshot ID; + // the coverage handler returns the latestAt shape instead. + expect(res.status).toBe(200); + expect(res.body).toEqual({ latestAt: null }); + }); + + it('scopes coverage to the requested node and stack', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + const matching = db.createSnapshot('covers web on node 0', 'tester', 1, 1, '[]'); + db.insertSnapshotFiles(matching, [ + { nodeId: 0, nodeName: 'local', stackName: 'web', filename: 'compose.yaml', content: 'services: {}' }, + ]); + const decoy = db.createSnapshot('covers other things', 'tester', 2, 2, '[]'); + db.insertSnapshotFiles(decoy, [ + { nodeId: 1, nodeName: 'remote', stackName: 'web', filename: 'compose.yaml', content: 'services: {}' }, + { nodeId: 0, nodeName: 'local', stackName: 'other', filename: 'compose.yaml', content: 'services: {}' }, + ]); + const matchingCreatedAt = db.getSnapshots().find(s => s.id === matching)!.created_at; + + const res = await request(app) + .get('/api/fleet/snapshots/coverage?nodeId=0&stackName=web') + .set('Cookie', authCookie); + expect(res.status).toBe(200); + expect(res.body).toEqual({ latestAt: matchingCreatedAt }); + }); + + it('rejects non-admin users', async () => { + const res = await request(app) + .get('/api/fleet/snapshots/coverage?nodeId=0&stackName=web') + .set('Cookie', viewerCookie); + expect(res.status).toBe(403); + }); + + it('validates nodeId and stackName', async () => { + const missingNode = await request(app) + .get('/api/fleet/snapshots/coverage?stackName=web') + .set('Cookie', authCookie); + expect(missingNode.status).toBe(400); + + const badNode = await request(app) + .get('/api/fleet/snapshots/coverage?nodeId=-2&stackName=web') + .set('Cookie', authCookie); + expect(badNode.status).toBe(400); + + // A trailing-garbage nodeId must be rejected, not coerced by parseInt. + const garbageNode = await request(app) + .get('/api/fleet/snapshots/coverage?nodeId=1abc&stackName=web') + .set('Cookie', authCookie); + expect(garbageNode.status).toBe(400); + + const badStack = await request(app) + .get('/api/fleet/snapshots/coverage?nodeId=0&stackName=..%2Fetc') + .set('Cookie', authCookie); + expect(badStack.status).toBe(400); + }); +}); diff --git a/backend/src/__tests__/update-guard-service.test.ts b/backend/src/__tests__/update-guard-service.test.ts new file mode 100644 index 00000000..d70e6de5 --- /dev/null +++ b/backend/src/__tests__/update-guard-service.test.ts @@ -0,0 +1,155 @@ +/** + * Wiring tests for UpdateGuardService: container probing resilience and the + * degrade-everything-to-unknown contract when every collaborator fails. The + * grading rules themselves are covered by the pure readiness tests. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const { + mockListContainers, + mockGetContainer, + mockGetLatest, + mockGetPreview, + mockGetBackupInfo, + mockGetOpenDriftFindings, + mockGetGlobalSettings, + mockFsSize, +} = vi.hoisted(() => ({ + mockListContainers: vi.fn(), + mockGetContainer: vi.fn(), + mockGetLatest: vi.fn(), + mockGetPreview: vi.fn(), + mockGetBackupInfo: vi.fn(), + mockGetOpenDriftFindings: vi.fn(), + mockGetGlobalSettings: vi.fn(), + mockFsSize: vi.fn(), +})); + +vi.mock('../services/DockerController', () => ({ + default: { + getInstance: () => ({ + getDocker: () => ({ + listContainers: mockListContainers, + getContainer: mockGetContainer, + }), + }), + }, +})); + +vi.mock('../services/ComposeDoctorService', () => ({ + ComposeDoctorService: { getInstance: () => ({ getLatest: mockGetLatest }) }, +})); + +vi.mock('../services/UpdatePreviewService', () => ({ + UpdatePreviewService: { getInstance: () => ({ getPreview: mockGetPreview }) }, +})); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBackupInfo: mockGetBackupInfo, + getBackupEnvSummary: vi.fn().mockRejectedValue(new Error('not used here')), + envExists: vi.fn().mockRejectedValue(new Error('not used here')), + }), + }, +})); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getOpenDriftFindings: mockGetOpenDriftFindings, + getGlobalSettings: mockGetGlobalSettings, + getStackActivity: vi.fn().mockReturnValue([]), + }), + }, +})); + +vi.mock('systeminformation', () => ({ + default: { fsSize: mockFsSize }, +})); + +import { UpdateGuardService } from '../services/UpdateGuardService'; + +const inspectResult = (over: Record = {}) => ({ + State: { Status: 'running', ExitCode: 0 }, + Config: { Healthcheck: { Test: ['CMD', 'true'] } }, + HostConfig: { RestartPolicy: { Name: 'unless-stopped' } }, + Mounts: [], + ...over, +}); + +beforeEach(() => { + vi.clearAllMocks(); + mockGetGlobalSettings.mockReturnValue({ host_disk_limit: '90' }); +}); + +describe('UpdateGuardService.probeContainers', () => { + it('skips a container that vanished between list and inspect (404)', async () => { + mockListContainers.mockResolvedValue([ + { Id: 'aaa', Names: ['/app-web-1'], State: 'running' }, + { Id: 'bbb', Names: ['/app-db-1'], State: 'running' }, + ]); + mockGetContainer.mockImplementation((id: string) => ({ + inspect: id === 'bbb' + ? vi.fn().mockRejectedValue(Object.assign(new Error('no such container'), { statusCode: 404 })) + : vi.fn().mockResolvedValue(inspectResult()), + })); + + const probes = await UpdateGuardService.getInstance().probeContainers(0, 'app'); + expect(probes).toHaveLength(1); + expect(probes[0].name).toBe('app-web-1'); + }); + + it('propagates non-404 inspect failures so the whole signal degrades honestly', async () => { + mockListContainers.mockResolvedValue([ + { Id: 'aaa', Names: ['/app-web-1'], State: 'running' }, + ]); + mockGetContainer.mockReturnValue({ + inspect: vi.fn().mockRejectedValue(Object.assign(new Error('daemon hiccup'), { statusCode: 500 })), + }); + + await expect(UpdateGuardService.getInstance().probeContainers(0, 'app')).rejects.toThrow('daemon hiccup'); + }); +}); + +describe('UpdateGuardService.computeUpdateReadiness wiring', () => { + it('returns a complete unknown-verdict report when every collaborator fails', async () => { + mockGetLatest.mockImplementation(() => { throw new Error('db gone'); }); + mockGetOpenDriftFindings.mockImplementation(() => { throw new Error('db gone'); }); + mockListContainers.mockRejectedValue(new Error('docker gone')); + mockGetPreview.mockRejectedValue(new Error('registry gone')); + mockGetBackupInfo.mockRejectedValue(new Error('fs gone')); + mockFsSize.mockRejectedValue(new Error('si gone')); + + const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app'); + + expect(report.stack).toBe('app'); + expect(report.signals.map(s => s.id)).toEqual([ + 'preflight', 'drift', 'containers', 'healthchecks', 'update_preview', 'backup_slot', 'disk', + ]); + // The container probe failure is the verdict-affecting unknown. + expect(report.verdict).toBe('unknown'); + }); + + it('produces a ready verdict from healthy collaborator outputs', async () => { + mockGetLatest.mockReturnValue({ status: 'pass' }); + mockGetOpenDriftFindings.mockReturnValue([]); + mockListContainers.mockResolvedValue([{ Id: 'aaa', Names: ['/app-web-1'], State: 'running' }]); + mockGetContainer.mockReturnValue({ inspect: vi.fn().mockResolvedValue(inspectResult()) }); + mockGetPreview.mockResolvedValue({ + stack_name: 'app', + images: [], + summary: { + has_update: true, primary_image: 'nginx', current_tag: '1.27.0', next_tag: '1.27.1', + semver_bump: 'patch', update_kind: 'tag', blocked: false, blocked_reason: null, + }, + rollback_target: 'nginx:1.27.0', + changelog: null, + }); + mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: Date.now() }); + mockFsSize.mockResolvedValue([{ mount: '/', use: 42 }]); + + const report = await UpdateGuardService.getInstance().computeUpdateReadiness(0, 'app'); + expect(report.verdict).toBe('ready'); + }); +}); diff --git a/backend/src/__tests__/webhooks-trigger.test.ts b/backend/src/__tests__/webhooks-trigger.test.ts index 45d1a68d..a6b953cc 100644 --- a/backend/src/__tests__/webhooks-trigger.test.ts +++ b/backend/src/__tests__/webhooks-trigger.test.ts @@ -443,3 +443,49 @@ describe('webhook_executions.error redaction (M6)', () => { expect(history[0].error).toContain('/home/'); }); }); + +describe('WebhookService.execute: health gate begin call sites', () => { + // FileSystemService and ComposeService return a fresh instance per nodeId, + // so those spies attach to the prototypes (matching the redaction tests + // above); HealthGateService is a singleton spied directly. The policy gate + // is stubbed to allow so the test isolates the begin wiring, not policy. + it('begins a deploy gate after a webhook deploy succeeds', async () => { + const stack = 'hook-deploy-gate'; + const { id } = createWebhook({ action: 'deploy', stack }); + const webhook = DatabaseService.getInstance().getWebhook(id)!; + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + + const fs = await import('../services/FileSystemService'); + const compose = await import('../services/ComposeService'); + const policyGate = await import('../helpers/policyGate'); + const { HealthGateService } = await import('../services/HealthGateService'); + vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined); + vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]); + vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue(undefined); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-hook'); + + const result = await WebhookService.getInstance().execute(webhook, 'deploy', 'test', true); + expect(result.success).toBe(true); + expect(beginSpy).toHaveBeenCalledWith(nodeId, stack, 'deploy', 'system:webhook'); + }); + + it('begins an update gate after a webhook pull succeeds', async () => { + const stack = 'hook-pull-gate'; + const { id } = createWebhook({ action: 'pull', stack }); + const webhook = DatabaseService.getInstance().getWebhook(id)!; + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + + const fs = await import('../services/FileSystemService'); + const compose = await import('../services/ComposeService'); + const policyGate = await import('../helpers/policyGate'); + const { HealthGateService } = await import('../services/HealthGateService'); + vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined); + vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]); + vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue(undefined); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'begin').mockReturnValue('gate-hook'); + + const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true); + expect(result.success).toBe(true); + expect(beginSpy).toHaveBeenCalledWith(nodeId, stack, 'update', 'system:webhook'); + }); +}); diff --git a/backend/src/bootstrap/shutdown.ts b/backend/src/bootstrap/shutdown.ts index d7f2ebe1..09421d45 100644 --- a/backend/src/bootstrap/shutdown.ts +++ b/backend/src/bootstrap/shutdown.ts @@ -3,6 +3,7 @@ import { DatabaseService } from '../services/DatabaseService'; import { LicenseService } from '../services/LicenseService'; import { MonitorService } from '../services/MonitorService'; import { AutoHealService } from '../services/AutoHealService'; +import { HealthGateService } from '../services/HealthGateService'; import { FleetSyncRetryService } from '../services/FleetSyncRetryService'; import { DockerEventManager } from '../services/DockerEventManager'; import { ImageUpdateService } from '../services/ImageUpdateService'; @@ -31,6 +32,7 @@ export function installShutdownHandlers(server: Server): void { console.warn('[Shutdown] MonitorService cleanup failed:', (e as Error).message); } try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); } + try { HealthGateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] HealthGateService cleanup failed:', (e as Error).message); } try { FleetSyncRetryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] FleetSyncRetryService cleanup failed:', (e as Error).message); } try { DockerEventManager.getInstance().stop(); } catch (e) { console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message); diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index 3af518c3..a727dd40 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -9,6 +9,7 @@ import SelfUpdateService from '../services/SelfUpdateService'; import SelfIdentityService from '../services/SelfIdentityService'; import { MonitorService } from '../services/MonitorService'; import { AutoHealService } from '../services/AutoHealService'; +import { HealthGateService } from '../services/HealthGateService'; import { FleetSyncRetryService } from '../services/FleetSyncRetryService'; import { DockerEventManager } from '../services/DockerEventManager'; import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService'; @@ -118,6 +119,7 @@ export async function startServer(server: Server): Promise { // safely run alongside the async initializers below. MonitorService.getInstance().start(); AutoHealService.getInstance().start(); + HealthGateService.getInstance().start(); FleetSyncRetryService.getInstance().start(); ImageUpdateService.getInstance().start(); SchedulerService.getInstance().start(); diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts index f8f78f70..4cc97476 100644 --- a/backend/src/routes/fleet.ts +++ b/backend/src/routes/fleet.ts @@ -1754,6 +1754,33 @@ fleetRouter.get('/snapshots', authMiddleware, async (req: Request, res: Response } }); +// Registered before /snapshots/:id so "coverage" is never parsed as an id. +// Hub-local by design: snapshot rows exist only in the hub database, so the +// readiness UI fetches this with localOnly and merges it client-side. +fleetRouter.get('/snapshots/coverage', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + + try { + // Strict digits only: parseInt would accept '1abc' as 1. + const nodeIdRaw = typeof req.query.nodeId === 'string' ? req.query.nodeId : ''; + const stackName = req.query.stackName as string; + if (!/^\d+$/.test(nodeIdRaw)) { + res.status(400).json({ error: 'nodeId must be a non-negative integer' }); + return; + } + const nodeId = parseInt(nodeIdRaw, 10); + if (typeof stackName !== 'string' || !isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + const latestAt = DatabaseService.getInstance().getLatestSnapshotTimestampFor(nodeId, stackName); + res.json({ latestAt }); + } catch (error) { + console.error('[Fleet Snapshot] Coverage lookup error:', error); + res.status(500).json({ error: 'Failed to look up snapshot coverage' }); + } +}); + fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; diff --git a/backend/src/routes/imageUpdates.ts b/backend/src/routes/imageUpdates.ts index 2fc0ce44..4fc6e194 100644 --- a/backend/src/routes/imageUpdates.ts +++ b/backend/src/routes/imageUpdates.ts @@ -8,6 +8,7 @@ import { FileSystemService } from '../services/FileSystemService'; import { ComposeService } from '../services/ComposeService'; import { NotificationService } from '../services/NotificationService'; import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; +import { HealthGateService } from '../services/HealthGateService'; import { authMiddleware } from '../middleware/auth'; import { requireAdmin } from '../middleware/tierGates'; import { buildPolicyGateOptions } from '../helpers/policyGate'; @@ -297,6 +298,7 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp await compose.updateStack(stackName, undefined, atomic); db.clearStackUpdateStatus(req.nodeId, stackName); + HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`); NotificationService.getInstance().broadcastEvent({ type: 'state-invalidate', diff --git a/backend/src/routes/settings.ts b/backend/src/routes/settings.ts index e6b33dc3..3755a34a 100644 --- a/backend/src/routes/settings.ts +++ b/backend/src/routes/settings.ts @@ -27,6 +27,8 @@ const ALLOWED_SETTING_KEYS = new Set([ 'prune_on_update', 'reclaim_hero', 'snapshot_documentation', + 'health_gate_enabled', + 'health_gate_window_seconds', ]); // Keys whose write requires a paid license, not just an admin role. @@ -52,6 +54,8 @@ const SettingsPatchSchema = z.object({ prune_on_update: z.enum(['0', '1']), reclaim_hero: z.enum(['0', '1']), snapshot_documentation: z.enum(['0', '1']), + health_gate_enabled: z.enum(['0', '1']), + health_gate_window_seconds: z.coerce.number().int().min(15).max(600).transform(String), }).partial(); export const settingsRouter = Router(); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 38288c6b..1729ec24 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -16,6 +16,9 @@ import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService'; import { DriftLedgerService, type DriftTemporal } from '../services/DriftLedgerService'; import { ComposeDoctorService } from '../services/ComposeDoctorService'; +import { UpdateGuardService } from '../services/UpdateGuardService'; +import { HealthGateService } from '../services/HealthGateService'; +import { classifyFailure } from '../services/updateGuard/failureClassifier'; import { requirePermission, checkPermission } from '../middleware/permissions'; import { NotificationService, type NotificationCategory } from '../services/NotificationService'; import { StackOpLockService, type StackOpAction } from '../services/StackOpLockService'; @@ -351,6 +354,8 @@ interface BulkResultItem { ok: boolean; error?: string; code?: string; + /** Health gate run started for a successful update, when gating is enabled. */ + healthGateId?: string | null; } async function runStackBulkOp( @@ -413,6 +418,8 @@ async function runStackBulkOp( triggerPostDeployScan(stackName, req.nodeId).catch(err => console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err), ); + const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', req.user?.username ?? null); + return { stackName, ok: true, healthGateId }; } else { const outcome = await containerActionForStack(req.nodeId, stackName, action); if (outcome.kind === 'no-containers') { @@ -1120,6 +1127,55 @@ stacksRouter.post('/:stackName/preflight/run', async (req: Request, res: Respons } }); +// Update guard: readiness reports computed on demand from existing stores +// (preflight runs, drift findings, backup slot, update preview, live Docker +// state). Node-scoped like preflight: a remote stack is evaluated on the node +// that owns it. Read-only, so stack:read is the correct gate. +stacksRouter.get('/:stackName/update-readiness', async (req: Request, res: Response) => { + const stackName = req.params.stackName as string; + if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return; + if (!(await requireStackExists(req.nodeId, stackName, res))) return; + try { + const report = await UpdateGuardService.getInstance().computeUpdateReadiness(req.nodeId, stackName); + res.json(report); + } catch (error) { + console.error('[Stacks] Failed to compute update readiness for %s:', sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown'))); + res.status(500).json({ error: 'Failed to compute update readiness' }); + } +}); + +stacksRouter.get('/:stackName/rollback-readiness', async (req: Request, res: Response) => { + const stackName = req.params.stackName as string; + if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return; + if (!(await requireStackExists(req.nodeId, stackName, res))) return; + try { + const report = await UpdateGuardService.getInstance().computeRollbackReadiness(req.nodeId, stackName); + res.json(report); + } catch (error) { + console.error('[Stacks] Failed to compute rollback readiness for %s:', sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown'))); + res.status(500).json({ error: 'Failed to compute rollback readiness' }); + } +}); + +// Post-update health gate result. `gateId` returns that specific run so a +// superseded gate still resolves to its terminal state; without it, the +// latest run (or a never-run sentinel). +stacksRouter.get('/:stackName/health-gate', async (req: Request, res: Response) => { + const stackName = req.params.stackName as string; + if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return; + if (!(await requireStackExists(req.nodeId, stackName, res))) return; + try { + const gateId = typeof req.query.gateId === 'string' && req.query.gateId.trim() ? req.query.gateId : undefined; + res.json(HealthGateService.getInstance().getReport(req.nodeId, stackName, gateId)); + } catch (error) { + console.error('[Stacks] Failed to load health gate for %s:', sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown'))); + res.status(500).json({ error: 'Failed to load health gate' }); + } +}); + stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { const stackName = req.params.stackName as string; if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; @@ -1139,7 +1195,8 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { dlog(`[Stacks] Deploy completed: ${sanitizeForLog(stackName)}`); if (debug) console.debug(`[Stacks:debug] Deploy finished in ${Date.now() - t0}ms`); ok = true; - res.json({ message: 'Deployed successfully' }); + const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'deploy', req.user?.username ?? null); + res.json({ message: 'Deployed successfully', healthGateId }); notifyActionSuccess('deploy_success', `${stackName} deployed`, stackName, req.user?.username ?? 'system'); if (!skipScan) { triggerPostDeployScan(stackName, req.nodeId).catch(err => @@ -1156,12 +1213,14 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { console.warn('[Stacks] Deploy failed, rollback did not complete: %s', sanitizeForLog(stackName)); } const message = getErrorMessage(error, 'Failed to deploy stack'); + // ComposeRollbackError already carries the cause's message; see classifyFailure. + const failure = classifyFailure(message, { dockerUnavailable: isDockerUnavailableError(error) }); notifyActionFailure('deploy', stackName, error, req.user?.username ?? 'system'); if (!res.headersSent) { if (isDockerUnavailableError(error)) { - res.status(503).json({ error: message, code: 'docker_unavailable', rolledBack }); + res.status(503).json({ error: message, code: 'docker_unavailable', rolledBack, failure }); } else { - res.status(500).json({ error: message, rolledBack }); + res.status(500).json({ error: message, rolledBack, failure }); } } } finally { @@ -1408,7 +1467,8 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { dlog(`[Stacks] Update completed: ${sanitizeForLog(stackName)}`); if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`); ok = true; - res.json({ status: 'Update completed' }); + const healthGateId = HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', req.user?.username ?? null); + res.json({ status: 'Update completed', healthGateId }); notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system'); if (!skipScan) { triggerPostDeployScan(stackName, req.nodeId).catch(err => @@ -1426,10 +1486,13 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { } notifyActionFailure('update', stackName, error, req.user?.username ?? 'system'); if (!res.headersSent) { + const message = getErrorMessage(error, 'Failed to update'); + // ComposeRollbackError already carries the cause's message; see classifyFailure. + const failure = classifyFailure(message, { dockerUnavailable: isDockerUnavailableError(error) }); if (isDockerUnavailableError(error)) { - res.status(503).json({ error: getErrorMessage(error, 'Docker daemon is unreachable'), code: 'docker_unavailable', rolledBack }); + res.status(503).json({ error: message, code: 'docker_unavailable', rolledBack, failure }); } else { - res.status(500).json({ error: getErrorMessage(error, 'Failed to update'), rolledBack }); + res.status(500).json({ error: message, rolledBack, failure }); } } } finally { diff --git a/backend/src/services/CapabilityRegistry.ts b/backend/src/services/CapabilityRegistry.ts index b18bff29..29de6c97 100644 --- a/backend/src/services/CapabilityRegistry.ts +++ b/backend/src/services/CapabilityRegistry.ts @@ -36,6 +36,7 @@ export const CAPABILITIES = [ 'self-update', 'vulnerability-scanning', 'compose-doctor', + 'update-guard', ] as const; export type Capability = (typeof CAPABILITIES)[number]; diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 18c397e9..41825d24 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -106,6 +106,23 @@ export interface PreflightRunRow { created_by: string | null; } +/** One post-update health gate observation run. */ +export interface HealthGateRunRow { + id: string; + node_id: number; + stack_name: string; + /** Named trigger_action because TRIGGER is reserved in SQLite. */ + trigger_action: 'update' | 'deploy'; + status: 'observing' | 'passed' | 'failed' | 'unknown'; + reason: string | null; + window_seconds: number; + /** JSON array of per-container end states for display. */ + containers_json: string; + started_at: number; + ended_at: number | null; + created_by: string | null; +} + /** One finding within a stored preflight run. Never carries an environment value. */ export interface PreflightFindingRow { id: string; @@ -914,6 +931,7 @@ export class DatabaseService { ); CREATE INDEX IF NOT EXISTS idx_snapshot_files_snapshot ON fleet_snapshot_files(snapshot_id); + CREATE INDEX IF NOT EXISTS idx_snapshot_files_node_stack ON fleet_snapshot_files(node_id, stack_name); CREATE TABLE IF NOT EXISTS audit_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -1272,6 +1290,22 @@ export class DatabaseService { CREATE INDEX IF NOT EXISTS idx_preflight_findings_run ON preflight_findings(run_id); + CREATE TABLE IF NOT EXISTS health_gate_runs ( + id TEXT PRIMARY KEY, + node_id INTEGER NOT NULL, + stack_name TEXT NOT NULL, + trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy')), + status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')), + reason TEXT, + window_seconds INTEGER NOT NULL, + containers_json TEXT NOT NULL DEFAULT '[]', + started_at INTEGER NOT NULL, + ended_at INTEGER, + created_by TEXT + ); + CREATE INDEX IF NOT EXISTS idx_health_gate_runs_node_stack + ON health_gate_runs(node_id, stack_name, started_at); + CREATE TABLE IF NOT EXISTS secrets ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE, @@ -1400,6 +1434,8 @@ export class DatabaseService { stmt.run('mesh_auto_recreate', '0'); stmt.run('prune_on_update', '1'); stmt.run('reclaim_hero', '1'); + stmt.run('health_gate_enabled', '1'); + stmt.run('health_gate_window_seconds', '90'); // Seed the default local node if none exists const nodeCount = (this.db.prepare('SELECT COUNT(*) as count FROM nodes').get() as any)?.count || 0; @@ -2301,6 +2337,61 @@ export class DatabaseService { ).all(runId) as PreflightFindingRow[]; } + // --- Health Gate Runs --- + + public insertHealthGateRun(run: HealthGateRunRow): void { + this.db.prepare( + `INSERT INTO health_gate_runs + (id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, started_at, ended_at, created_by) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + run.id, run.node_id, run.stack_name, run.trigger_action, run.status, run.reason, + run.window_seconds, run.containers_json, run.started_at, run.ended_at, run.created_by, + ); + // Bounded history: keep only the 10 most recent runs per stack. + this.db.prepare( + `DELETE FROM health_gate_runs + WHERE node_id = ? AND stack_name = ? + AND id NOT IN ( + SELECT id FROM health_gate_runs + WHERE node_id = ? AND stack_name = ? + ORDER BY started_at DESC, id DESC LIMIT 10 + )` + ).run(run.node_id, run.stack_name, run.node_id, run.stack_name); + } + + public finalizeHealthGateRun( + id: string, + status: 'passed' | 'failed' | 'unknown', + reason: string | null, + endedAt: number, + containersJson: string, + ): void { + this.db.prepare( + 'UPDATE health_gate_runs SET status = ?, reason = ?, ended_at = ?, containers_json = ? WHERE id = ?' + ).run(status, reason, endedAt, containersJson, id); + } + + public getHealthGateRun(nodeId: number, stackName: string, id: string): HealthGateRunRow | undefined { + return this.db.prepare( + 'SELECT * FROM health_gate_runs WHERE node_id = ? AND stack_name = ? AND id = ?' + ).get(nodeId, stackName, id) as HealthGateRunRow | undefined; + } + + public getLatestHealthGateRun(nodeId: number, stackName: string): HealthGateRunRow | undefined { + return this.db.prepare( + 'SELECT * FROM health_gate_runs WHERE node_id = ? AND stack_name = ? ORDER BY started_at DESC, id DESC LIMIT 1' + ).get(nodeId, stackName) as HealthGateRunRow | undefined; + } + + /** Finalize runs left observing by a previous process (startup sweep). */ + public markInterruptedHealthGateRuns(reason: string, endedAt: number): number { + const result = this.db.prepare( + "UPDATE health_gate_runs SET status = 'unknown', reason = ?, ended_at = ? WHERE status = 'observing'" + ).run(reason, endedAt); + return result.changes; + } + // --- Notification History --- private mapNotificationRow(row: any): NotificationHistory { @@ -2641,6 +2732,7 @@ export class DatabaseService { this.db.prepare('DELETE FROM stack_drift_findings WHERE node_id = ?').run(id); this.db.prepare('DELETE FROM preflight_findings WHERE run_id IN (SELECT id FROM preflight_runs WHERE node_id = ?)').run(id); this.db.prepare('DELETE FROM preflight_runs WHERE node_id = ?').run(id); + this.db.prepare('DELETE FROM health_gate_runs WHERE node_id = ?').run(id); this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id); this.deleteRoleAssignmentsByResource('node', String(id)); this.db.prepare('DELETE FROM fleet_sync_status WHERE node_id = ?').run(id); @@ -3267,6 +3359,17 @@ export class DatabaseService { return rows.map(row => ({ ...row, content: crypto.decrypt(row.content) })); } + /** Created-at of the most recent fleet snapshot covering a stack, or null. */ + public getLatestSnapshotTimestampFor(nodeId: number, stackName: string): number | null { + const row = this.db.prepare( + `SELECT MAX(s.created_at) AS latest + FROM fleet_snapshots s + JOIN fleet_snapshot_files f ON f.snapshot_id = s.id + WHERE f.node_id = ? AND f.stack_name = ?` + ).get(nodeId, stackName) as { latest: number | null } | undefined; + return row?.latest ?? null; + } + public deleteSnapshot(id: number): void { this.db.prepare('DELETE FROM fleet_snapshots WHERE id = ?').run(id); } diff --git a/backend/src/services/FileSystemService.ts b/backend/src/services/FileSystemService.ts index ca36dc52..68c8eb5d 100644 --- a/backend/src/services/FileSystemService.ts +++ b/backend/src/services/FileSystemService.ts @@ -361,8 +361,17 @@ export class FileSystemService { async envExists(stackName: string): Promise { const stackDir = this.resolveStackDir(stackName); + // Canonical js/path-injection barrier inline with the access sink, the + // same pattern backupStackFiles/restoreStackFiles use: stackName is + // already validated by resolveStackDir above, but static analysis only + // credits the containment check when it sits at the sink itself. + const baseResolved = path.resolve(this.baseDir); + const target = path.resolve(stackDir, '.env'); + if (!target.startsWith(baseResolved + path.sep)) { + return false; + } try { - await fsPromises.access(path.join(stackDir, '.env')); + await fsPromises.access(target); return true; } catch { return false; @@ -920,6 +929,48 @@ export class FileSystemService { }; } + /** + * Names-only summary of the backup slot's env coverage for rollback + * readiness: whether a backup exists, whether it contains a .env, and the + * variable names defined in it. Values never leave this method. + */ + async getBackupEnvSummary(stackName: string): Promise<{ exists: boolean; envPresent: boolean; keys: string[] }> { + if (!isValidStackName(stackName)) { + return { exists: false, envPresent: false, keys: [] }; + } + // Canonical js/path-injection barrier inline with the read sink, mirroring + // backupStackFiles/restoreStackFiles. + const backupRoot = path.resolve(getBackupBaseDir()); + const backupDir = path.resolve(backupRoot, String(this.nodeId), stackName); + if (!backupDir.startsWith(backupRoot + path.sep)) { + throw Object.assign(new Error('Path escapes backup directory'), { code: 'INVALID_PATH' }); + } + try { + await fsPromises.access(backupDir); + } catch (e: unknown) { + // Only a missing slot may report "no backup"; an unreadable one (EACCES + // on a root-created dir) must propagate so callers degrade to unknown + // instead of falsely promising the next update will create one. + if ((e as NodeJS.ErrnoException)?.code !== 'ENOENT') throw e; + return { exists: false, envPresent: false, keys: [] }; + } + try { + const content = await fsPromises.readFile(path.join(backupDir, '.env'), 'utf-8'); + const keys: string[] = []; + for (const line of content.split(/\r?\n/)) { + const match = /^\s*([A-Za-z_][A-Za-z0-9_]*)=/.exec(line); + if (match) keys.push(match[1]); + } + return { exists: true, envPresent: true, keys }; + } catch (e: unknown) { + // ENOENT means the backup genuinely has no env file. Anything else + // (EACCES, EISDIR) must propagate: reporting it as "no env in backup" + // would falsely claim a rollback cannot restore env changes. + if ((e as NodeJS.ErrnoException)?.code !== 'ENOENT') throw e; + return { exists: true, envPresent: false, keys: [] }; + } + } + async getBackupInfo(stackName: string): Promise<{ exists: boolean; timestamp: number | null }> { const backupDir = this.getBackupDir(stackName); try { diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index 85504aa8..25577ea6 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -8,6 +8,7 @@ import { CryptoService } from './CryptoService'; import { DatabaseService, type StackGitSource, type GitSourceAuthType } from './DatabaseService'; import { FileSystemService } from './FileSystemService'; import { ComposeService } from './ComposeService'; +import { HealthGateService } from './HealthGateService'; import { NodeRegistry } from './NodeRegistry'; import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate'; import { isDebugEnabled } from '../utils/debug'; @@ -1062,6 +1063,7 @@ export class GitSourceService { }), ); await ComposeService.getInstance().deployStack(stackName); + HealthGateService.getInstance().begin(nodeId, stackName, 'deploy', 'system:git-source'); console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`); return { applied: true, deployed: true }; } catch (e) { diff --git a/backend/src/services/HealthGateService.ts b/backend/src/services/HealthGateService.ts new file mode 100644 index 00000000..c085bcfb --- /dev/null +++ b/backend/src/services/HealthGateService.ts @@ -0,0 +1,472 @@ +import { randomUUID } from 'crypto'; +import DockerController from './DockerController'; +import { DatabaseService, type HealthGateRunRow } from './DatabaseService'; +import { sanitizeForLog } from '../utils/safeLog'; +import { getErrorMessage } from '../utils/errors'; +import { withTimeout } from '../utils/withTimeout'; +import type { HealthGateContainer, HealthGateReport } from './updateGuard/types'; + +const POLL_INTERVAL_MS = 5_000; +// Per-observe ceiling so a hung Docker socket cannot leave a poll pending +// forever. Above POLL_INTERVAL_MS so a slow-but-live probe is not cut short; a +// timeout counts as a poll error and three in a row resolve the gate unknown. +const OBSERVE_TIMEOUT_MS = 8_000; +// A stack whose containers never appear gives up after this long. +const EMPTY_GRACE_MS = 15_000; +const DEFAULT_WINDOW_SECONDS = 90; +const MIN_WINDOW_SECONDS = 15; +const MAX_WINDOW_SECONDS = 600; +// Backstop against runaway concurrency (a burst of webhook or scheduled +// updates). Gates past the cap finalize immediately as unknown. +const MAX_CONCURRENT_GATES = 25; + +interface ObservedContainer { + id: string; + name: string; + startedAt: string | null; + /** Docker's raw RestartCount at observation time. */ + restartCount: number; + /** Gate-maintained restart tally since the baseline; 0 on a fresh snapshot, + * set by poll()'s accounting pass, at most one increment per poll. */ + restarts: number; + state: string; + health: string | null; +} + +interface ActiveGate { + runId: string; + nodeId: number; + stackName: string; + windowSeconds: number; + startedAt: number; + /** Self-scheduling poll timer, armed only between settled poll cycles. */ + timer: ReturnType | null; + /** Expected container set, keyed by name; null until the first non-empty poll. */ + expected: Map | null; + consecutivePollErrors: number; + /** Names missing on the previous poll (two consecutive misses fail the gate). */ + missingLastPoll: Set; + /** Names in `restarting` state on the previous poll. */ + restartingLastPoll: Set; + /** + * Set by finalize so a poll that was mid-await when this gate was superseded + * or stopped can never overwrite the terminal verdict with a stale one. + */ + finalized: boolean; +} + +/** + * Post-update health gate: after a deploy/update succeeds, observe the + * stack's containers for a configurable window and record a passed / failed / + * unknown verdict plus an activity timeline event. Purely observational: it + * never restarts, heals, or rolls anything back. AutoHeal needs no special + * handling: the unhealthy or exited state that triggers it is seen by the + * gate's own polls and fails the gate, and repeated restarts trip the + * restart-loop check. + * + * begin() is the single shared post-success hook for every gated deploy and + * update path; excluded paths (rollback, installs, reconciler loops) simply + * never call it. + */ +export class HealthGateService { + private static instance: HealthGateService; + private readonly active = new Map(); + private started = false; + + public static getInstance(): HealthGateService { + if (!HealthGateService.instance) { + HealthGateService.instance = new HealthGateService(); + } + return HealthGateService.instance; + } + + /** Sweep runs left observing by a previous process, then accept begin() calls. */ + public start(): void { + this.started = true; + try { + const swept = DatabaseService.getInstance().markInterruptedHealthGateRuns( + 'Sencho restarted during observation', Date.now(), + ); + if (swept > 0) { + console.log(`[HealthGate] Marked ${swept} interrupted observation(s) as unknown`); + } + } catch (error) { + console.error('[HealthGate] Startup sweep failed:', getErrorMessage(error, 'unknown')); + } + } + + /** Clear every poll timer and finalize in-flight gates as unknown. */ + public stop(): void { + this.started = false; + for (const gate of [...this.active.values()]) { + this.finalize(gate, 'unknown', 'shutdown during observation', []); + } + } + + /** + * Begin observing a stack after a successful deploy/update. Returns the gate + * run id for response correlation, or null when gating is disabled, the + * service is not started, or recording fails internally. Inserts the row + * synchronously so the caller can include the id in its response; + * observation then runs on a timer. Never throws. + * + * Also records the `update_started` activity event for update triggers, so + * every gated update path gets the timeline marker even when the gate + * itself is disabled. + */ + public begin( + nodeId: number, + stackName: string, + trigger: 'update' | 'deploy', + actor: string | null, + ): string | null { + // Refuses work outside the start()/stop() lifecycle so a late call during + // shutdown cannot leave a dangling poll timer. + if (!this.started) return null; + try { + const db = DatabaseService.getInstance(); + const settings = this.readSettings(); + + if (trigger === 'update') { + this.recordActivity(nodeId, stackName, 'info', 'update_started', `${stackName} update started`, actor); + } + if (!settings.enabled) return null; + + // A newer operation supersedes an in-flight gate for the same stack. + const key = `${nodeId}:${stackName}`; + const existing = this.active.get(key); + if (existing) { + this.finalize(existing, 'unknown', 'superseded by a newer update', []); + } + + const runId = randomUUID(); + const startedAt = Date.now(); + const row: HealthGateRunRow = { + id: runId, + node_id: nodeId, + stack_name: stackName, + trigger_action: trigger, + status: 'observing', + reason: null, + window_seconds: settings.windowSeconds, + containers_json: '[]', + started_at: startedAt, + ended_at: null, + created_by: actor, + }; + + if (this.active.size >= MAX_CONCURRENT_GATES) { + db.insertHealthGateRun({ ...row, status: 'unknown', reason: 'too many concurrent observations', ended_at: startedAt }); + return runId; + } + + db.insertHealthGateRun(row); + const gate: ActiveGate = { + runId, + nodeId, + stackName, + windowSeconds: settings.windowSeconds, + startedAt, + timer: null, + expected: null, + consecutivePollErrors: 0, + missingLastPoll: new Set(), + restartingLastPoll: new Set(), + finalized: false, + }; + this.active.set(key, gate); + this.scheduleNextPoll(gate); + return runId; + } catch (error) { + // The gate is an observer; its failure must never fail the operation. + console.error( + '[HealthGate] begin (%s) failed for %s on node %d:', + trigger, sanitizeForLog(stackName), nodeId, error, + ); + return null; + } + } + + /** A specific run by id, the latest run, or the never-run sentinel. */ + public getReport(nodeId: number, stackName: string, gateId?: string): HealthGateReport { + const db = DatabaseService.getInstance(); + const row = gateId + ? db.getHealthGateRun(nodeId, stackName, gateId) + : db.getLatestHealthGateRun(nodeId, stackName); + if (!row) { + return { + stack: stackName, id: null, status: 'never-run', trigger: null, reason: null, + windowSeconds: null, startedAt: null, endedAt: null, containers: [], + }; + } + let containers: HealthGateContainer[] = []; + try { + const parsed: unknown = JSON.parse(row.containers_json); + if (Array.isArray(parsed)) containers = parsed as HealthGateContainer[]; + } catch { + // A corrupt blob only loses the per-container detail, never the verdict. + console.warn('[HealthGate] Unreadable containers_json for run %s', sanitizeForLog(row.id)); + } + return { + stack: stackName, + id: row.id, + status: row.status, + trigger: row.trigger_action, + reason: row.reason, + windowSeconds: row.window_seconds, + startedAt: row.started_at, + endedAt: row.ended_at, + containers, + }; + } + + /** + * Orchestrate one poll cycle and arm the next. Polls are single-flight: the + * next timer is scheduled only after this cycle fully settles (the finally + * below), so a slow or timed-out observe can never overlap the following poll + * or corrupt the restart/missing accounting that assumes one poll at a time. + */ + private async poll(gate: ActiveGate): Promise { + const key = `${gate.nodeId}:${gate.stackName}`; + // A late timer fire after supersede, stop, or finalize must do nothing. + if (gate.finalized || this.active.get(key) !== gate) return; + try { + await this.runPollCycle(gate, key); + } finally { + if (!gate.finalized && this.active.get(key) === gate) { + this.scheduleNextPoll(gate); + } + } + } + + /** Arm the next poll. No-op once the gate is finalized. */ + private scheduleNextPoll(gate: ActiveGate): void { + if (gate.finalized) return; + gate.timer = setTimeout(() => { void this.poll(gate); }, POLL_INTERVAL_MS); + } + + private async runPollCycle(gate: ActiveGate, key: string): Promise { + let observed: ObservedContainer[]; + try { + observed = await withTimeout( + this.observeContainers(gate), OBSERVE_TIMEOUT_MS, 'health gate observe', + ); + } catch (error) { + gate.consecutivePollErrors += 1; + console.warn( + '[HealthGate] poll error %d for %s:', + gate.consecutivePollErrors, sanitizeForLog(gate.stackName), getErrorMessage(error, 'unknown'), + ); + if (gate.consecutivePollErrors >= 3) { + this.finalize(gate, 'unknown', 'Docker became unreachable during observation', []); + } + return; + } + // The await above can straddle a supersede or stop; never act on a gate + // that was finalized mid-flight. + if (gate.finalized || this.active.get(key) !== gate) return; + gate.consecutivePollErrors = 0; + + const elapsedMs = Date.now() - gate.startedAt; + + if (gate.expected === null) { + if (observed.length > 0) { + gate.expected = new Map(observed.map(c => [c.name, c])); + } else if (elapsedMs >= EMPTY_GRACE_MS) { + this.finalize(gate, 'unknown', 'no containers found to observe', []); + } + return; + } + + const byName = new Map(observed.map(c => [c.name, c])); + + // First pass: restart accounting for every expected container still + // present, so the summary below reflects the tallies the checks act on. A + // restart counts when the container was replaced (new id), relaunched + // (StartedAt moved), or Docker bumped its RestartCount; at most one + // restart is tallied per poll regardless of how many occurred in the gap. + for (const [name, baseline] of gate.expected) { + const current = byName.get(name); + if (!current) continue; + const restarted = + current.id !== baseline.id || + current.restartCount > baseline.restartCount || + (current.startedAt !== null && baseline.startedAt !== null && current.startedAt !== baseline.startedAt); + current.restarts = baseline.restarts + (restarted ? 1 : 0); + } + const summary = this.summarize(gate.expected, byName); + + // Second pass: fail fast on a clearly bad state. + for (const [name, baseline] of gate.expected) { + const current = byName.get(name); + if (!current) { + if (gate.missingLastPoll.has(name)) { + this.finalize(gate, 'failed', `container ${name} disappeared during observation`, summary); + return; + } + gate.missingLastPoll.add(name); + continue; + } + gate.missingLastPoll.delete(name); + + if (current.state === 'exited' && baseline.restarts === current.restarts) { + // An exit with no restart attempt is terminal for the window. + this.finalize(gate, 'failed', `container ${name} exited during observation`, summary); + return; + } + if (current.health === 'unhealthy') { + this.finalize(gate, 'failed', `container ${name} reported unhealthy`, summary); + return; + } + if (current.restarts >= 2) { + this.finalize(gate, 'failed', `container ${name} is restart looping`, summary); + return; + } + if (current.state === 'restarting') { + if (gate.restartingLastPoll.has(name)) { + this.finalize(gate, 'failed', `container ${name} is stuck restarting`, summary); + return; + } + gate.restartingLastPoll.add(name); + } else { + gate.restartingLastPoll.delete(name); + } + // Carry the running restart tally forward as the new baseline. + gate.expected.set(name, current); + } + + if (elapsedMs < gate.windowSeconds * 1000) return; + + // Window complete: pass requires everything running and healthy wherever a + // healthcheck exists. A health state still 'starting' is not a pass. + const stillStarting = observed.filter(c => c.health === 'starting'); + if (stillStarting.length > 0) { + this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary); + return; + } + const notRunning = [...gate.expected.keys()].filter(name => byName.get(name)?.state !== 'running'); + if (notRunning.length > 0) { + this.finalize(gate, 'failed', `not running at the end of the window: ${notRunning.join(', ')}`, summary); + return; + } + this.finalize(gate, 'passed', null, summary); + } + + private async observeContainers(gate: ActiveGate): Promise { + const docker = DockerController.getInstance(gate.nodeId).getDocker(); + const listed = await docker.listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${gate.stackName}`] }, + }); + const observed = await Promise.all( + listed.map(async (info): Promise => { + try { + const inspect = await docker.getContainer(info.Id).inspect(); + return { + id: info.Id, + name: info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12), + startedAt: inspect.State?.StartedAt ?? null, + restartCount: typeof inspect.RestartCount === 'number' ? inspect.RestartCount : 0, + restarts: 0, + state: inspect.State?.Status ?? info.State ?? 'unknown', + health: inspect.State?.Health?.Status ?? null, + }; + } catch (e: unknown) { + // Removed between list and inspect; the missing-container logic will + // see its absence on this or the next poll. + if ((e as { statusCode?: number })?.statusCode === 404) return null; + throw e; + } + }), + ); + return observed.filter((c): c is ObservedContainer => c !== null); + } + + private summarize( + expected: Map, + current: Map, + ): HealthGateContainer[] { + return [...expected.values()].map(baseline => { + const now = current.get(baseline.name); + return { + name: baseline.name, + state: now?.state ?? 'missing', + health: now?.health ?? null, + restarts: now?.restarts ?? baseline.restarts, + }; + }); + } + + private finalize( + gate: ActiveGate, + status: 'passed' | 'failed' | 'unknown', + reason: string | null, + containers: HealthGateContainer[], + ): void { + if (gate.finalized) return; + gate.finalized = true; + if (gate.timer) clearTimeout(gate.timer); + const key = `${gate.nodeId}:${gate.stackName}`; + if (this.active.get(key) === gate) this.active.delete(key); + + try { + DatabaseService.getInstance().finalizeHealthGateRun( + gate.runId, status, reason, Date.now(), JSON.stringify(containers), + ); + } catch (error) { + // The verdict is lost from the DB (the startup sweep will later rewrite + // the row as unknown), so log everything needed to reconstruct it. + console.error( + '[HealthGate] Failed to persist verdict %s (%s) for run %s, stack %s:', + status, sanitizeForLog(reason ?? 'no reason'), gate.runId, sanitizeForLog(gate.stackName), + getErrorMessage(error, 'unknown'), + ); + } + + if (status === 'passed') { + this.recordActivity(gate.nodeId, gate.stackName, 'info', 'health_gate_passed', + `${gate.stackName} health gate passed after ${gate.windowSeconds}s`, 'system'); + } else if (status === 'failed') { + this.recordActivity(gate.nodeId, gate.stackName, 'warning', 'health_gate_failed', + `${gate.stackName} health gate failed: ${reason ?? 'unknown reason'}`, 'system'); + } + } + + private recordActivity( + nodeId: number, + stackName: string, + level: 'info' | 'warning', + category: 'update_started' | 'health_gate_passed' | 'health_gate_failed', + message: string, + actor: string | null, + ): void { + try { + DatabaseService.getInstance().addNotificationHistory(nodeId, { + level, + category, + message, + timestamp: Date.now(), + stack_name: stackName, + actor_username: actor, + }); + } catch (error) { + console.warn('[HealthGate] Failed to record activity for %s:', sanitizeForLog(stackName), getErrorMessage(error, 'unknown')); + } + } + + private readSettings(): { enabled: boolean; windowSeconds: number } { + try { + const settings = DatabaseService.getInstance().getGlobalSettings(); + const windowRaw = parseInt(settings['health_gate_window_seconds'] ?? '', 10); + const windowSeconds = Number.isFinite(windowRaw) + ? Math.min(MAX_WINDOW_SECONDS, Math.max(MIN_WINDOW_SECONDS, windowRaw)) + : DEFAULT_WINDOW_SECONDS; + return { enabled: settings['health_gate_enabled'] !== '0', windowSeconds }; + } catch (error) { + // Safe default: observing is non-destructive, so a settings read + // failure keeps the gate on with the default window. + console.warn('[HealthGate] Settings read failed; using defaults:', getErrorMessage(error, 'unknown')); + return { enabled: true, windowSeconds: DEFAULT_WINDOW_SECONDS }; + } + } +} diff --git a/backend/src/services/NotificationService.ts b/backend/src/services/NotificationService.ts index 9e509020..b5771b50 100644 --- a/backend/src/services/NotificationService.ts +++ b/backend/src/services/NotificationService.ts @@ -27,6 +27,11 @@ export type NotificationCategory = // from ALL_NOTIFICATION_CATEGORIES (the routable-category whitelist) below. | 'drift_detected' | 'drift_resolved' + // Update lifecycle markers from the post-update health gate. History-only + // for the same reason as the drift pair above. + | 'update_started' + | 'health_gate_passed' + | 'health_gate_failed' | 'system'; export const ALL_NOTIFICATION_CATEGORIES: readonly NotificationCategory[] = [ diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index d2cf29cf..e1247b4e 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -6,6 +6,7 @@ import { PROXY_TIER_HEADER } from './license-headers'; import DockerController from './DockerController'; import { ComposeService } from './ComposeService'; import { FileSystemService } from './FileSystemService'; +import { HealthGateService } from './HealthGateService'; import { ImageUpdateService } from './ImageUpdateService'; import type { ImageCheckResult } from './ImageUpdateService'; import { isDebugEnabled } from '../utils/debug'; @@ -861,6 +862,7 @@ export class SchedulerService { const atomic = true; await compose.updateStack(stackName, undefined, atomic); db.clearStackUpdateStatus(nodeId, stackName); + HealthGateService.getInstance().begin(nodeId, stackName, 'update', 'system:scheduler'); this.safeDispatch( 'info', diff --git a/backend/src/services/UpdateGuardService.ts b/backend/src/services/UpdateGuardService.ts new file mode 100644 index 00000000..ee62db37 --- /dev/null +++ b/backend/src/services/UpdateGuardService.ts @@ -0,0 +1,175 @@ +import si from 'systeminformation'; +import DockerController from './DockerController'; +import { DatabaseService } from './DatabaseService'; +import { FileSystemService } from './FileSystemService'; +import { ComposeDoctorService } from './ComposeDoctorService'; +import { UpdatePreviewService } from './UpdatePreviewService'; +import { withTimeout } from '../utils/withTimeout'; +import { getErrorMessage } from '../utils/errors'; +import { sanitizeForLog } from '../utils/safeLog'; +import { + aggregateRollbackOverall, + aggregateVerdict, + backupSlotSignal, + buildRollbackItems, + containersSignal, + diskSignal, + driftSignal, + healthchecksSignal, + preflightSignal, + updatePreviewSignal, + type Errored, +} from './updateGuard/readiness'; +import type { ContainerProbe, RollbackReadinessReport, UpdateReadinessReport } from './updateGuard/types'; + +// Bound on the network-and-socket-backed inputs (container probe, update +// preview, disk stats) so a hung registry or Docker socket cannot stall the +// report past the dialog's own fetch timeout; the remaining inputs are local +// DB/file reads. A timed-out input degrades to its 'unknown' signal instead +// of failing the report. +const INPUT_TIMEOUT_MS = 3_000; + +/** + * Computes update readiness and rollback readiness for a stack, on demand, + * from existing per-feature stores (preflight runs, drift findings, the atomic + * backup slot, the update preview, live Docker state). Derived data only; + * nothing here is persisted. + */ +export class UpdateGuardService { + private static instance: UpdateGuardService; + + public static getInstance(): UpdateGuardService { + if (!UpdateGuardService.instance) { + UpdateGuardService.instance = new UpdateGuardService(); + } + return UpdateGuardService.instance; + } + + /** + * Probe the stack's containers via the compose project label, normalized for + * the pure scoring functions. Throws on Docker errors; callers map that to + * the 'error' sentinel. + */ + async probeContainers(nodeId: number, stackName: string): Promise { + const docker = DockerController.getInstance(nodeId).getDocker(); + const listed = await docker.listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${stackName}`] }, + }); + const probes = await Promise.all( + listed.map(async (info): Promise => { + const name = info.Names?.[0]?.replace(/^\//, '') ?? info.Id.slice(0, 12); + let inspect: Awaited['inspect']>>; + try { + inspect = await docker.getContainer(info.Id).inspect(); + } catch (e: unknown) { + // A container removed between list and inspect (auto-heal or update + // churn) should not collapse the whole probe; skip just that one. + if ((e as { statusCode?: number })?.statusCode === 404) return null; + throw e; + } + const mounts = (inspect.Mounts ?? []).map(m => + m.Type === 'volume' ? `volume ${m.Name ?? 'unnamed'}` : `${m.Type} ${m.Source ?? ''}`.trim(), + ); + return { + name, + state: inspect.State?.Status ?? info.State ?? 'unknown', + health: inspect.State?.Health?.Status ?? null, + exitCode: typeof inspect.State?.ExitCode === 'number' ? inspect.State.ExitCode : null, + hasHealthcheck: !!inspect.Config?.Healthcheck?.Test?.length, + restartPolicy: inspect.HostConfig?.RestartPolicy?.Name || null, + mounts, + }; + }), + ); + return probes.filter((p): p is ContainerProbe => p !== null); + } + + async computeUpdateReadiness(nodeId: number, stackName: string): Promise { + const db = DatabaseService.getInstance(); + const now = Date.now(); + + const [preflight, drift, containers, preview, backup, disk] = await Promise.all([ + this.collect('preflight', stackName, async () => ComposeDoctorService.getInstance().getLatest(nodeId, stackName)), + this.collect('drift', stackName, async () => db.getOpenDriftFindings(nodeId, stackName).length), + this.collect('containers', stackName, () => + withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness container probe')), + this.collect('update preview', stackName, () => + withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview')), + this.collect('backup info', stackName, () => FileSystemService.getInstance(nodeId).getBackupInfo(stackName)), + this.collect('disk', stackName, () => this.readDiskUsage()), + ]); + + const settings = db.getGlobalSettings(); + const limitPercent = parseInt(settings['host_disk_limit'] ?? '90', 10) || 90; + + const signals = [ + preflightSignal(preflight), + driftSignal(drift), + containersSignal(containers), + healthchecksSignal(containers), + updatePreviewSignal(preview === 'error' ? 'error' : preview.summary), + backupSlotSignal(backup, now), + diskSignal(typeof disk === 'number' ? { usePercent: disk, limitPercent } : 'error'), + ]; + + return { stack: stackName, computedAt: now, verdict: aggregateVerdict(signals), signals }; + } + + async computeRollbackReadiness(nodeId: number, stackName: string): Promise { + const db = DatabaseService.getInstance(); + const fsSvc = FileSystemService.getInstance(nodeId); + const now = Date.now(); + + const [backup, envSummary, stackHasEnv, preview, lastDeployAt, containers] = await Promise.all([ + this.collect('backup info', stackName, () => fsSvc.getBackupInfo(stackName)), + this.collect('backup env summary', stackName, () => fsSvc.getBackupEnvSummary(stackName)), + this.collect('stack env presence', stackName, () => fsSvc.envExists(stackName)), + this.collect('update preview', stackName, () => + withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness update preview')), + this.collect('activity history', stackName, async () => { + const events = db.getStackActivity(nodeId, stackName, { limit: 50 }); + // A successful update is as good a known-good marker as a deploy. + return events.find(e => e.category === 'deploy_success' || e.category === 'image_update_applied')?.timestamp ?? null; + }), + this.collect('containers', stackName, () => + withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness container probe')), + ]); + + const items = buildRollbackItems({ + backup, + envSummary, + stackHasEnv, + rollbackTarget: preview === 'error' ? 'error' : { target: preview.rollback_target }, + lastDeployAt, + containers, + }, now); + + return { stack: stackName, computedAt: now, overall: aggregateRollbackOverall(items), items }; + } + + /** Host disk use percent for the main filesystem, or null when unavailable. */ + private async readDiskUsage(): Promise { + const fsSize = await withTimeout(si.fsSize(), INPUT_TIMEOUT_MS, 'readiness disk stats'); + const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0]; + if (typeof mainDisk?.use !== 'number') { + console.warn('[UpdateGuard] disk stats returned no usable mount; disk signal degrades to unknown'); + return null; + } + return mainDisk.use; + } + + /** Run one input collector; a failure degrades to the 'error' sentinel. */ + private async collect(label: string, stackName: string, fn: () => Promise): Promise { + try { + return await fn(); + } catch (error) { + console.warn( + '[UpdateGuard] %s unavailable for %s:', + label, sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return 'error'; + } + } +} diff --git a/backend/src/services/WebhookService.ts b/backend/src/services/WebhookService.ts index 078112cc..cc71a811 100644 --- a/backend/src/services/WebhookService.ts +++ b/backend/src/services/WebhookService.ts @@ -3,6 +3,7 @@ import { ComposeService } from './ComposeService'; import { DatabaseService, type Webhook } from './DatabaseService'; import { FileSystemService } from './FileSystemService'; import { GitSourceService } from './GitSourceService'; +import { HealthGateService } from './HealthGateService'; import { LicenseService } from './LicenseService'; import { PROXY_TIER_HEADER } from './license-headers'; import { NodeRegistry } from './NodeRegistry'; @@ -133,6 +134,7 @@ export class WebhookService { buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }), ); await compose.deployStack(stackName, undefined, atomic); + HealthGateService.getInstance().begin(nodeId, stackName, 'deploy', 'system:webhook'); break; case 'restart': await compose.runCommand(stackName, 'restart'); @@ -150,6 +152,7 @@ export class WebhookService { buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }), ); await compose.updateStack(stackName, undefined, atomic); + HealthGateService.getInstance().begin(nodeId, stackName, 'update', 'system:webhook'); break; case 'git-pull': return this.executeLocalGitPull(webhookId, stackName, action, triggerSource, startTime); diff --git a/backend/src/services/updateGuard/failureClassifier.ts b/backend/src/services/updateGuard/failureClassifier.ts new file mode 100644 index 00000000..9ccaefe9 --- /dev/null +++ b/backend/src/services/updateGuard/failureClassifier.ts @@ -0,0 +1,125 @@ +import type { FailureClassification } from './types'; + +interface ClassifierRule extends FailureClassification { + pattern: RegExp; +} + +const DOCKER_UNREACHABLE: FailureClassification = { + reason: 'node_unreachable', + label: 'Docker unreachable', + suggestion: 'Check that Docker is running and reachable on this node, then retry.', +}; + +/** + * Maps the redacted error text a failed deploy/update throws (the accumulated + * compose stdout/stderr from ComposeService.execute, or a sentinel like + * CONTAINER_CRASHED) onto an operator-facing cause and next step. + * + * First match wins, so ordering is load-bearing: + * - the CONTAINER_CRASHED and stall sentinels are exact, so they go first; + * - env_missing precedes compose_render_failed because a render failure caused + * by a missing variable should classify as the actionable cause; + * - healthcheck_failed precedes dependency_unavailable because compose phrases + * an unhealthy dependency as "dependency failed to start: ... is unhealthy". + * + * The app-store install route has its own message-prettifier (utils/ErrorParser) + * with a different output shape; installs are out of scope here. + */ +const RULES: ClassifierRule[] = [ + { + reason: 'container_exited', + label: 'Container exited after start', + suggestion: 'Check the container logs for the exit cause; roll back if the previous version was healthy.', + pattern: /CONTAINER_CRASHED/, + }, + { + // The idle-stall backstop terminated the step; the real cause is unknown. + reason: 'unknown', + label: 'Operation stalled', + suggestion: 'The operation stopped producing output and was terminated. Check Docker activity on the node, then retry.', + pattern: /STACK_STALLED_OUTPUT/, + }, + { + ...DOCKER_UNREACHABLE, + pattern: /cannot connect to the docker daemon|docker daemon is not running|error during connect|docker daemon is unreachable/i, + }, + { + reason: 'env_missing', + label: 'Missing environment variable', + suggestion: 'Define the missing variable in the stack environment file, then retry.', + pattern: /required variable\s+\S+ is missing|variable is not set|invalid interpolation format|env file .+ not found|couldn't find env file/i, + }, + { + reason: 'image_pull_failed', + label: 'Image pull failed', + suggestion: 'Check the image name and tag, registry credentials, and registry rate limits, then retry.', + pattern: /pull access denied|manifest unknown|manifest for .+ not found|toomanyrequests|failed to resolve reference|no matching manifest|error pulling image|repository does not exist|unauthorized: authentication required/i, + }, + { + reason: 'port_conflict', + label: 'Host port conflict', + suggestion: 'Free the conflicting host port or change the published port, then retry.', + pattern: /port is already allocated|bind: address already in use|ports are not available|failed to bind host port/i, + }, + { + reason: 'bind_path_missing', + label: 'Bind mount path missing', + suggestion: 'Create the missing host path or correct the bind mount source, then retry.', + pattern: /bind source path does not exist|mounts denied|invalid mount config/i, + }, + { + reason: 'permission_denied', + label: 'Permission denied', + suggestion: 'Check file and Docker socket permissions for the affected path, then retry.', + pattern: /permission denied|EACCES|operation not permitted/i, + }, + { + reason: 'healthcheck_failed', + label: 'Healthcheck failed', + suggestion: 'Check the failing service logs and its healthcheck command; roll back if the previous version was healthy.', + pattern: /is unhealthy/i, + }, + { + reason: 'dependency_unavailable', + label: 'Dependency unavailable', + suggestion: 'Start or create the missing dependency (service, external network, or volume) first, then retry.', + pattern: /dependency failed to start|depends on undefined service|declared as external, but could not be found/i, + }, + { + reason: 'compose_render_failed', + label: 'Compose file invalid', + suggestion: 'Review the compose file syntax (Compose Doctor can pinpoint the issue), then retry.', + pattern: /yaml:|mapping values are not allowed|cannot unmarshal|additional propert|undefined volume|undefined network|invalid compose/i, + }, +]; + +const UNKNOWN_FAILURE: FailureClassification = { + reason: 'unknown', + label: 'Unclassified failure', + suggestion: 'Open the deploy log and copy the troubleshooting details for the full error.', +}; + +/** + * Classify a failed deploy/update. Total: always returns a classification, + * falling back to `unknown`. `opts.dockerUnavailable` short-circuits to + * node_unreachable for errors the route already identified as a dead daemon + * (their message shape varies too much for patterns alone). + * + * Note: a ComposeRollbackError's message is already the underlying cause's + * message (its constructor copies it), so callers can pass + * getErrorMessage(error) for wrapped rollback failures unchanged. + */ +export function classifyFailure( + message: string, + opts?: { dockerUnavailable?: boolean }, +): FailureClassification { + if (opts?.dockerUnavailable) { + return { ...DOCKER_UNREACHABLE }; + } + for (const rule of RULES) { + if (rule.pattern.test(message)) { + return { reason: rule.reason, label: rule.label, suggestion: rule.suggestion }; + } + } + return { ...UNKNOWN_FAILURE }; +} diff --git a/backend/src/services/updateGuard/readiness.ts b/backend/src/services/updateGuard/readiness.ts new file mode 100644 index 00000000..d3aedb8c --- /dev/null +++ b/backend/src/services/updateGuard/readiness.ts @@ -0,0 +1,277 @@ +import type { PreflightStatus } from '../preflight/types'; +import type { UpdatePreviewSummary } from '../UpdatePreviewService'; +import type { + ContainerProbe, + ReadinessSignal, + ReadinessVerdict, + RollbackOverall, + RollbackReadinessItem, + SignalStatus, +} from './types'; + +/** + * Pure readiness scoring. UpdateGuardService gathers the inputs (each of which + * degrades independently to the 'error' sentinel) and these functions map them + * to signals and a verdict, so every grading rule is synchronously testable. + */ + +/** Sentinel for an input whose collection failed. */ +export type Errored = 'error'; + +const formatAge = (timestamp: number, now: number): string => { + const minutes = Math.max(0, Math.round((now - timestamp) / 60_000)); + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.round(minutes / 60); + if (hours < 48) return `${hours}h ago`; + return `${Math.round(hours / 24)}d ago`; +}; + +export function preflightSignal( + input: { status: PreflightStatus } | Errored, +): ReadinessSignal { + const base = { id: 'preflight' as const, title: 'Compose Doctor' }; + if (input === 'error') { + return { ...base, status: 'unknown', affectsVerdict: false, detail: 'The stored preflight report could not be read.' }; + } + switch (input.status) { + case 'never-run': + return { ...base, status: 'unknown', affectsVerdict: false, detail: 'Compose Doctor has not been run for this stack yet. Run it for a deeper pre-update check.' }; + case 'blocker': + return { ...base, status: 'blocked', affectsVerdict: true, detail: 'The last preflight found a blocker. Resolve it before updating.' }; + case 'unrenderable': + return { ...base, status: 'attention', affectsVerdict: true, detail: 'The compose file did not render in the last preflight; the update is likely to fail the same way.' }; + case 'high': + return { ...base, status: 'attention', affectsVerdict: true, detail: 'The last preflight found high-risk findings. Review them before updating.' }; + case 'warning': + return { ...base, status: 'warning', affectsVerdict: true, detail: 'The last preflight found warnings.' }; + case 'pass': + case 'info': + return { ...base, status: 'ok', affectsVerdict: true, detail: 'The last preflight passed.' }; + } +} + +export function driftSignal(input: number | Errored): ReadinessSignal { + const base = { id: 'drift' as const, title: 'Drift' }; + if (input === 'error') { + return { ...base, status: 'unknown', affectsVerdict: false, detail: 'Drift findings could not be read.' }; + } + if (input > 0) { + const plural = input === 1 ? 'finding' : 'findings'; + return { + ...base, + status: 'warning', + affectsVerdict: true, + detail: `${input} open drift ${plural}: the running state has diverged from the compose file, so the rollback target may not match what is running.`, + }; + } + return { ...base, status: 'ok', affectsVerdict: true, detail: 'No open drift findings.' }; +} + +export function containersSignal(input: ContainerProbe[] | Errored): ReadinessSignal { + const base = { id: 'containers' as const, title: 'Current containers' }; + if (input === 'error') { + return { ...base, status: 'unknown', affectsVerdict: true, detail: 'Container state could not be read from Docker.' }; + } + if (input.length === 0) { + return { ...base, status: 'warning', affectsVerdict: true, detail: 'The stack is not running; updating will start it.' }; + } + const troubled = input.filter( + c => c.health === 'unhealthy' || c.state === 'restarting' || (c.state === 'exited' && (c.exitCode ?? 0) !== 0), + ); + if (troubled.length > 0) { + const names = troubled.map(c => c.name).join(', '); + return { + ...base, + status: 'attention', + affectsVerdict: true, + detail: `Already unhealthy before the update: ${names}. An update on top of a failing stack is hard to evaluate; consider fixing or stopping it first.`, + }; + } + return { ...base, status: 'ok', affectsVerdict: true, detail: `${input.length} container${input.length === 1 ? '' : 's'} running normally.` }; +} + +export function healthchecksSignal(input: ContainerProbe[] | Errored): ReadinessSignal { + const base = { id: 'healthchecks' as const, title: 'Healthcheck coverage', status: 'ok' as SignalStatus, affectsVerdict: false }; + if (input === 'error' || input.length === 0) { + return { ...base, detail: 'Coverage is unknown until the stack runs. Containers without healthchecks are verified by run state only after an update.' }; + } + const withCheck = input.filter(c => c.hasHealthcheck).length; + const withoutRestart = input.filter(c => !c.restartPolicy || c.restartPolicy === 'no').length; + const parts = [ + `${withCheck} of ${input.length} container${input.length === 1 ? '' : 's'} define a healthcheck; the rest are verified by run state only after an update.`, + ]; + if (withoutRestart > 0) { + parts.push(`${withoutRestart} ha${withoutRestart === 1 ? 's' : 've'} no restart policy.`); + } + return { ...base, detail: parts.join(' ') }; +} + +export function updatePreviewSignal(input: UpdatePreviewSummary | Errored): ReadinessSignal { + const base = { id: 'update_preview' as const, title: 'Pending update' }; + if (input === 'error') { + return { ...base, status: 'unknown', affectsVerdict: false, detail: 'The update preview is unavailable.' }; + } + if (input.blocked) { + return { + ...base, + status: 'blocked', + affectsVerdict: true, + detail: input.blocked_reason ?? 'A scan policy blocks this update.', + }; + } + if (input.has_update && input.semver_bump === 'major') { + const change = input.current_tag && input.next_tag ? ` (${input.current_tag} to ${input.next_tag})` : ''; + return { ...base, status: 'attention', affectsVerdict: true, detail: `A major version bump is pending${change}. Review the upstream changelog for breaking changes.` }; + } + if (input.has_update && input.semver_bump === 'unknown') { + return { ...base, status: 'warning', affectsVerdict: true, detail: 'An image update is pending but the version change could not be classified.' }; + } + if (input.has_update) { + const kind = input.update_kind === 'digest' ? 'a same-tag image refresh' : `a ${input.semver_bump} update`; + return { ...base, status: 'ok', affectsVerdict: true, detail: `Pending: ${kind}.` }; + } + return { ...base, status: 'ok', affectsVerdict: true, detail: 'No pending image update detected; the update re-pulls and recreates with current tags.' }; +} + +export function backupSlotSignal( + input: { exists: boolean; timestamp: number | null } | Errored, + now: number, +): ReadinessSignal { + const base = { id: 'backup_slot' as const, title: 'Rollback backup' }; + if (input === 'error') { + return { ...base, status: 'unknown', affectsVerdict: false, detail: 'The backup slot could not be read.' }; + } + if (!input.exists) { + return { ...base, status: 'warning', affectsVerdict: true, detail: 'No rollback backup exists yet; one is created automatically when the update starts.' }; + } + const age = input.timestamp ? ` (from ${formatAge(input.timestamp, now)})` : ''; + return { ...base, status: 'ok', affectsVerdict: true, detail: `A compose and env file backup exists${age} and is refreshed when the update starts.` }; +} + +export function diskSignal( + input: { usePercent: number; limitPercent: number } | null | Errored, +): ReadinessSignal { + const base = { id: 'disk' as const, title: 'Node disk' }; + if (input === 'error' || input === null) { + return { ...base, status: 'unknown', affectsVerdict: false, detail: 'Disk usage could not be read.' }; + } + const use = Math.round(input.usePercent); + if (input.usePercent >= input.limitPercent) { + return { ...base, status: 'attention', affectsVerdict: true, detail: `Disk usage is at ${use}%, at or above the ${input.limitPercent}% alert threshold. Image pulls may fail; free space first.` }; + } + if (input.usePercent >= input.limitPercent - 5) { + return { ...base, status: 'warning', affectsVerdict: true, detail: `Disk usage is at ${use}%, close to the ${input.limitPercent}% alert threshold.` }; + } + return { ...base, status: 'ok', affectsVerdict: true, detail: `Disk usage is at ${use}%.` }; +} + +/** + * Severity precedence: blocked > attention (review required) > verdict-affecting + * unknown > warning > ready. Informational unknowns never affect the verdict. + */ +export function aggregateVerdict(signals: ReadinessSignal[]): ReadinessVerdict { + const affecting = signals.filter(s => s.affectsVerdict); + if (affecting.some(s => s.status === 'blocked')) return 'blocked'; + if (affecting.some(s => s.status === 'attention')) return 'review_required'; + if (affecting.some(s => s.status === 'unknown')) return 'unknown'; + if (affecting.some(s => s.status === 'warning')) return 'ready_with_warnings'; + return 'ready'; +} + +// ── Rollback readiness ─────────────────────────────────────────────────────── + +export interface RollbackInputs { + backup: { exists: boolean; timestamp: number | null } | Errored; + envSummary: { exists: boolean; envPresent: boolean; keys: string[] } | Errored; + /** Whether the stack currently has an env file (distinguishes "no env to cover"). */ + stackHasEnv: boolean | Errored; + /** + * UpdatePreview.rollback_target wrapped in an object so the Errored sentinel + * cannot be absorbed into the string domain (an image literally named + * "error" must not read as a failed preview). + */ + rollbackTarget: { target: string | null } | Errored; + /** Timestamp of the most recent deploy_success activity event, if any. */ + lastDeployAt: number | null | Errored; + containers: ContainerProbe[] | Errored; +} + +export function buildRollbackItems(inputs: RollbackInputs, now: number): RollbackReadinessItem[] { + const items: RollbackReadinessItem[] = []; + + const backupExists = inputs.backup !== 'error' && inputs.backup.exists; + if (inputs.backup === 'error') { + items.push({ id: 'compose_source', state: 'unknown', label: 'Previous compose file', detail: 'The backup slot could not be read.' }); + } else if (backupExists) { + const age = inputs.backup.timestamp ? ` from ${formatAge(inputs.backup.timestamp, now)}` : ''; + items.push({ id: 'compose_source', state: 'ready', label: 'Previous compose file', detail: `A backup${age} is available to restore.` }); + } else { + items.push({ id: 'compose_source', state: 'missing', label: 'Previous compose file', detail: 'No backup exists yet. One is created automatically by the next update or deploy.' }); + } + + if (inputs.envSummary === 'error') { + items.push({ id: 'env_keys', state: 'unknown', label: 'Previous env file', detail: 'The backed-up env file could not be read.' }); + } else if (inputs.envSummary.envPresent) { + const n = inputs.envSummary.keys.length; + items.push({ id: 'env_keys', state: 'ready', label: 'Previous env file', detail: `${n} variable name${n === 1 ? '' : 's'} captured in the backup (values are restored with the file, never shown here).` }); + } else if (inputs.stackHasEnv === true && backupExists) { + items.push({ id: 'env_keys', state: 'missing', label: 'Previous env file', detail: 'The stack has an env file but the backup does not contain one; a rollback would not restore env changes.' }); + } else if (!backupExists) { + items.push({ id: 'env_keys', state: 'missing', label: 'Previous env file', detail: 'No backup exists yet.' }); + } else { + items.push({ id: 'env_keys', state: 'ready', label: 'Previous env file', detail: 'The stack uses no env file, so there is nothing to restore.' }); + } + + if (inputs.rollbackTarget === 'error') { + items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The update preview is unavailable.' }); + } else if (inputs.rollbackTarget.target) { + items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Known rollback target: ${inputs.rollbackTarget.target}. If the compose file uses a moving tag, restoring files alone does not revert the image; pin this tag to be exact.` }); + } else { + items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. A rollback restores compose and env files; a moving tag may keep the newer image.' }); + } + + if (inputs.lastDeployAt === 'error') { + items.push({ id: 'last_deploy', state: 'unknown', label: 'Last successful deploy', detail: 'The activity history could not be read.' }); + } else if (inputs.lastDeployAt) { + items.push({ id: 'last_deploy', state: 'ready', label: 'Last successful deploy', detail: `Recorded ${formatAge(inputs.lastDeployAt, now)}; the backup reflects a configuration that deployed successfully.` }); + } else { + items.push({ id: 'last_deploy', state: 'missing', label: 'Last successful deploy', detail: 'No successful deploy is recorded in the recent activity history.' }); + } + + if (inputs.containers === 'error') { + items.push({ id: 'healthchecks', state: 'unknown', label: 'Healthchecks', detail: 'Container state could not be read from Docker.' }); + } else if (inputs.containers.some(c => c.hasHealthcheck)) { + items.push({ id: 'healthchecks', state: 'ready', label: 'Healthchecks', detail: 'At least one service defines a healthcheck, so a rollback can be verified beyond run state.' }); + } else { + items.push({ id: 'healthchecks', state: 'missing', label: 'Healthchecks', detail: 'No service defines a healthcheck; rollback verification relies on run state only.' }); + } + + const mounts = inputs.containers === 'error' + ? [] + : [...new Set(inputs.containers.flatMap(c => c.mounts))]; + const mountDetail = mounts.length > 0 ? ` This stack mounts: ${mounts.join(', ')}.` : ''; + items.push({ + id: 'volume_data', + state: 'not_covered', + label: 'Application data', + detail: `Named volumes and bind-mounted data are not included in file backups. Rolling back restores compose and env files only; application data keeps its current state.${mountDetail}`, + }); + + return items; +} + +/** + * compose_source gates not_ready; ready additionally requires env coverage and + * a known previous image tag. volume_data, healthchecks, and last_deploy are + * disclosures and never gate the overall state. + */ +export function aggregateRollbackOverall(items: RollbackReadinessItem[]): RollbackOverall { + const byId = new Map(items.map(i => [i.id, i.state])); + if (byId.get('compose_source') !== 'ready') { + return byId.get('compose_source') === 'unknown' ? 'partial' : 'not_ready'; + } + if (byId.get('env_keys') === 'ready' && byId.get('previous_images') === 'ready') { + return 'ready'; + } + return 'partial'; +} diff --git a/backend/src/services/updateGuard/types.ts b/backend/src/services/updateGuard/types.ts new file mode 100644 index 00000000..b3ed2ad4 --- /dev/null +++ b/backend/src/services/updateGuard/types.ts @@ -0,0 +1,115 @@ +/** Overall verdict of an update readiness check. */ +export type ReadinessVerdict = 'ready' | 'ready_with_warnings' | 'review_required' | 'blocked' | 'unknown'; + +/** Graded status of a single readiness signal. */ +export type SignalStatus = 'ok' | 'warning' | 'attention' | 'blocked' | 'unknown'; + +/** One input to the readiness verdict (preflight, drift, containers, ...). */ +export interface ReadinessSignal { + id: 'preflight' | 'drift' | 'containers' | 'healthchecks' | 'update_preview' | 'backup_slot' | 'disk'; + status: SignalStatus; + /** Short headline ("Compose Doctor", "Running containers"). */ + title: string; + /** What was observed and why it matters. Never carries an env value. */ + detail: string; + /** + * False for informational unknowns (preflight never run, preview + * unavailable) that should not drag the overall verdict to `unknown`. + */ + affectsVerdict: boolean; +} + +/** Computed on demand; not persisted (the inputs keep their own history). */ +export interface UpdateReadinessReport { + stack: string; + computedAt: number; + verdict: ReadinessVerdict; + signals: ReadinessSignal[]; +} + +/** State of one rollback readiness item. */ +export type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered'; + +export interface RollbackReadinessItem { + id: 'compose_source' | 'env_keys' | 'previous_images' | 'last_deploy' | 'healthchecks' | 'volume_data'; + state: RollbackItemState; + label: string; + /** Names only for env coverage; values never appear here. */ + detail: string; +} + +export type RollbackOverall = 'ready' | 'partial' | 'not_ready'; + +export interface RollbackReadinessReport { + stack: string; + computedAt: number; + overall: RollbackOverall; + items: RollbackReadinessItem[]; +} + +/** Normalized per-container probe used by readiness and the health gate. */ +export interface ContainerProbe { + name: string; + /** Docker container state (running, exited, restarting, ...), or 'unknown'. */ + state: string; + /** Docker health status (healthy | unhealthy | starting) or null without a healthcheck. */ + health: string | null; + exitCode: number | null; + hasHealthcheck: boolean; + /** RestartPolicy.Name, or null/'no' when none is set. */ + restartPolicy: string | null; + /** Mount descriptors ("volume data", "bind /srv/app"), for coverage disclosure. */ + mounts: string[]; +} + +/** Lifecycle of a post-update health gate observation. */ +export type HealthGateStatus = 'observing' | 'passed' | 'failed' | 'unknown'; + +/** Per-container end state captured when a gate run finalizes. */ +export interface HealthGateContainer { + name: string; + /** Docker container state, or 'missing' when it vanished mid-observation. */ + state: string; + health: string | null; + restarts: number; +} + +/** Payload of GET /:stackName/health-gate ('never-run' when no run exists). */ +export interface HealthGateReport { + stack: string; + id: string | null; + status: HealthGateStatus | 'never-run'; + trigger: 'update' | 'deploy' | null; + reason: string | null; + windowSeconds: number | null; + startedAt: number | null; + endedAt: number | null; + containers: HealthGateContainer[]; +} + +/** Categories a failed stack deploy or update can be classified into. */ +export type FailureReason = + | 'image_pull_failed' + | 'compose_render_failed' + | 'env_missing' + | 'port_conflict' + | 'bind_path_missing' + | 'permission_denied' + | 'container_exited' + | 'healthcheck_failed' + | 'dependency_unavailable' + | 'node_unreachable' + | 'unknown'; + +/** + * Operator-facing classification of a failed deploy/update, attached to the + * route's error response so the recovery surfaces can show a cause and a next + * step instead of only the raw compose output. + */ +export interface FailureClassification { + reason: FailureReason; + /** Short display headline ("Host port conflict"). */ + label: string; + /** Suggested next action, one sentence. */ + suggestion: string; +} diff --git a/docs/docs.json b/docs/docs.json index c9eddd18..e509fe14 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -116,6 +116,7 @@ "pages": [ "features/deploy-progress", "features/atomic-deployments", + "features/health-gated-updates", "features/deploy-enforcement", "features/git-sources", "features/app-store", diff --git a/docs/features/atomic-deployments.mdx b/docs/features/atomic-deployments.mdx index fcd4e925..265cc072 100644 --- a/docs/features/atomic-deployments.mdx +++ b/docs/features/atomic-deployments.mdx @@ -5,7 +5,7 @@ description: Wrap every deploy and update in a backup, a 3-second health probe, Sencho wraps every protected deploy in a four-step safety net: it copies the current `compose.yaml` and `.env` to a writable backup directory, runs the requested compose action, waits 3 seconds for the new containers to settle, then checks them for a non-zero exit code. If any container has crashed, Sencho restores the backed-up files and re-deploys automatically. -The same backup also powers the **Rollback** action in the stack editor, so you can roll a stack back to its last good configuration on demand. +The same backup also powers the **Rollback** action in the stack editor, so you can roll a stack back to its last good configuration on demand. To see in advance whether that rollback would actually help, and to watch container health for longer than the 3-second probe, see [Health-Gated Updates](/features/health-gated-updates). ## How it works @@ -54,7 +54,7 @@ A restore is a faithful revert, not an overlay. Sencho replaces the compose file A user without the `stack:deploy` permission will still see the menu entry; the rejection comes from the backend with a permission error after they click. Ask an admin to grant `stack:deploy` through **Settings · Roles & Access** if that happens. - The health probe is a 3-second window after `docker compose up -d` returns. Crashes that happen after that window are out of scope for atomic rollback by design, because Sencho cannot tell whether a late exit is a real failure or a normal restart. For ongoing health, use **Auto-Heal Policies** to restart unhealthy containers automatically and **Alert Rules** to page you when a container exits unexpectedly. + The health probe is a 3-second window after `docker compose up -d` returns. Crashes that happen after that window are out of scope for atomic rollback by design, because Sencho cannot tell whether a late exit is a real failure or a normal restart. The [health gate](/features/health-gated-updates) covers exactly this period: it observes the stack for a configurable window after the update, records a verdict on the stack timeline, and offers a manual rollback when containers do not stay healthy. For ongoing health beyond that, use **Auto-Heal Policies** to restart unhealthy containers automatically and **Alert Rules** to page you when a container exits unexpectedly. This message means the auto-rollback attempted to restore the backup and re-deploy, but the restore step or the re-deploy itself errored out. The backup files are still at `/backups//`. To recover: diff --git a/docs/features/deploy-progress.mdx b/docs/features/deploy-progress.mdx index d71f00d6..e3f127f3 100644 --- a/docs/features/deploy-progress.mdx +++ b/docs/features/deploy-progress.mdx @@ -58,6 +58,8 @@ Click **Raw output** in the footer to expand a 200 px raw terminal panel beneath When an action completes successfully, the status indicator switches to `Succeeded · closes in s` and a 4-second countdown begins. Hover anywhere over the modal to pause the countdown; moving the cursor away restarts a fresh 4 seconds. Clicking **Close** dismisses the modal immediately. +When the [health gate](/features/health-gated-updates) is observing the stack after a deploy or update, the modal withholds the success verdict: the status indicator shows **Verifying health** instead of **Succeeded**, and the auto-close waits for the gate. A passed gate shows the success state and resumes the countdown; a failed or unknown verdict becomes the headline result and keeps the modal open with the reason. Closing the modal never stops the observation. + Deploy progress modal in the succeeded state with a green checkmark, the text 'Succeeded' and 'closes in 2s' in the header, and the full structured log body visible beneath diff --git a/docs/features/health-gated-updates.mdx b/docs/features/health-gated-updates.mdx new file mode 100644 index 00000000..f1228b2f --- /dev/null +++ b/docs/features/health-gated-updates.mdx @@ -0,0 +1,119 @@ +--- +title: Health-Gated Updates +description: Check update readiness before applying, watch container health after the update lands, and know exactly what a rollback can restore. +--- + +Updating a stack is the highest-anxiety operation in a homelab: a pull-and-recreate can break containers, networking, env configuration, or storage assumptions, and `docker compose` alone gives you no answer to "is it safe to update this right now?". Sencho makes updates deliberate and reversible in three layers: a readiness check before the update, a health gate after it, and an always-honest rollback readiness report on the stack. + +None of these layers blocks you. Readiness is advisory, the health gate is purely observational, and rollback is always an explicit action you confirm. The only thing that hard-blocks an update is a [deploy enforcement policy](/features/deploy-enforcement). + +## Update readiness + +When you trigger **Update** from the stack editor toolbar or the sidebar menu, Sencho first opens a readiness dialog with a single verdict and the signals behind it: + +| Verdict | Meaning | +|---------|---------| +| Ready | Nothing stands out; the update can proceed. | +| Ready with warnings | Proceed, but read the warnings first. | +| Review required | Something needs a look: a high-risk preflight finding, an already-unhealthy container, a pending major version bump, or low disk. | +| Blocked | A blocker was found, such as a Compose Doctor blocker or a scan policy that will stop the update. | +| Unknown | Readiness could not be verified, for example when the node or Docker is unreachable. | + +The verdict is computed from signals Sencho already tracks, so the check is fast and never re-runs anything heavy: + +- **Compose Doctor**: the stored result of the last [preflight run](/features/compose-doctor). A blocker finding makes the verdict Blocked; high-risk findings ask for review. If Compose Doctor has never run, the dialog says so without dragging the verdict down. +- **Drift**: open [drift findings](/features/stack-drift) warn you that the running state has diverged, so the rollback target may not match what is running. +- **Current containers**: a container that is already unhealthy, restarting, or crashed before the update makes the result hard to evaluate; the dialog asks you to look first. +- **Pending image change**: the [update preview](/features/auto-update-policies) classifies the pending change. A major version bump asks for review; patch updates and same-tag refreshes pass quietly. +- **Rollback backup**: whether a backup slot exists and how old it is. A missing backup is only a note, because the update itself creates a fresh one when it starts. +- **Node disk**: disk usage near or above the node's alert threshold warns you before a large pull fills the disk. +- **Healthcheck coverage**: how many services define healthchecks, which determines how thoroughly the post-update health gate can verify the result. + +Admins also see whether a [fleet snapshot](/features/fleet-backups) covers this stack and can tick **Create a fleet snapshot before updating** to capture one as part of proceeding. If the snapshot fails, the update does not start. + +Every verdict keeps the **Update now** button enabled. The dialog informs the decision; it does not make it for you. + + + Update readiness dialog for a stack, showing the verdict chip, the signal rows for Compose Doctor, drift, containers, pending update, rollback backup and node disk, the fleet snapshot row with the create-snapshot checkbox, and the Cancel and Update now buttons + + +## The post-update health gate + +The 3-second crash probe from [atomic deployments](/features/atomic-deployments) catches instant failures. The health gate is the longer observational layer behind it: after a deploy or update succeeds, Sencho watches the stack's containers for an observation window (90 seconds by default) and records a verdict. + +During the window, the gate checks every container that came up with the stack: + +- all expected containers stay **running**, +- Docker healthchecks report **healthy** wherever a service defines one, +- no container **exits**, **disappears**, or enters a **restart loop**. + +A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period when the window ends records the verdict as unknown rather than claiming success. + +The deploy progress modal treats the gate verdict as the real result: while the gate observes, the status reads **Verifying health** rather than claiming success, and the auto-close waits. A passed gate shows the success state; a failed or unknown gate becomes the headline result with its reason. Closing the modal never stops the observation; the gate runs server-side and its result appears on the stack timeline either way, as **health gate passed** or **health gate failed** events alongside the **update started** marker. + +The live verifying and recovery view is part of the deploy progress panel. If you have turned that panel off, the in-browser gate view does not appear, but the gate still runs on the node and records its verdict on the stack timeline. + + + Deploy progress modal after an update finished, with the status indicator showing Verifying health and the health gate banner reading observing containers + + + + Deploy progress modal with the headline status Health gate failed and the banner naming the unhealthy container, with the hint that rollback options are available on the stack + + +When the gate fails, the stack page surfaces the same [recovery actions](/features/deploy-progress#recovery-actions) as a failed update: retry, restart, roll back when a backup exists, refresh the container state, or copy diagnostics. Rolling back is always your call; the gate never rolls anything back on its own. + +The gate also runs for updates you did not click: scheduled image updates, webhook-triggered deploys and pulls, bulk updates, and Git source applies all record gate verdicts on the stack timeline. Rollbacks, App Store installs, and Sencho's own automation loops are deliberately not gated. + +### Tuning or disabling the gate + +Open **Settings > Host Alerts > Update health gate** on the node you want to configure: + +- **Observe health after updates** turns the gate on or off for that node. On by default; turning it off only stops the observation and its timeline events, never the update itself. +- **Observation window** sets how long containers are watched, from 15 to 600 seconds. The default is 90 seconds; raise it for stacks that take a while to settle, since a healthcheck still starting at the end of the window records an unknown verdict instead of a pass. + + + Host Alerts settings page with the Update health gate group, showing the Observe health after updates toggle and the Observation window field + + +## Rollback readiness + +The Stack Dossier carries a **Rollback readiness** section that answers one question honestly: if this update goes wrong, what can a rollback actually restore? It reports an overall state of Ready, Partial, or Not ready, built from: + +- **Previous compose file**: whether a backup slot exists and how old it is. +- **Previous env file**: whether the backup contains the stack's env file. Sencho lists how many variable names are covered; the values themselves are restored with the file and never displayed. +- **Previous image tag**: the known rollback target from the update preview. When the compose file uses a moving tag, restoring files alone does not revert the image, so the report names the exact tag you would pin to be precise. +- **Last successful deploy**: whether the backup reflects a configuration that actually deployed successfully. +- **Healthchecks**: whether a rollback can be verified beyond run state. +- **Application data**: always reported as not covered. Named volumes and bind-mounted data are not included in file backups; a rollback restores compose and env files only, and your application data keeps its current state. This row exists so the limit is stated where you decide, not discovered during an incident. + + + Rollback readiness section in the Stack Dossier with the overall Ready chip and the six rows: previous compose file, previous env file with variable names only, previous image tag, last successful deploy, healthchecks, and the application data non-coverage disclosure + + +## Classified failures + +When a deploy or update fails, Sencho classifies the failure from the compose output and shows the cause with a suggested next step in the recovery panel: an image pull failure, a missing environment variable, a host port conflict, a missing bind-mount path, a permission problem, a crashed container, a failed healthcheck, an unavailable dependency, an unreachable node or Docker daemon, or an invalid compose file. The classification also lands in **Copy details**, so a bug report carries the cause, not just the raw output. + +## Troubleshooting + + + + Unknown means one of the verdict-affecting signals could not be verified, most often because Docker or the node did not answer in time. The update path is never blocked by an unknown verdict; check the node's connection if it persists, and proceed when you are confident in the stack's state. + + + The gate fails on the first clear problem it observes: a container exit, an unhealthy healthcheck, a restart loop, or a container disappearing mid-window. Check the named container's logs from the stack page. If the container legitimately restarts during startup (a migration step, for example), raise the observation window under **Settings > Host Alerts > Update health gate** so the gate watches past the settle period. + + + Unknown means the observation could not finish: Sencho restarted mid-window, a newer update superseded the observation, Docker became unreachable, a healthcheck was still starting when the window ended, or no containers appeared to observe. The stack timeline records the reason. Check the containers directly; an unknown verdict makes no claim either way. + + + The modal holds its auto-close while the gate observes so the verdict is not lost. You can close it at any time; the observation continues server-side and the verdict lands on the stack timeline. If the gate result repeatedly cannot be retrieved, the modal gives up with an unknown verdict instead of waiting forever. + + + That is by design and true for every stack: file backups cover compose and env files, never named volumes or bind-mounted data. For point-in-time copies of stack files across the fleet, use [fleet snapshots](/features/fleet-backups); for application data, use a backup tool appropriate to the workload (database dumps, volume backups) before risky updates. + + + The sidebar's per-stack **Update** action runs the same path as the editor toolbar, so it shows the same readiness dialog and deploy progress. One click on **Update now** proceeds. On nodes that do not advertise the capability, updates run directly without the dialog. + + diff --git a/docs/images/health-gated-updates/dossier-rollback-readiness.png b/docs/images/health-gated-updates/dossier-rollback-readiness.png new file mode 100644 index 00000000..3cb8eee4 Binary files /dev/null and b/docs/images/health-gated-updates/dossier-rollback-readiness.png differ diff --git a/docs/images/health-gated-updates/modal-gate-failed.png b/docs/images/health-gated-updates/modal-gate-failed.png new file mode 100644 index 00000000..91200de1 Binary files /dev/null and b/docs/images/health-gated-updates/modal-gate-failed.png differ diff --git a/docs/images/health-gated-updates/modal-verifying.png b/docs/images/health-gated-updates/modal-verifying.png new file mode 100644 index 00000000..ad1df899 Binary files /dev/null and b/docs/images/health-gated-updates/modal-verifying.png differ diff --git a/docs/images/health-gated-updates/readiness-dialog.png b/docs/images/health-gated-updates/readiness-dialog.png new file mode 100644 index 00000000..43edf1e6 Binary files /dev/null and b/docs/images/health-gated-updates/readiness-dialog.png differ diff --git a/docs/images/health-gated-updates/settings.png b/docs/images/health-gated-updates/settings.png new file mode 100644 index 00000000..77277bdb Binary files /dev/null and b/docs/images/health-gated-updates/settings.png differ diff --git a/docs/openapi.yaml b/docs/openapi.yaml index daa21fdb..5ef81ea0 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -149,6 +149,32 @@ components: type: boolean example: true + FailureClassification: + type: object + description: Classified cause of a failed deploy or update, with a suggested next step. + required: [reason, label, suggestion] + properties: + reason: + type: string + enum: + - image_pull_failed + - compose_render_failed + - env_missing + - port_conflict + - bind_path_missing + - permission_denied + - container_exited + - healthcheck_failed + - dependency_unavailable + - node_unreachable + - unknown + label: + type: string + description: Short display headline for the cause. + suggestion: + type: string + description: Suggested next action, one sentence. + Container: type: object properties: @@ -945,9 +971,19 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/SuccessMessage" - example: - message: Deployed successfully + type: object + required: [message] + properties: + message: + type: string + example: Deployed successfully + healthGateId: + type: string + nullable: true + description: | + Id of the post-deploy health gate observation started for + this deploy, for use with the health-gate endpoint. Null + when the health gate is disabled on the node. "403": $ref: "#/components/responses/Forbidden" "500": @@ -963,6 +999,8 @@ paths: rolledBack: type: boolean description: Whether the stack was automatically rolled back. + failure: + $ref: "#/components/schemas/FailureClassification" /api/stacks/{stackName}/down: post: @@ -1095,6 +1133,13 @@ paths: status: type: string example: Update completed + healthGateId: + type: string + nullable: true + description: | + Id of the post-update health gate observation started for + this update, for use with the health-gate endpoint. Null + when the health gate is disabled on the node. "403": $ref: "#/components/responses/Forbidden" "500": @@ -1109,6 +1154,8 @@ paths: type: string rolledBack: type: boolean + failure: + $ref: "#/components/schemas/FailureClassification" /api/stacks/{stackName}/rollback: post: @@ -1169,6 +1216,188 @@ paths: "500": $ref: "#/components/responses/InternalError" + /api/stacks/{stackName}/update-readiness: + get: + operationId: getStackUpdateReadiness + tags: [Stacks] + summary: Check update readiness + description: | + Computes an advisory readiness verdict for updating the stack right + now, from the stored preflight result, open drift findings, live + container health, the pending image change, the rollback backup slot, + and node disk headroom. Advisory only: no verdict blocks an update. + Requires `stack:read` permission. + parameters: + - $ref: "#/components/parameters/stackName" + - $ref: "#/components/parameters/nodeId" + responses: + "200": + description: Readiness report. + content: + application/json: + schema: + type: object + required: [stack, computedAt, verdict, signals] + properties: + stack: + type: string + computedAt: + type: integer + description: Unix timestamp (ms) the report was computed. + verdict: + type: string + enum: [ready, ready_with_warnings, review_required, blocked, unknown] + signals: + type: array + items: + type: object + required: [id, status, title, detail, affectsVerdict] + properties: + id: + type: string + status: + type: string + enum: [ok, warning, attention, blocked, unknown] + title: + type: string + detail: + type: string + affectsVerdict: + type: boolean + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "500": + $ref: "#/components/responses/InternalError" + + /api/stacks/{stackName}/rollback-readiness: + get: + operationId: getStackRollbackReadiness + tags: [Stacks] + summary: Check rollback readiness + description: | + Reports what a rollback of this stack can actually restore: the backed + up compose file, env coverage (variable names only, never values), the + known previous image tag, and an explicit disclosure that volume and + bind-mounted data are not covered by file backups. + Requires `stack:read` permission. + parameters: + - $ref: "#/components/parameters/stackName" + - $ref: "#/components/parameters/nodeId" + responses: + "200": + description: Rollback readiness report. + content: + application/json: + schema: + type: object + required: [stack, computedAt, overall, items] + properties: + stack: + type: string + computedAt: + type: integer + overall: + type: string + enum: [ready, partial, not_ready] + items: + type: array + items: + type: object + required: [id, state, label, detail] + properties: + id: + type: string + state: + type: string + enum: [ready, missing, unknown, not_covered] + label: + type: string + detail: + type: string + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "500": + $ref: "#/components/responses/InternalError" + + /api/stacks/{stackName}/health-gate: + get: + operationId: getStackHealthGate + tags: [Stacks] + summary: Get post-update health gate result + description: | + Returns a post-deploy health gate observation for the stack: the run + named by `gateId` (as returned in a deploy or update response), or the + most recent run when omitted. Status `never-run` means no observation + exists. Requires `stack:read` permission. + parameters: + - $ref: "#/components/parameters/stackName" + - $ref: "#/components/parameters/nodeId" + - name: gateId + in: query + required: false + schema: + type: string + description: A specific gate run id to read instead of the latest. + responses: + "200": + description: Health gate report. + content: + application/json: + schema: + type: object + required: [stack, status] + properties: + stack: + type: string + id: + type: string + nullable: true + status: + type: string + enum: [observing, passed, failed, unknown, never-run] + trigger: + type: string + nullable: true + enum: [update, deploy, null] + reason: + type: string + nullable: true + windowSeconds: + type: integer + nullable: true + startedAt: + type: integer + nullable: true + endedAt: + type: integer + nullable: true + containers: + type: array + items: + type: object + required: [name, state, restarts] + properties: + name: + type: string + state: + type: string + description: Docker container state, or `missing` when it vanished mid-observation. + health: + type: string + nullable: true + restarts: + type: integer + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "500": + $ref: "#/components/responses/InternalError" + # ── Containers ────────────────────────────────────────── /api/containers: diff --git a/e2e/deploy-log-panel.spec.ts b/e2e/deploy-log-panel.spec.ts index 3398fcb4..f5e35bb0 100644 --- a/e2e/deploy-log-panel.spec.ts +++ b/e2e/deploy-log-panel.spec.ts @@ -77,6 +77,25 @@ async function enableDeployFeedback(page: Page): Promise { }, DEPLOY_FEEDBACK_KEY); } +/** + * Shrink the post-deploy health gate observation window so success-path tests + * can watch the full deploy, verify, succeed sequence without waiting out the + * 90s default. 15 seconds is the smallest value the settings API accepts. + */ +async function setHealthGateWindow(page: Page, seconds: number): Promise { + await page.evaluate(async (windowSeconds: number) => { + const res = await fetch('/api/settings', { + method: 'PATCH', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ health_gate_window_seconds: windowSeconds }), + }); + if (!res.ok) { + throw new Error(`Failed to set health gate window: ${res.status}`); + } + }, seconds); +} + /** * Force the React hook to re-read localStorage and update its state. Used * right before a deploy click to defeat any stale-state edge cases after @@ -185,9 +204,12 @@ test.describe('Deploy feedback modal', () => { }); test('modal opens, streams output, and auto-closes on success', async ({ page }) => { - test.setTimeout(120_000); + // Worst-case assertion budget: 90s compose run + 45s gate verdict + 15s + // auto-close, so the per-assertion timeout fires before the test timeout. + test.setTimeout(160_000); await enableDeployFeedback(page); + await setHealthGateWindow(page, 15); await setupDeployStack(page, HAPPY_STACK, HAPPY_COMPOSE); await syncDeployFeedbackState(page); @@ -209,8 +231,13 @@ test.describe('Deploy feedback modal', () => { streamingIndicator.waitFor({ state: 'visible', timeout: 10_000 }), ]); - // Wait for the operation to succeed - await expect(page.getByText('Succeeded')).toBeVisible({ timeout: 90_000 }); + // The compose run finishes first (up to 90s for a cold image pull), then + // the post-deploy health gate observes the containers before the modal + // commits to a verdict. + await expect(modal.getByText('Verifying health')).toBeVisible({ timeout: 90_000 }); + + // Gate window is 15s; allow the 4s status polling cadence plus slack. + await expect(modal.getByText('Succeeded')).toBeVisible({ timeout: 45_000 }); // Modal auto-closes AUTO_CLOSE_SECONDS (4s) after success; allow up to 15s total await expect(modal).toBeHidden({ timeout: 15_000 }); @@ -275,6 +302,7 @@ test.describe('Deploy feedback modal', () => { test.setTimeout(120_000); await enableDeployFeedback(page); + await setHealthGateWindow(page, 15); await setupDeployStack(page, HAPPY_STACK, HAPPY_COMPOSE); await syncDeployFeedbackState(page); @@ -350,5 +378,9 @@ test.describe('Deploy feedback modal', () => { await deleteStackViaApi(page, HAPPY_STACK); await deleteStackViaApi(page, FAIL_STACK); await disableDeployFeedback(page); + // Restore the default observation window so the shortened test value does + // not leak into later suites or manual sessions against the same instance. + // Best-effort: a failed test may have already lost the page session. + await setHealthGateWindow(page, 90).catch(() => {}); }); }); diff --git a/frontend/src/components/DeployFeedbackModal.tsx b/frontend/src/components/DeployFeedbackModal.tsx index 3e54d42f..572383f4 100644 --- a/frontend/src/components/DeployFeedbackModal.tsx +++ b/frontend/src/components/DeployFeedbackModal.tsx @@ -4,6 +4,8 @@ import { CheckCircle2, AlertCircle, AlertTriangle, + CircleHelp, + HeartPulse, X, Minimize2, Terminal as TerminalIcon, @@ -13,7 +15,7 @@ import { DialogTitle } from '@/components/ui/dialog'; import { Button } from '@/components/ui/button'; import { StructuredLogRow } from '@/components/log-rendering/StructuredLogRow'; import TerminalComponent from '@/components/Terminal'; -import { useDeployFeedback, VERB_LABELS } from '@/context/DeployFeedbackContext'; +import { useDeployFeedback, VERB_LABELS, type HealthGateUiState } from '@/context/DeployFeedbackContext'; const AUTO_CLOSE_SECONDS = 4; @@ -37,7 +39,7 @@ function formatElapsed(seconds: number): string { } export function DeployFeedbackModal({ isMinimized, onMinimize }: DeployFeedbackModalProps) { - const { panelState, logRows, lastOutputAt, onTerminalReady, onTerminalError, onMessage, onPanelClose } = useDeployFeedback(); + const { panelState, healthGate, logRows, lastOutputAt, onTerminalReady, onTerminalError, onMessage, onPanelClose } = useDeployFeedback(); const [showRaw, setShowRaw] = useState(false); const [elapsedSeconds, setElapsedSeconds] = useState(0); @@ -110,8 +112,12 @@ export function DeployFeedbackModal({ isMinimized, onMinimize }: DeployFeedbackM }, 1000); }, [clearCountdownInterval, onPanelClose]); + // Auto-close only when there is nothing left to watch: success with no gate, + // or success whose gate passed. An observing gate suspends the countdown; a + // failed/unknown gate keeps the modal open until the user closes it. + const gateHoldsOpen = healthGate !== null && healthGate.status !== 'passed'; useEffect(() => { - if (status === 'succeeded' && isOpen) { + if (status === 'succeeded' && isOpen && !gateHoldsOpen) { // eslint-disable-next-line react-hooks/set-state-in-effect startCountdown(); } else { @@ -121,7 +127,7 @@ export function DeployFeedbackModal({ isMinimized, onMinimize }: DeployFeedbackM return () => { clearCountdownInterval(); }; - }, [status, isOpen, startCountdown, clearCountdownInterval]); + }, [status, isOpen, gateHoldsOpen, startCountdown, clearCountdownInterval]); useEffect(() => { if (!userScrolledUp && scrollRef.current) { @@ -143,10 +149,10 @@ export function DeployFeedbackModal({ isMinimized, onMinimize }: DeployFeedbackM const handleMouseLeave = useCallback(() => { autoCloseHoveredRef.current = false; - if (status === 'succeeded' && isOpen) { + if (status === 'succeeded' && isOpen && !gateHoldsOpen) { startCountdown(); } - }, [status, isOpen, startCountdown]); + }, [status, isOpen, gateHoldsOpen, startCountdown]); const handleOpenChange = useCallback( (open: boolean) => { @@ -207,6 +213,8 @@ export function DeployFeedbackModal({ isMinimized, onMinimize }: DeployFeedbackM rowCount={logRows.length} errorMessage={errorMessage} countdown={countdown} + showCountdown={!gateHoldsOpen} + gateStatus={healthGate?.status ?? null} /> + + + + + ); diff --git a/frontend/src/components/EditorLayout/ShellOverlays.tsx b/frontend/src/components/EditorLayout/ShellOverlays.tsx index f6c5fa4b..a9a4d568 100644 --- a/frontend/src/components/EditorLayout/ShellOverlays.tsx +++ b/frontend/src/components/EditorLayout/ShellOverlays.tsx @@ -2,6 +2,7 @@ import { lazy, Suspense } from 'react'; import BashExecModal from '../BashExecModal'; import LazyBoundary from '../LazyBoundary'; import { PolicyBlockDialog } from '../stack/PolicyBlockDialog'; +import { UpdateReadinessDialog } from '../stack/UpdateReadinessDialog'; import { DeleteStackDialog } from './DeleteStackDialog'; import { UnsavedChangesDialog } from './UnsavedChangesDialog'; import { StackAlertSheet } from '../StackAlertSheet'; @@ -55,6 +56,7 @@ export function ShellOverlays({ logViewerOpen, logContainer, stackMonitor, closeStackMonitor, policyBlock, setPolicyBlock, policyBypassing, + updateReadiness, setUpdateReadiness, stackMisconfigScanId, setStackMisconfigScanId, diffPreview, setDiffPreview, diffPreviewConfirming, setDiffPreviewConfirming, } = overlayState; @@ -102,6 +104,14 @@ export function ShellOverlays({ initialTab={stackMonitor?.tab ?? 'alerts'} /> + {/* Pre-update readiness check */} + setUpdateReadiness(null)} + onProceed={() => updateReadiness?.proceed()} + /> + {/* Pre-deploy policy block */} [0]> = {}) { return props; } +// The actions live behind the Take action menu so the mobile card stays small. +function openActions() { + fireEvent.click(screen.getByText('Take action')); +} + describe('RecoveryPanel', () => { beforeEach(() => vi.clearAllMocks()); - it('shows the failed action title and error message', () => { + it('shows the failed action title and error message on the card', () => { setup(); expect(screen.getByText(/Update failed/)).toBeInTheDocument(); expect(screen.getByText(/pull failed: connection reset/)).toBeInTheDocument(); }); - it('calls onRetry from the retry button', () => { + it('keeps the actions collapsed behind the Take action menu', () => { + setup(); + expect(screen.queryByText('Retry update')).not.toBeInTheDocument(); + openActions(); + expect(screen.getByText('Retry update')).toBeInTheDocument(); + }); + + it('calls onRetry from the retry action', () => { const props = setup(); + openActions(); fireEvent.click(screen.getByText('Retry update')); expect(props.onRetry).toHaveBeenCalledTimes(1); }); it('hides retry/restart/rollback without the deploy permission', () => { setup({ canDeploy: false, backupInfo: { exists: true, timestamp: 1 } }); + openActions(); expect(screen.queryByText('Retry update')).not.toBeInTheDocument(); expect(screen.queryByText('Restart')).not.toBeInTheDocument(); expect(screen.queryByText('Roll back')).not.toBeInTheDocument(); @@ -61,19 +75,23 @@ describe('RecoveryPanel', () => { it('offers rollback only when a backup exists', () => { setup({ backupInfo: { exists: false, timestamp: null } }); + openActions(); expect(screen.queryByText('Roll back')).not.toBeInTheDocument(); setup({ backupInfo: { exists: true, timestamp: 123 } }); + fireEvent.click(screen.getAllByText('Take action')[1]); expect(screen.getByText('Roll back')).toBeInTheDocument(); }); it('does not show a redundant restart button when the failed action was a restart', () => { setup({ result: { ...baseResult, action: 'restart' } }); + openActions(); expect(screen.getByText('Retry restart')).toBeInTheDocument(); - expect(screen.queryByText('Restart')).not.toBeInTheDocument(); + expect(screen.queryByText('Restart', { exact: true })).not.toBeInTheDocument(); }); it('copies session-safe diagnostics including stack and error', () => { setup({ result: { ...baseResult, lastOutputLine: 'pulling app ...' } }); + openActions(); fireEvent.click(screen.getByText('Copy details')); expect(copyToClipboard).toHaveBeenCalledTimes(1); const blob = vi.mocked(copyToClipboard).mock.calls[0][0]; @@ -82,8 +100,47 @@ describe('RecoveryPanel', () => { expect(blob).toContain('Last output: pulling app ...'); }); + it('renders the failure classification on the card without opening the menu', () => { + setup({ + result: { + ...baseResult, + failure: { + reason: 'port_conflict', + label: 'Host port conflict', + suggestion: 'Free the conflicting host port, then retry.', + }, + }, + }); + expect(screen.getByText('Host port conflict')).toBeInTheDocument(); + expect(screen.getByText('Free the conflicting host port, then retry.')).toBeInTheDocument(); + }); + + it('renders no classification block without a failure field', () => { + setup(); + expect(screen.queryByTestId('recovery-classification')).not.toBeInTheDocument(); + }); + + it('includes the classification in copied diagnostics', () => { + setup({ + result: { + ...baseResult, + failure: { + reason: 'env_missing', + label: 'Missing environment variable', + suggestion: 'Define the missing variable, then retry.', + }, + }, + }); + openActions(); + fireEvent.click(screen.getByText('Copy details')); + const blob = vi.mocked(copyToClipboard).mock.calls[0][0]; + expect(blob).toContain('Classified: Missing environment variable'); + expect(blob).toContain('Suggestion: Define the missing variable, then retry.'); + }); + it('wires refresh and dismiss callbacks', () => { const props = setup(); + openActions(); fireEvent.click(screen.getByText('Refresh')); fireEvent.click(screen.getByLabelText('Dismiss recovery panel')); expect(props.onRefreshState).toHaveBeenCalledTimes(1); diff --git a/frontend/src/components/EditorLayout/hooks/useOverlayState.ts b/frontend/src/components/EditorLayout/hooks/useOverlayState.ts index e152b295..19bebe59 100644 --- a/frontend/src/components/EditorLayout/hooks/useOverlayState.ts +++ b/frontend/src/components/EditorLayout/hooks/useOverlayState.ts @@ -88,6 +88,14 @@ export function useOverlayState() { const [policyBlock, setPolicyBlock] = useState(null); const [policyBypassing, setPolicyBypassing] = useState(false); + // Pre-update readiness dialog. `proceed` runs the actual update when the + // user confirms; opened by useStackActions.requestStackUpdate. + const [updateReadiness, setUpdateReadiness] = useState<{ + stackName: string; + stackFile: string; + proceed: () => void; + } | null>(null); + const [stackMisconfigScanId, setStackMisconfigScanId] = useState(null); const [diffPreview, setDiffPreview] = useState(null); @@ -103,6 +111,7 @@ export function useOverlayState() { logViewerOpen, logContainer, openLogViewer, closeLogViewer, stackMonitor, openAlertSheet, openAutoHeal, closeStackMonitor, policyBlock, setPolicyBlock, policyBypassing, setPolicyBypassing, + updateReadiness, setUpdateReadiness, stackMisconfigScanId, setStackMisconfigScanId, diffPreview, setDiffPreview, diffPreviewConfirming, setDiffPreviewConfirming, } as const; diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts index 992ad361..14e6c5b5 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts @@ -83,6 +83,8 @@ function makeOverlay(over: Partial = {}): OverlayState { policyBlock: null, setPolicyBlock: vi.fn(), setPolicyBypassing: vi.fn(), + updateReadiness: null, + setUpdateReadiness: vi.fn(), setDiffPreview: vi.fn(), ...over, } as unknown as OverlayState; @@ -96,6 +98,7 @@ function setup(over: { overlay?: Partial; stackList?: Partial; getLastDeployOutputLine?: (stackName: string) => string | undefined; + hasUpdateGuard?: boolean; } = {}) { const editorState = makeEditorState(over.editorState); const stackListState = makeStackListState(over.stackList); @@ -114,6 +117,7 @@ function setup(over: { runWithLog, getLastDeployOutputLine: over.getLastDeployOutputLine ?? (() => undefined), diffPreviewEnabled: false, + hasUpdateGuard: over.hasUpdateGuard ?? false, }), ); return { result, editorState, stackListState, overlayState }; @@ -365,6 +369,75 @@ describe('useStackActions.attemptLeaveEditor (mobile back / nav guard)', () => { }); }); +describe('useStackActions update readiness routing', () => { + beforeEach(() => { + vi.mocked(apiFetch).mockReset(); + }); + + function routeUpdateOk() { + vi.mocked(apiFetch).mockImplementation((url: string) => { + const u = String(url); + if (u.includes('/update')) return Promise.resolve(new Response('', { status: 200 })); + return Promise.resolve(new Response('[]', { status: 200 })); + }); + } + + it('opens the readiness dialog instead of posting when the node has update-guard', async () => { + routeUpdateOk(); + const { result, overlayState } = setup({ hasUpdateGuard: true }); + await act(async () => { await result.current.updateStack(); }); + expect(overlayState.setUpdateReadiness).toHaveBeenCalledWith( + expect.objectContaining({ stackName: 'web', stackFile: 'web.yml' }), + ); + expect(apiFetch).not.toHaveBeenCalled(); + }); + + it('routes a sidebar/context-menu update through the readiness dialog too', async () => { + routeUpdateOk(); + const { result, overlayState } = setup({ hasUpdateGuard: true }); + await act(async () => { await result.current.executeStackActionByFile('web.yml', 'update', 'update'); }); + expect(overlayState.setUpdateReadiness).toHaveBeenCalledWith( + expect.objectContaining({ stackName: 'web', stackFile: 'web.yml' }), + ); + expect(apiFetch).not.toHaveBeenCalled(); + }); + + it('runs the shared update executor when the dialog proceeds', async () => { + routeUpdateOk(); + const { result, overlayState, stackListState } = setup({ hasUpdateGuard: true }); + await act(async () => { await result.current.updateStack(); }); + const pending = vi.mocked(overlayState.setUpdateReadiness).mock.calls[0][0] as + { stackName: string; stackFile: string; proceed: () => void }; + expect(pending).not.toBeNull(); + await act(async () => { pending.proceed(); }); + expect(overlayState.setUpdateReadiness).toHaveBeenLastCalledWith(null); + const urls = vi.mocked(apiFetch).mock.calls.map(c => String(c[0])); + expect(urls).toContain('/stacks/web/update'); + expect(stackListState.recordActionSuccess).toHaveBeenCalledWith('web.yml'); + }); + + it('does nothing while the stack is busy, with or without the dialog', async () => { + routeUpdateOk(); + const { result, overlayState } = setup({ + hasUpdateGuard: true, + stackList: { isStackBusy: vi.fn().mockReturnValue(true) as never }, + }); + await act(async () => { await result.current.updateStack(); }); + expect(overlayState.setUpdateReadiness).not.toHaveBeenCalled(); + expect(apiFetch).not.toHaveBeenCalled(); + }); + + it('updates directly without the capability, from both entry points', async () => { + routeUpdateOk(); + const { result, overlayState } = setup(); + await act(async () => { await result.current.updateStack(); }); + await act(async () => { await result.current.executeStackActionByFile('web.yml', 'update', 'update'); }); + expect(overlayState.setUpdateReadiness).not.toHaveBeenCalled(); + const updatePosts = vi.mocked(apiFetch).mock.calls.filter(c => String(c[0]) === '/stacks/web/update'); + expect(updatePosts).toHaveLength(2); + }); +}); + describe('useStackActions recovery records', () => { beforeEach(() => { vi.mocked(apiFetch).mockReset(); @@ -453,6 +526,67 @@ describe('useStackActions recovery records', () => { ); }); + it('carries the server failure classification into the recovery record', async () => { + const body = JSON.stringify({ + error: 'port is already allocated', + rolledBack: false, + failure: { reason: 'port_conflict', label: 'Host port conflict', suggestion: 'Free the port, then retry.' }, + }); + routeApi(500, body); + const { result, stackListState } = setup(); + await act(async () => { await result.current.updateStack(); }); + expect(stackListState.recordActionFailure).toHaveBeenCalledWith( + 'web.yml', + expect.objectContaining({ + failure: { reason: 'port_conflict', label: 'Host port conflict', suggestion: 'Free the port, then retry.' }, + }), + ); + }); + + it('ignores a malformed failure field in the response body', async () => { + routeApi(500, JSON.stringify({ error: 'boom', failure: { reason: 42 } })); + const { result, stackListState } = setup(); + await act(async () => { await result.current.updateStack(); }); + expect(stackListState.recordActionFailure).toHaveBeenCalledWith( + 'web.yml', + expect.objectContaining({ failure: undefined }), + ); + }); + + it('synthesizes a node_unreachable classification for a gateway 502 with no body', async () => { + routeApi(502, 'Bad Gateway'); + const { result, stackListState } = setup(); + await act(async () => { await result.current.updateStack(); }); + expect(stackListState.recordActionFailure).toHaveBeenCalledWith( + 'web.yml', + expect.objectContaining({ + failure: expect.objectContaining({ reason: 'node_unreachable' }), + }), + ); + }); + + it('does not mislabel an unrelated JSON 503 as node_unreachable', async () => { + routeApi(503, JSON.stringify({ error: 'maintenance window' })); + const { result, stackListState } = setup(); + await act(async () => { await result.current.updateStack(); }); + expect(stackListState.recordActionFailure).toHaveBeenCalledWith( + 'web.yml', + expect.objectContaining({ failure: undefined }), + ); + }); + + it('synthesizes node_unreachable for a docker_unavailable 503 without a classified body', async () => { + routeApi(503, JSON.stringify({ error: 'daemon gone', code: 'docker_unavailable' })); + const { result, stackListState } = setup(); + await act(async () => { await result.current.updateStack(); }); + expect(stackListState.recordActionFailure).toHaveBeenCalledWith( + 'web.yml', + expect.objectContaining({ + failure: expect.objectContaining({ reason: 'node_unreachable' }), + }), + ); + }); + it('records a rollback failure', async () => { vi.mocked(apiFetch).mockImplementation((url: string) => { const u = String(url); diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.ts index 6bca226b..66ef7dbc 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.ts @@ -7,7 +7,7 @@ import type { useViewNavigationState } from './useViewNavigationState'; import type { OverlayState } from './useOverlayState'; import type { Node } from '@/context/NodeContext'; import type { ActionVerb } from '@/context/DeployFeedbackContext'; -import type { StackAction, RecoverableAction } from '../EditorView'; +import type { StackAction, RecoverableAction, FailureClassification } from '../EditorView'; import type { NotificationItem } from '../../dashboard/types'; import type { PolicyBlockPayload, PolicyBlockableAction } from '../../stack/PolicyBlockDialog'; @@ -15,15 +15,57 @@ interface RunResult { ok: boolean; errorMessage?: string; rolledBack?: boolean; + /** Health gate run id from the success body, when the backend started one. */ + healthGateId?: string | null; } +/** healthGateId from a success body, or null when absent or unreadable. */ +const parseHealthGateId = async (response: Response): Promise => { + try { + const body: unknown = await response.json(); + if (isRecord(body) && typeof body.healthGateId === 'string') return body.healthGateId; + } catch (e) { + // A success body should always parse; the warn surfaces a future + // double-read bug instead of silently disabling the gate UI. + console.warn('[HealthGate] could not read the success body:', e); + } + return null; +}; + // Sentinel stored in overlayState.pendingUnsavedLoad to mark that the pending // confirmation is a node switch (not a stack load). When the user confirms the // discard, discardAndLoadPending calls setActiveNode(targetNode) and skips the // stack-load branch. export const NODE_SWITCH_PENDING_TOKEN = '__node-switch-pending__'; -type StackActionError = Error & { rolledBack?: boolean }; +type StackActionError = Error & { rolledBack?: boolean; failure?: FailureClassification }; + +// Fallback classification when the response never reached a Sencho backend +// (proxy 502/504 for a dead remote, or a 503 with no classified body). +const NODE_UNREACHABLE_FAILURE: FailureClassification = { + reason: 'node_unreachable', + label: 'Node or Docker unreachable', + suggestion: 'Check that the node is online and Docker is running, then retry.', +}; + +const UNREACHABLE_STATUSES: ReadonlySet = new Set([502, 503, 504]); + +const parseFailureClassification = (value: unknown): FailureClassification | undefined => { + if ( + isRecord(value) && + typeof value.reason === 'string' && + typeof value.label === 'string' && value.label.trim() && + typeof value.suggestion === 'string' && value.suggestion.trim() + ) { + return { reason: value.reason, label: value.label, suggestion: value.suggestion }; + } + if (value !== undefined) { + // Likely hub/node version skew or a mangled proxy body; the raw error + // message still renders, only the classification panel is degraded. + console.warn('Unrecognized failure classification shape in error response:', value); + } + return undefined; +}; type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update'; @@ -66,6 +108,10 @@ interface UseStackActionsOptions { // is streaming that exact stack; used to enrich failure diagnostics safely. getLastDeployOutputLine: (stackName: string) => string | undefined; diffPreviewEnabled: boolean; + // Active node advertises the update-guard capability, so manual updates show + // the pre-update readiness dialog. Defaults to false: without the + // capability, updates run directly with no dialog. + hasUpdateGuard?: boolean; } const isRecord = (value: unknown): value is Record => @@ -100,24 +146,40 @@ const stackOpInProgressMessage = (stackName: string, info: StackOpInProgressInfo return `${stackName} is already ${verb}${actor}.`; }; -const parseStackActionError = (rawBody: string, fallback: string): StackActionError => { +const parseStackActionError = (rawBody: string, fallback: string, status?: number): StackActionError => { let message = rawBody || fallback; let rolledBack = false; + let failure: FailureClassification | undefined; + let parsedCode: string | undefined; + let bodyWasJson = false; try { const parsed: unknown = JSON.parse(rawBody); + bodyWasJson = true; if (isRecord(parsed)) { if (typeof parsed.error === 'string' && parsed.error.trim()) { message = parsed.error; } rolledBack = parsed.rolledBack === true; + failure = parseFailureClassification(parsed.failure); + if (typeof parsed.code === 'string') parsedCode = parsed.code; } } catch { /* not JSON */ } + // A gateway-style status with no classified body means the request likely + // never reached the owning node's backend; surface that as the cause. A 503 + // qualifies only when it is body-less (proxy generated) or the backend's own + // docker_unavailable shape, so an unrelated future 503 is not mislabeled. + if (!failure && status !== undefined && UNREACHABLE_STATUSES.has(status)) { + const qualifies = status !== 503 || !bodyWasJson || parsedCode === 'docker_unavailable'; + if (qualifies) failure = { ...NODE_UNREACHABLE_FAILURE }; + } + const error = new Error(message) as StackActionError; error.rolledBack = rolledBack; + error.failure = failure; return error; }; @@ -133,6 +195,7 @@ export function useStackActions(options: UseStackActionsOptions) { runWithLog, getLastDeployOutputLine, diffPreviewEnabled, + hasUpdateGuard = false, } = options; const pendingStackLoadRef = useRef(null); @@ -235,6 +298,7 @@ export function useStackActions(options: UseStackActionsOptions) { startedAt: number, errorMessage: string | undefined, rolledBack: boolean, + failure?: FailureClassification, ) => { if (!isRecoverableAction(action)) return; stackListState.recordActionFailure(stackFile, { @@ -244,6 +308,7 @@ export function useStackActions(options: UseStackActionsOptions) { startedAt, endedAt: Date.now(), lastOutputLine: getLastDeployOutputLine(stackName), + failure, }); }; @@ -619,12 +684,17 @@ export function useStackActions(options: UseStackActionsOptions) { return { ok: false, errorMessage: message }; } } - throw parseStackActionError(rawBody, 'Deploy failed'); + throw parseStackActionError(rawBody, 'Deploy failed', response.status); } overlayState.setPolicyBlock(null); - toast.success( - ignorePolicy ? 'Stack deployed (policy bypassed).' : 'Stack deployed successfully!', - ); + const healthGateId = await parseHealthGateId(response); + // With a health gate observing, the operation finishing is not the + // final verdict yet; soften the toast so success is not claimed twice. + if (healthGateId) { + toast.info(ignorePolicy ? 'Stack deployed (policy bypassed). Verifying health...' : 'Stack deployed. Verifying health...'); + } else { + toast.success(ignorePolicy ? 'Stack deployed (policy bypassed).' : 'Stack deployed successfully!'); + } await refreshSelectedContainers(stackName, stackFile); try { const backupRes = await apiFetch(`/stacks/${stackName}/backup`); @@ -633,7 +703,7 @@ export function useStackActions(options: UseStackActionsOptions) { /* ignore */ } stackListState.recordActionSuccess(stackFile); - return { ok: true }; + return { ok: true, healthGateId }; } catch (error) { console.error('Failed to deploy:', error); if (previousStatus !== undefined) @@ -645,7 +715,7 @@ export function useStackActions(options: UseStackActionsOptions) { ? `${errorMessage} - automatically rolled back to previous version.` : errorMessage, ); - recordActionFailureFor(stackFile, stackName, 'deploy', startedAt, errorMessage, deployError.rolledBack === true); + recordActionFailureFor(stackFile, stackName, 'deploy', startedAt, errorMessage, deployError.rolledBack === true, deployError.failure); await refreshSelectedContainers(stackName, stackFile); return { ok: false, errorMessage, rolledBack: deployError.rolledBack }; } @@ -736,7 +806,7 @@ export function useStackActions(options: UseStackActionsOptions) { return; } } - throw parseStackActionError(rawBody, 'Rollback failed'); + throw parseStackActionError(rawBody, 'Rollback failed', res.status); } overlayState.setPolicyBlock(null); toast.success('Stack rolled back successfully.'); @@ -758,7 +828,8 @@ export function useStackActions(options: UseStackActionsOptions) { } catch (error: unknown) { const msg = error instanceof Error ? error.message : 'Rollback failed'; toast.error(msg); - recordActionFailureFor(stackFile, stackName, 'rollback', startedAt, msg, false); + recordActionFailureFor(stackFile, stackName, 'rollback', startedAt, msg, false, + error instanceof Error ? (error as StackActionError).failure : undefined); await refreshSelectedContainers(stackName, stackFile); } finally { stackListState.clearStackAction(stackFile); @@ -850,8 +921,8 @@ export function useStackActions(options: UseStackActionsOptions) { } } } - const actionError = parseStackActionError(errText, `${action} failed`); - recordActionFailureFor(stackFile, stackName, action, startedAt, actionError.message, actionError.rolledBack === true); + const actionError = parseStackActionError(errText, `${action} failed`, response.status); + recordActionFailureFor(stackFile, stackName, action, startedAt, actionError.message, actionError.rolledBack === true, actionError.failure); await refreshSelectedContainers(stackName, stackFile); return { ok: false as const, @@ -860,11 +931,18 @@ export function useStackActions(options: UseStackActionsOptions) { }; } overlayState.setPolicyBlock(null); - toast.success(successMessage); + const healthGateId = await parseHealthGateId(response); + // With a health gate observing, the operation finishing is not the + // final verdict yet; soften the toast so success is not claimed twice. + if (healthGateId && action === 'update') { + toast.info('Stack updated. Verifying health...'); + } else { + toast.success(successMessage); + } if (action === 'update') stackListState.fetchImageUpdates(); await refreshSelectedContainers(stackName, stackFile); stackListState.recordActionSuccess(stackFile); - return { ok: true as const }; + return { ok: true as const, healthGateId }; } catch (err) { const message = (err as Error).message || `${action} failed`; recordActionFailureFor(stackFile, stackName, action, startedAt, message, false); @@ -923,11 +1001,33 @@ export function useStackActions(options: UseStackActionsOptions) { } }; + // Single entry point for every manual update trigger (toolbar, sidebar + // context menu, recovery retry). With the update-guard capability it opens + // the readiness dialog first; the dialog's proceed runs the same + // runWithLog-backed executor either way, so there is exactly one update path. + const requestStackUpdate = async (stackFile: string): Promise => { + if (stackListState.isStackBusy(stackFile)) return; + const stackName = stackFile.replace(/\.(yml|yaml)$/, ''); + const run = () => runStackAction(stackFile, 'update', 'update', 'running', 'Stack updated successfully!'); + if (hasUpdateGuard) { + overlayState.setUpdateReadiness({ + stackName, + stackFile, + proceed: () => { + overlayState.setUpdateReadiness(null); + void run(); + }, + }); + return; + } + await run(); + }; + const updateStack = async (e?: React.MouseEvent) => { e?.preventDefault(); e?.stopPropagation(); if (!stackListState.selectedFile) return; - await runStackAction(stackListState.selectedFile, 'update', 'update', 'running', 'Stack updated successfully!'); + await requestStackUpdate(stackListState.selectedFile); }; const deleteStack = async (pruneVolumes: boolean) => { @@ -1016,13 +1116,20 @@ export function useStackActions(options: UseStackActionsOptions) { endpoint: string, ) => { if (stackListState.isStackBusy(stackFile)) return; + // Updates route through the shared update path so the sidebar gets the + // readiness dialog, the deploy-feedback modal, and the same failure + // handling as the toolbar. + if (action === 'update') { + await requestStackUpdate(stackFile); + return; + } const stackName = stackFile.replace(/\.(yml|yaml)$/, ''); const startedAt = Date.now(); stackListState.setStackAction(stackFile, action); if (action === 'stop') { stackListState.setOptimisticStatus(stackFile, 'exited'); - } else if (action === 'deploy' || action === 'restart' || action === 'update') { + } else if (action === 'deploy' || action === 'restart') { stackListState.setOptimisticStatus(stackFile, 'running'); } @@ -1036,19 +1143,18 @@ export function useStackActions(options: UseStackActionsOptions) { toast.error(stackOpInProgressMessage(stackName, inProgress)); return; } - if (action === 'deploy' || action === 'update') { + if (action === 'deploy') { const blockedBy = tryOpenPolicyBlock(errText, stackName, stackFile, action); if (blockedBy) { - toast.error(`${action === 'update' ? 'Update' : 'Deploy'} blocked by policy "${blockedBy}"`); + toast.error(`Deploy blocked by policy "${blockedBy}"`); return; } } } - throw parseStackActionError(errText, `${action} failed`); + throw parseStackActionError(errText, `${action} failed`, response.status); } toast.success(`Stack ${action}ed successfully!`); await refreshSelectedContainers(stackName, stackFile); - if (action === 'update') stackListState.fetchImageUpdates(); if (action === 'deploy') { try { const backupRes = await apiFetch(`/stacks/${stackName}/backup`); @@ -1067,7 +1173,7 @@ export function useStackActions(options: UseStackActionsOptions) { ? `${msg} - automatically rolled back to previous version.` : msg, ); - recordActionFailureFor(stackFile, stackName, action, startedAt, msg, actionError.rolledBack === true); + recordActionFailureFor(stackFile, stackName, action, startedAt, msg, actionError.rolledBack === true, actionError.failure); await refreshSelectedContainers(stackName, stackFile); } finally { stackListState.clearStackAction(stackFile); @@ -1154,6 +1260,7 @@ export function useStackActions(options: UseStackActionsOptions) { restartStack, serviceAction, updateStack, + requestStackUpdate, deleteStack, attemptLeaveEditor, cancelPendingUnsavedLoad, diff --git a/frontend/src/components/EditorLayout/recovery-format.ts b/frontend/src/components/EditorLayout/recovery-format.ts index 335075b3..67164f2f 100644 --- a/frontend/src/components/EditorLayout/recovery-format.ts +++ b/frontend/src/components/EditorLayout/recovery-format.ts @@ -34,6 +34,10 @@ export function buildDiagnostics( ? `available${backupInfo.timestamp ? ` (${new Date(backupInfo.timestamp).toISOString()})` : ''}` : 'none'}`, ]; + if (result.failure) { + lines.push(`Classified: ${result.failure.label}`); + lines.push(`Suggestion: ${result.failure.suggestion}`); + } if (result.lastOutputLine) lines.push(`Last output: ${result.lastOutputLine}`); return lines.join('\n'); } diff --git a/frontend/src/components/__tests__/DeployFeedbackModal.test.tsx b/frontend/src/components/__tests__/DeployFeedbackModal.test.tsx index 5e5992b8..d0ef58f3 100644 --- a/frontend/src/components/__tests__/DeployFeedbackModal.test.tsx +++ b/frontend/src/components/__tests__/DeployFeedbackModal.test.tsx @@ -4,6 +4,9 @@ import { render, screen, act } from '@testing-library/react'; import { DeployFeedbackProvider, useDeployFeedback } from '@/context/DeployFeedbackContext'; import { DeployFeedbackModal } from '../DeployFeedbackModal'; +vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); +import { apiFetch } from '@/lib/api'; + // Lets a test simulate a mid-stream drop (onReady then onError) so the panel // reaches 'streaming' with progressUnavailable set. const ctl = vi.hoisted(() => ({ drop: false })); @@ -23,14 +26,16 @@ vi.mock('@/components/Terminal', () => { // Resolver for the in-flight operation, assigned inside the run callback (async, // after render) so the test can leave it pending or settle it on demand. -let resolveRun: ((r: { ok: boolean; errorMessage?: string }) => void) | null = null; +let resolveRun: ((r: { ok: boolean; errorMessage?: string; healthGateId?: string | null }) => void) | null = null; +// The runWithLog promise itself, so a test can await full result propagation. +let runOuter: Promise | null = null; function Driver() { const { runWithLog } = useDeployFeedback(); React.useEffect(() => { - void runWithLog({ stackName: 'web', action: 'update' }, async (started) => { + runOuter = runWithLog({ stackName: 'web', action: 'update' }, async (started) => { await started; - return new Promise<{ ok: boolean; errorMessage?: string }>((res) => { resolveRun = res; }); + return new Promise<{ ok: boolean; errorMessage?: string; healthGateId?: string | null }>((res) => { resolveRun = res; }); }); }, [runWithLog]); return null; @@ -49,6 +54,193 @@ async function renderStreaming() { }); } +type GateStatus = 'observing' | 'passed' | 'failed' | 'unknown'; + +function routeGateApi(responses: Array<{ id: string; status: GateStatus; reason?: string | null }>) { + let call = 0; + vi.mocked(apiFetch).mockImplementation((url: string) => { + if (!String(url).includes('/health-gate')) { + return Promise.resolve(new Response('{}', { status: 200 })); + } + const r = responses[Math.min(call, responses.length - 1)]; + call += 1; + return Promise.resolve(new Response(JSON.stringify({ + stack: 'web', id: r.id, status: r.status, trigger: 'update', + reason: r.reason ?? null, windowSeconds: 90, startedAt: Date.now(), endedAt: null, containers: [], + }), { status: 200 })); + }); +} + +describe('DeployFeedbackModal health gate', () => { + beforeEach(() => { + vi.useFakeTimers(); + localStorage.clear(); + resolveRun = null; + ctl.drop = false; + vi.mocked(apiFetch).mockReset(); + vi.mocked(apiFetch).mockResolvedValue(new Response('{}', { status: 200 })); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + async function succeedWithGate(gateId: string | null) { + await renderStreaming(); + // The Terminal onReady effect flushes at the end of renderStreaming's act, + // scheduling the 50ms handshake timer after that act's advance already + // ran; fire it here so the run reaches its resolver. + await act(async () => { await vi.advanceTimersByTimeAsync(60); }); + expect(resolveRun).not.toBeNull(); + await act(async () => { + resolveRun?.({ ok: true, healthGateId: gateId }); + await runOuter; + }); + } + + it('shows the observing banner and suspends auto-close while the gate observes', async () => { + routeGateApi([{ id: 'gate-1', status: 'observing' }]); + await succeedWithGate('gate-1'); + await act(async () => { await vi.advanceTimersByTimeAsync(100); }); + expect(screen.getByTestId('health-gate-banner')).toHaveAttribute('data-status', 'observing'); + expect(screen.queryByText(/closes in/)).toBeNull(); + // The verdict is withheld while observing: no green Succeeded yet. + expect(screen.queryByText('Succeeded')).toBeNull(); + expect(screen.getByText('Verifying health')).toBeInTheDocument(); + // Far past the normal 4s auto-close: the modal must still be open. + await act(async () => { await vi.advanceTimersByTimeAsync(10_000); }); + expect(screen.getByTestId('deploy-feedback-modal')).toBeInTheDocument(); + }); + + it('resumes the auto-close countdown once the gate passes', async () => { + routeGateApi([{ id: 'gate-1', status: 'observing' }, { id: 'gate-1', status: 'passed' }]); + await succeedWithGate('gate-1'); + await act(async () => { await vi.advanceTimersByTimeAsync(100); }); + expect(screen.getByTestId('health-gate-banner')).toHaveAttribute('data-status', 'observing'); + // The next 4s poll returns passed; the countdown then runs to auto-close. + await act(async () => { await vi.advanceTimersByTimeAsync(4_100); }); + expect(screen.getByTestId('health-gate-banner')).toHaveAttribute('data-status', 'passed'); + expect(screen.getByText(/closes in/)).toBeInTheDocument(); + await act(async () => { await vi.advanceTimersByTimeAsync(5_000); }); + // onPanelClose resets the panel (Radix may keep the dialog DOM mounted + // briefly under fake timers, so assert on the reset, not the unmount). + expect(screen.queryByText('Succeeded')).toBeNull(); + expect(screen.queryByTestId('health-gate-banner')).toBeNull(); + }); + + it('keeps the modal open and shows the reason when the gate fails', async () => { + routeGateApi([{ id: 'gate-1', status: 'failed', reason: 'container web-app-1 exited during observation' }]); + await succeedWithGate('gate-1'); + await act(async () => { await vi.advanceTimersByTimeAsync(100); }); + expect(screen.getByTestId('health-gate-banner')).toHaveAttribute('data-status', 'failed'); + expect(screen.getByText(/exited during observation/)).toBeInTheDocument(); + // The headline indicator reports the gate verdict, never a green Succeeded. + expect(screen.queryByText('Succeeded')).toBeNull(); + expect(screen.getAllByText(/Health gate failed/).length).toBeGreaterThan(0); + await act(async () => { await vi.advanceTimersByTimeAsync(20_000); }); + expect(screen.getByTestId('deploy-feedback-modal')).toBeInTheDocument(); + }); + + it('gives up with an unknown verdict after repeated poll failures', async () => { + vi.mocked(apiFetch).mockImplementation((url: string) => { + if (String(url).includes('/health-gate')) { + return Promise.resolve(new Response('{"error":"boom"}', { status: 500 })); + } + return Promise.resolve(new Response('{}', { status: 200 })); + }); + await succeedWithGate('gate-1'); + // Four strikes at the 4s poll cadence flip the gate to a client-side + // unknown and stop the interval (gateHoldsOpen keeps the modal up). + await act(async () => { await vi.advanceTimersByTimeAsync(17_000); }); + expect(screen.getByTestId('health-gate-banner')).toHaveAttribute('data-status', 'unknown'); + expect(screen.getByText(/could not be retrieved/)).toBeInTheDocument(); + const callsAfterGiveUp = vi.mocked(apiFetch).mock.calls.length; + await act(async () => { await vi.advanceTimersByTimeAsync(20_000); }); + expect(vi.mocked(apiFetch).mock.calls.length).toBe(callsAfterGiveUp); + }); + + it('ignores a report for a different gate id', async () => { + routeGateApi([{ id: 'some-other-gate', status: 'failed', reason: 'stale' }]); + await succeedWithGate('gate-1'); + await act(async () => { await vi.advanceTimersByTimeAsync(100); }); + // The mismatched report never replaces the optimistic observing state. + expect(screen.getByTestId('health-gate-banner')).toHaveAttribute('data-status', 'observing'); + }); + + it('polls single-flight and latches the terminal verdict against a late response', async () => { + // Hold each health-gate response open so we can release it deliberately and + // count how many requests overlap. + const release: Array<(body: object) => void> = []; + let healthGateCalls = 0; + vi.mocked(apiFetch).mockImplementation((url: string) => { + if (!String(url).includes('/health-gate')) { + return Promise.resolve(new Response('{}', { status: 200 })); + } + healthGateCalls += 1; + return new Promise((resolve) => { + release.push((body) => resolve(new Response(JSON.stringify(body), { status: 200 }))); + }); + }); + + await succeedWithGate('gate-1'); + // The first poll is still pending; advancing several intervals must not + // start a second one (single-flight), so no two responses can race. + await act(async () => { await vi.advanceTimersByTimeAsync(12_000); }); + expect(healthGateCalls).toBe(1); + + // Release the first poll as a terminal passed; the latch stops the interval. + await act(async () => { + release[0]({ + stack: 'web', id: 'gate-1', status: 'passed', trigger: 'update', + reason: null, windowSeconds: 90, startedAt: Date.now(), endedAt: null, containers: [], + }); + }); + expect(screen.getByTestId('health-gate-banner')).toHaveAttribute('data-status', 'passed'); + + // No further polls after a terminal verdict: a stale observing response can + // never arrive to roll the UI back. + await act(async () => { await vi.advanceTimersByTimeAsync(12_000); }); + expect(healthGateCalls).toBe(1); + }); + + it('does not poll the gate or show the panel when deploy feedback is disabled', async () => { + // Turning off the deploy progress panel opts out of the live gate UI: there + // is no surface to render it on. The backend gate still runs and records + // timeline events; only the in-browser verifying/recovery view is skipped. + localStorage.setItem('sencho.deploy-feedback.enabled', 'false'); + await act(async () => { + render( + + + {}} /> + , + ); + }); + await act(async () => { + expect(resolveRun).not.toBeNull(); + resolveRun?.({ ok: true, healthGateId: 'gate-1' }); + await runOuter; + }); + expect(screen.queryByTestId('deploy-feedback-modal')).toBeNull(); + expect(screen.queryByTestId('health-gate-banner')).toBeNull(); + // No poll is ever issued for the gate even though the backend returned an id. + await act(async () => { await vi.advanceTimersByTimeAsync(12_000); }); + expect(apiFetch).not.toHaveBeenCalled(); + }); + + it('renders no gate banner and auto-closes normally without a healthGateId', async () => { + await succeedWithGate(null); + await act(async () => { await vi.advanceTimersByTimeAsync(100); }); + expect(screen.getByText('Succeeded')).toBeInTheDocument(); + expect(screen.getByText(/closes in/)).toBeInTheDocument(); + expect(screen.queryByTestId('health-gate-banner')).toBeNull(); + expect(apiFetch).not.toHaveBeenCalled(); + await act(async () => { await vi.advanceTimersByTimeAsync(5_000); }); + // onPanelClose resets the panel (Radix may keep the dialog DOM mounted + // briefly under fake timers, so assert on the reset, not the unmount). + expect(screen.queryByText('Succeeded')).toBeNull(); + }); +}); + describe('DeployFeedbackModal stalled-output warning', () => { beforeEach(() => { vi.useFakeTimers(); diff --git a/frontend/src/components/settings/HostAlertsSection.tsx b/frontend/src/components/settings/HostAlertsSection.tsx index ac728112..ce8dde23 100644 --- a/frontend/src/components/settings/HostAlertsSection.tsx +++ b/frontend/src/components/settings/HostAlertsSection.tsx @@ -29,7 +29,7 @@ function SectionSkeleton() { ); } -type HostAlertFields = Pick; +type HostAlertFields = Pick; const DEFAULT_HOST_ALERTS: HostAlertFields = { host_cpu_limit: DEFAULT_SETTINGS.host_cpu_limit, @@ -37,6 +37,8 @@ const DEFAULT_HOST_ALERTS: HostAlertFields = { host_disk_limit: DEFAULT_SETTINGS.host_disk_limit, host_alert_suppression_mins: DEFAULT_SETTINGS.host_alert_suppression_mins, global_crash: DEFAULT_SETTINGS.global_crash, + health_gate_enabled: DEFAULT_SETTINGS.health_gate_enabled, + health_gate_window_seconds: DEFAULT_SETTINGS.health_gate_window_seconds, }; export function HostAlertsSection({ onDirtyChange }: HostAlertsSectionProps) { @@ -56,6 +58,8 @@ export function HostAlertsSection({ onDirtyChange }: HostAlertsSectionProps) { if (settings.host_disk_limit !== baseline.host_disk_limit) n++; if (settings.host_alert_suppression_mins !== baseline.host_alert_suppression_mins) n++; if (settings.global_crash !== baseline.global_crash) n++; + if (settings.health_gate_enabled !== baseline.health_gate_enabled) n++; + if (settings.health_gate_window_seconds !== baseline.health_gate_window_seconds) n++; return n; }, [settings]); @@ -89,6 +93,8 @@ export function HostAlertsSection({ onDirtyChange }: HostAlertsSectionProps) { host_disk_limit: nodeData.host_disk_limit ?? DEFAULT_SETTINGS.host_disk_limit, host_alert_suppression_mins: nodeData.host_alert_suppression_mins ?? DEFAULT_SETTINGS.host_alert_suppression_mins, global_crash: (nodeData.global_crash as '0' | '1') ?? DEFAULT_SETTINGS.global_crash, + health_gate_enabled: (nodeData.health_gate_enabled as '0' | '1') ?? DEFAULT_SETTINGS.health_gate_enabled, + health_gate_window_seconds: nodeData.health_gate_window_seconds ?? DEFAULT_SETTINGS.health_gate_window_seconds, }; setSettings(safe); serverSettingsRef.current = { ...safe }; @@ -197,6 +203,30 @@ export function HostAlertsSection({ onDirtyChange }: HostAlertsSectionProps) { + + + onSettingChange('health_gate_enabled', next ? '1' : '0')} + /> + + + onSettingChange('health_gate_window_seconds', v)} + suffix="s" + min={15} + max={600} + /> + + + {!readOnly && ( diff --git a/frontend/src/components/settings/__tests__/SectionSavePayloads.test.tsx b/frontend/src/components/settings/__tests__/SectionSavePayloads.test.tsx index 95d06368..1853c7c1 100644 --- a/frontend/src/components/settings/__tests__/SectionSavePayloads.test.tsx +++ b/frontend/src/components/settings/__tests__/SectionSavePayloads.test.tsx @@ -60,14 +60,16 @@ beforeEach(() => { }); describe('split section save payloads', () => { - it('HostAlertsSection patches only host alert keys', async () => { + it('HostAlertsSection patches only host alert and health gate keys', async () => { render(); const save = await screen.findByRole('button', { name: /save alerts/i }); - fireEvent.click(screen.getByRole('switch')); // global_crash + fireEvent.click(screen.getAllByRole('switch')[0]); // global_crash fireEvent.click(save); await waitFor(() => expect(mockedFetch.mock.calls.some(c => c[1]?.method === 'PATCH')).toBe(true)); expect(patchedKeys()).toEqual([ 'global_crash', + 'health_gate_enabled', + 'health_gate_window_seconds', 'host_alert_suppression_mins', 'host_cpu_limit', 'host_disk_limit', diff --git a/frontend/src/components/settings/types.ts b/frontend/src/components/settings/types.ts index cf7b2d8c..d2f1b1d8 100644 --- a/frontend/src/components/settings/types.ts +++ b/frontend/src/components/settings/types.ts @@ -15,6 +15,8 @@ export interface PatchableSettings { prune_on_update?: '0' | '1'; reclaim_hero?: '0' | '1'; snapshot_documentation?: '0' | '1'; + health_gate_enabled?: '0' | '1'; + health_gate_window_seconds?: string; } export const DEFAULT_SETTINGS: PatchableSettings = { @@ -34,6 +36,8 @@ export const DEFAULT_SETTINGS: PatchableSettings = { prune_on_update: '1', reclaim_hero: '1', snapshot_documentation: '0', + health_gate_enabled: '1', + health_gate_window_seconds: '90', }; export type SectionId = diff --git a/frontend/src/components/stack/RollbackReadinessSection.tsx b/frontend/src/components/stack/RollbackReadinessSection.tsx new file mode 100644 index 00000000..d893b208 --- /dev/null +++ b/frontend/src/components/stack/RollbackReadinessSection.tsx @@ -0,0 +1,107 @@ +import { useEffect, useState } from 'react'; +import { Check, CircleHelp, Database, X, type LucideIcon } from 'lucide-react'; +import { apiFetch } from '@/lib/api'; +import { cn } from '@/lib/utils'; +import { useNodes } from '@/context/NodeContext'; + +// Mirrors the backend payload shape (the frontend never imports backend). +type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered'; +type RollbackOverall = 'ready' | 'partial' | 'not_ready'; + +interface RollbackReadinessItem { + id: string; + state: RollbackItemState; + label: string; + detail: string; +} + +interface RollbackReadinessReport { + stack: string; + computedAt: number; + overall: RollbackOverall; + items: RollbackReadinessItem[]; +} + +const LABEL_CLASS = 'font-mono text-[10px] uppercase tracking-[0.18em] text-stat-subtitle'; + +const OVERALL_META: Record = { + ready: { label: 'ready', tone: 'border-success/40 bg-success/[0.06] text-success' }, + partial: { label: 'partial', tone: 'border-warning/40 bg-warning/[0.06] text-warning' }, + not_ready: { label: 'not ready', tone: 'border-destructive/40 bg-destructive/[0.06] text-destructive' }, +}; + +const STATE_META: Record = { + ready: { icon: Check, tone: 'text-success' }, + missing: { icon: X, tone: 'text-destructive' }, + unknown: { icon: CircleHelp, tone: 'text-stat-subtitle' }, + not_covered: { icon: Database, tone: 'text-warning' }, +}; + +/** + * "Would the existing rollback actually save me?" disclosure for the Stack + * Dossier. Read-only; renders nothing while loading, on error, or when the + * active node does not advertise the update-guard capability. + */ +export function RollbackReadinessSection({ stackName }: { stackName: string }) { + const { activeNode, hasCapability } = useNodes(); + const nodeId = activeNode?.id; + const enabled = hasCapability('update-guard'); + const [report, setReport] = useState(null); + + useEffect(() => { + setReport(null); + if (!enabled) return; + let cancelled = false; + const load = async () => { + try { + const res = await apiFetch(`/stacks/${stackName}/rollback-readiness`); + if (cancelled) return; + if (!res.ok) { + console.warn('[RollbackReadiness] unavailable for %s:', stackName, res.status); + return; + } + setReport(await res.json() as RollbackReadinessReport); + } catch (e) { + // Render-nothing on failure: the dossier remains fully usable without + // this section. The warn keeps the cause findable in the console. + if (!cancelled) console.warn('[RollbackReadiness] unavailable for %s:', stackName, e); + } + }; + void load(); + return () => { cancelled = true; }; + }, [stackName, nodeId, enabled]); + + if (!enabled || !report) return null; + + const overall = OVERALL_META[report.overall] ?? OVERALL_META.partial; + + return ( +
+
+ rollback readiness + + {overall.label} + +
+
+ {report.items.map(item => { + const meta = STATE_META[item.state] ?? STATE_META.unknown; + const StateIcon = meta.icon; + return ( +
+
+ + {item.label} +
+
{item.detail}
+
+ ); + })} +
+
+ ); +} diff --git a/frontend/src/components/stack/StackActivityTimeline.tsx b/frontend/src/components/stack/StackActivityTimeline.tsx index 11ffde65..fd5bf49e 100644 --- a/frontend/src/components/stack/StackActivityTimeline.tsx +++ b/frontend/src/components/stack/StackActivityTimeline.tsx @@ -1,7 +1,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { Rocket, RefreshCcw, CircleStop, Play, ArrowUp, Activity, Loader2, AlertCircle, - TriangleAlert, CircleCheck, + TriangleAlert, CircleCheck, HeartPulse, HeartCrack, } from 'lucide-react'; import type { LucideIcon } from 'lucide-react'; import { Button } from '@/components/ui/button'; @@ -36,6 +36,9 @@ const CATEGORY_ICON: Record = { image_update_applied: ArrowUp, drift_detected: TriangleAlert, drift_resolved: CircleCheck, + update_started: ArrowUp, + health_gate_passed: HeartPulse, + health_gate_failed: HeartCrack, }; const DAY_MS = 86_400_000; diff --git a/frontend/src/components/stack/StackDossierPanel.test.tsx b/frontend/src/components/stack/StackDossierPanel.test.tsx index 43de3178..4467d0f2 100644 --- a/frontend/src/components/stack/StackDossierPanel.test.tsx +++ b/frontend/src/components/stack/StackDossierPanel.test.tsx @@ -10,7 +10,9 @@ import { render, screen, fireEvent, waitFor } from '@testing-library/react'; vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); vi.mock('@/components/ui/toast-store', () => ({ toast: { success: vi.fn(), error: vi.fn() } })); -vi.mock('@/context/NodeContext', () => ({ useNodes: () => ({ activeNode: { id: 1 } }) })); +// hasCapability false keeps the rollback readiness section (tested in its own +// file) out of these dossier-focused tests. +vi.mock('@/context/NodeContext', () => ({ useNodes: () => ({ activeNode: { id: 1 }, hasCapability: () => false }) })); vi.mock('@/lib/clipboard', () => ({ copyToClipboard: vi.fn().mockResolvedValue(undefined) })); vi.mock('@/lib/download', () => ({ downloadTextFile: vi.fn() })); diff --git a/frontend/src/components/stack/StackDossierPanel.tsx b/frontend/src/components/stack/StackDossierPanel.tsx index c213d365..71af0b2c 100644 --- a/frontend/src/components/stack/StackDossierPanel.tsx +++ b/frontend/src/components/stack/StackDossierPanel.tsx @@ -13,6 +13,7 @@ import { } from '@/lib/dossierMarkdown'; import type { AnatomyMarkdownInput } from '@/lib/anatomyMarkdown'; import { computeDocDrift, type DocDriftFinding } from '@/lib/docDrift'; +import { RollbackReadinessSection } from './RollbackReadinessSection'; import { useNodes } from '@/context/NodeContext'; interface StackDossierPanelProps { @@ -357,6 +358,8 @@ export default function StackDossierPanel({ stackName, anatomy, canEdit }: Stack )} + + ); } diff --git a/frontend/src/components/stack/UpdateReadinessDialog.tsx b/frontend/src/components/stack/UpdateReadinessDialog.tsx new file mode 100644 index 00000000..227f84c4 --- /dev/null +++ b/frontend/src/components/stack/UpdateReadinessDialog.tsx @@ -0,0 +1,303 @@ +import { useEffect, useState } from 'react'; +import { + Check, CircleHelp, Info, ShieldAlert, TriangleAlert, Camera, type LucideIcon, +} from 'lucide-react'; +import { Modal, ModalHeader, ModalBody, ModalFooter } from '@/components/ui/modal'; +import { Button } from '@/components/ui/button'; +import { Checkbox } from '@/components/ui/checkbox'; +import { toast } from '@/components/ui/toast-store'; +import { apiFetch } from '@/lib/api'; +import { cn } from '@/lib/utils'; +import { formatTimeAgo } from '@/lib/relativeTime'; +import { useNodes } from '@/context/NodeContext'; +import { useAuth } from '@/context/AuthContext'; + +// Mirrors the backend payload shape (the frontend never imports backend). +type ReadinessVerdict = 'ready' | 'ready_with_warnings' | 'review_required' | 'blocked' | 'unknown'; +type SignalStatus = 'ok' | 'warning' | 'attention' | 'blocked' | 'unknown'; + +interface ReadinessSignal { + id: string; + status: SignalStatus; + title: string; + detail: string; + affectsVerdict: boolean; +} + +interface UpdateReadinessReport { + stack: string; + computedAt: number; + verdict: ReadinessVerdict; + signals: ReadinessSignal[]; +} + +const FETCH_TIMEOUT_MS = 4_000; + +const VERDICT_META: Record = { + ready: { + label: 'ready', + icon: Check, + tone: 'border-success/40 bg-success/[0.06] text-success', + line: 'Nothing stands out; the update can proceed.', + }, + ready_with_warnings: { + label: 'ready with warnings', + icon: Info, + tone: 'border-info/40 bg-info/[0.06] text-info', + line: 'The update can proceed; review the warnings below first.', + }, + review_required: { + label: 'review required', + icon: TriangleAlert, + tone: 'border-warning/40 bg-warning/[0.06] text-warning', + line: 'Something needs a look before this update.', + }, + blocked: { + label: 'blocked', + icon: ShieldAlert, + tone: 'border-destructive/40 bg-destructive/[0.06] text-destructive', + line: 'A blocker was found. Proceeding is likely to fail or be stopped by policy.', + }, + unknown: { + label: 'unknown', + icon: CircleHelp, + tone: 'border-muted bg-card/40 text-stat-subtitle', + line: 'Readiness could not be fully verified; proceed with care.', + }, +}; + +// The `?? unknown` fallbacks at the lookup sites are forward-compat guards: a +// newer backend may send statuses this build does not know. +const SIGNAL_META: Record = { + ok: { icon: Check, tone: 'text-success' }, + warning: { icon: Info, tone: 'text-info' }, + attention: { icon: TriangleAlert, tone: 'text-warning' }, + blocked: { icon: ShieldAlert, tone: 'text-destructive' }, + unknown: { icon: CircleHelp, tone: 'text-stat-subtitle' }, +}; + +const UNKNOWN_FALLBACK = (detail: string): UpdateReadinessReport => ({ + stack: '', + computedAt: Date.now(), + verdict: 'unknown', + signals: [{ + id: 'readiness', + status: 'unknown', + title: 'Readiness check', + detail, + affectsVerdict: true, + }], +}); + +interface UpdateReadinessDialogProps { + open: boolean; + stackName: string; + onCancel: () => void; + /** Caller closes the dialog and starts the update. */ + onProceed: () => void; +} + +/** + * Pre-update readiness check. Advisory only: every verdict, including + * blocked and unknown, keeps Proceed enabled; the scan-policy gate remains + * the single hard block. A slow or failed readiness fetch degrades to an + * unknown verdict so this dialog can never strand the update path. + */ +export function UpdateReadinessDialog({ open, stackName, onCancel, onProceed }: UpdateReadinessDialogProps) { + const { activeNode } = useNodes(); + const { isAdmin } = useAuth(); + const nodeId = activeNode?.id ?? null; + + const [report, setReport] = useState(null); + const [snapshotAt, setSnapshotAt] = useState(null); + const [snapshotKnown, setSnapshotKnown] = useState(false); + const [snapshotFirst, setSnapshotFirst] = useState(false); + const [working, setWorking] = useState(false); + + useEffect(() => { + if (!open) { + setReport(null); + setSnapshotAt(null); + setSnapshotKnown(false); + setSnapshotFirst(false); + setWorking(false); + return; + } + setReport(null); + const controller = new AbortController(); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + controller.abort(); + }, FETCH_TIMEOUT_MS); + + const load = async () => { + try { + const res = await apiFetch(`/stacks/${stackName}/update-readiness`, { signal: controller.signal }); + if (!res.ok) { + const unreachable = res.status === 502 || res.status === 503 || res.status === 504; + setReport(UNKNOWN_FALLBACK(unreachable + ? 'The node may be unreachable; readiness could not be verified.' + : 'The readiness check failed; readiness could not be verified.')); + return; + } + setReport(await res.json() as UpdateReadinessReport); + } catch { + // The cleanup abort (dialog closed) must not write a stale fake + // verdict; only the 4s timer earns the timed-out wording. + if (controller.signal.aborted && !timedOut) return; + setReport(UNKNOWN_FALLBACK(timedOut + ? 'The readiness check did not respond in time.' + : 'The readiness check could not be reached.')); + } + }; + void load(); + + // Snapshot coverage lives only in the hub database; merged client-side. + const loadCoverage = async () => { + if (!isAdmin || nodeId === null) return; + try { + const res = await apiFetch( + `/fleet/snapshots/coverage?nodeId=${nodeId}&stackName=${encodeURIComponent(stackName)}`, + { localOnly: true, signal: controller.signal }, + ); + if (!res.ok) { + console.warn('[UpdateReadiness] snapshot coverage unavailable:', res.status); + return; + } + const body = await res.json() as { latestAt: number | null }; + setSnapshotAt(body.latestAt); + setSnapshotKnown(true); + } catch (e) { + // Coverage is supplemental; the dialog renders without the row. + if (!controller.signal.aborted) { + console.warn('[UpdateReadiness] snapshot coverage unavailable:', e); + } + } + }; + void loadCoverage(); + + return () => { + clearTimeout(timer); + controller.abort(); + }; + }, [open, stackName, nodeId, isAdmin]); + + const proceed = async () => { + if (snapshotFirst) { + setWorking(true); + try { + const res = await apiFetch('/fleet/snapshots', { + method: 'POST', + localOnly: true, + body: JSON.stringify({ description: `Pre-update snapshot: ${stackName}` }), + }); + if (!res.ok) { + let serverError = ''; + try { + serverError = ((await res.json()) as { error?: string }).error ?? ''; + } catch { /* non-JSON body */ } + toast.error(`The pre-update snapshot failed; the update was not started.${serverError ? ` ${serverError}` : ''}`); + return; + } + toast.success('Fleet snapshot created.'); + } catch (e) { + console.error('[UpdateReadiness] pre-update snapshot failed:', e); + toast.error('The pre-update snapshot failed; the update was not started.'); + return; + } finally { + setWorking(false); + } + } + onProceed(); + }; + + const verdict = report ? VERDICT_META[report.verdict] ?? VERDICT_META.unknown : null; + const VerdictIcon = verdict?.icon; + + return ( + { if (!next && !working) onCancel(); }} size="lg"> + + + {!report || !verdict || !VerdictIcon ? ( +
Checking readiness…
+ ) : ( + <> +
+
+ + {verdict.label} +
+
{verdict.line}
+
+ +
+ {report.signals.map(signal => { + const meta = SIGNAL_META[signal.status] ?? SIGNAL_META.unknown; + const SignalIcon = meta.icon; + return ( +
+
+ + {signal.title} +
+
{signal.detail}
+
+ ); + })} + {snapshotKnown && ( +
+
+ + Fleet snapshot +
+
+ {snapshotAt + ? `The most recent fleet snapshot covering this stack was taken ${formatTimeAgo(snapshotAt)}.` + : 'No fleet snapshot covers this stack yet.'} +
+
+ )} +
+ + {isAdmin && ( + + )} + + )} + + + Cancel + + } + primary={ + + } + /> + + ); +} diff --git a/frontend/src/components/stack/__tests__/RollbackReadinessSection.test.tsx b/frontend/src/components/stack/__tests__/RollbackReadinessSection.test.tsx new file mode 100644 index 00000000..329d6201 --- /dev/null +++ b/frontend/src/components/stack/__tests__/RollbackReadinessSection.test.tsx @@ -0,0 +1,61 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import { RollbackReadinessSection } from '../RollbackReadinessSection'; + +vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); + +const nodesState = { + activeNode: { id: 1, type: 'local', name: 'local' }, + hasCapability: vi.fn().mockReturnValue(true), +}; +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => nodesState, +})); + +import { apiFetch } from '@/lib/api'; + +type Overall = 'ready' | 'partial' | 'not_ready'; + +const report = (overall: Overall) => ({ + stack: 'web', + computedAt: Date.now(), + overall, + items: [ + { id: 'compose_source', state: 'ready', label: 'Previous compose file', detail: 'A backup is available to restore.' }, + { id: 'volume_data', state: 'not_covered', label: 'Application data', detail: 'Named volumes and bind-mounted data are not included in file backups.' }, + ], +}); + +describe('RollbackReadinessSection', () => { + beforeEach(() => { + vi.mocked(apiFetch).mockReset(); + nodesState.hasCapability.mockReturnValue(true); + }); + + it.each(['ready', 'partial', 'not_ready'] as const)('renders the %s overall chip', async (overall) => { + vi.mocked(apiFetch).mockResolvedValue(new Response(JSON.stringify(report(overall)), { status: 200 })); + render(); + await waitFor(() => expect(screen.getByTestId('rollback-overall')).toHaveAttribute('data-overall', overall)); + }); + + it('always shows the application-data non-coverage disclosure', async () => { + vi.mocked(apiFetch).mockResolvedValue(new Response(JSON.stringify(report('ready')), { status: 200 })); + render(); + await waitFor(() => expect(screen.getByText('Application data')).toBeInTheDocument()); + expect(screen.getByText(/not included in file backups/)).toBeInTheDocument(); + }); + + it('renders nothing without the update-guard capability and never fetches', () => { + nodesState.hasCapability.mockReturnValue(false); + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + expect(apiFetch).not.toHaveBeenCalled(); + }); + + it('renders nothing when the fetch fails', async () => { + vi.mocked(apiFetch).mockRejectedValue(new Error('down')); + const { container } = render(); + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/frontend/src/components/stack/__tests__/UpdateReadinessDialog.test.tsx b/frontend/src/components/stack/__tests__/UpdateReadinessDialog.test.tsx new file mode 100644 index 00000000..a0c5bf3b --- /dev/null +++ b/frontend/src/components/stack/__tests__/UpdateReadinessDialog.test.tsx @@ -0,0 +1,211 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, fireEvent, waitFor, act } from '@testing-library/react'; +import { UpdateReadinessDialog } from '../UpdateReadinessDialog'; + +vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { success: vi.fn(), error: vi.fn(), info: vi.fn() }, +})); + +const nodesState = { activeNode: { id: 1, type: 'local', name: 'local' } }; +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => nodesState, +})); + +const authState = { isAdmin: true }; +vi.mock('@/context/AuthContext', () => ({ + useAuth: () => authState, +})); + +import { apiFetch } from '@/lib/api'; +import { toast } from '@/components/ui/toast-store'; + +type Verdict = 'ready' | 'ready_with_warnings' | 'review_required' | 'blocked' | 'unknown'; + +const report = (verdict: Verdict) => ({ + stack: 'web', + computedAt: Date.now(), + verdict, + signals: [ + { id: 'preflight', status: 'ok', title: 'Compose Doctor', detail: 'The last preflight passed.', affectsVerdict: true }, + ], +}); + +function routeApi(over: { + readiness?: () => Promise; + coverage?: () => Promise; + snapshot?: () => Promise; +} = {}) { + vi.mocked(apiFetch).mockImplementation((url: string, options?: { method?: string }) => { + const u = String(url); + if (u.includes('/update-readiness')) { + return (over.readiness ?? (() => Promise.resolve(new Response(JSON.stringify(report('ready')), { status: 200 }))))(); + } + if (u.includes('/snapshots/coverage')) { + return (over.coverage ?? (() => Promise.resolve(new Response(JSON.stringify({ latestAt: null }), { status: 200 }))))(); + } + if (u.includes('/fleet/snapshots') && options?.method === 'POST') { + return (over.snapshot ?? (() => Promise.resolve(new Response('{}', { status: 200 }))))(); + } + return Promise.resolve(new Response('{}', { status: 200 })); + }); +} + +function setup(props: Partial[0]> = {}) { + const base = { + open: true, + stackName: 'web', + onCancel: vi.fn(), + onProceed: vi.fn(), + ...props, + }; + render(); + return base; +} + +describe('UpdateReadinessDialog', () => { + beforeEach(() => { + vi.mocked(apiFetch).mockReset(); + vi.clearAllMocks(); + authState.isAdmin = true; + }); + + const verdictCases: Array<{ verdict: Verdict; label: string }> = [ + { verdict: 'ready', label: 'ready' }, + { verdict: 'ready_with_warnings', label: 'ready with warnings' }, + { verdict: 'review_required', label: 'review required' }, + { verdict: 'blocked', label: 'blocked' }, + { verdict: 'unknown', label: 'unknown' }, + ]; + + it.each(verdictCases)('renders the $verdict verdict with Proceed enabled', async ({ verdict, label }) => { + routeApi({ readiness: () => Promise.resolve(new Response(JSON.stringify(report(verdict)), { status: 200 })) }); + setup(); + await waitFor(() => expect(screen.getByTestId('readiness-verdict')).toHaveAttribute('data-verdict', verdict)); + expect(screen.getByText(label)).toBeInTheDocument(); + expect(screen.getByTestId('readiness-proceed')).toBeEnabled(); + }); + + it('degrades to unknown on a plain network failure, and stays non-blocking', async () => { + routeApi({ readiness: () => Promise.reject(new Error('connection refused')) }); + const props = setup(); + await waitFor(() => expect(screen.getByTestId('readiness-verdict')).toHaveAttribute('data-verdict', 'unknown')); + expect(screen.getByText(/could not be reached/)).toBeInTheDocument(); + fireEvent.click(screen.getByTestId('readiness-proceed')); + await waitFor(() => expect(props.onProceed).toHaveBeenCalledTimes(1)); + }); + + it('reports a timed-out readiness check as unknown after the 4s timer aborts it', async () => { + vi.useFakeTimers(); + try { + vi.mocked(apiFetch).mockImplementation((url: string, options?: RequestInit) => { + if (String(url).includes('/update-readiness')) { + return new Promise((_, reject) => { + options?.signal?.addEventListener('abort', () => + reject(new DOMException('Aborted', 'AbortError'))); + }); + } + return Promise.resolve(new Response(JSON.stringify({ latestAt: null }), { status: 200 })); + }); + render(); + await act(async () => { await vi.advanceTimersByTimeAsync(4_100); }); + expect(screen.getByTestId('readiness-verdict')).toHaveAttribute('data-verdict', 'unknown'); + expect(screen.getByText(/did not respond in time/)).toBeInTheDocument(); + } finally { + vi.useRealTimers(); + } + }); + + it('does not paint a stale fallback verdict after closing mid-fetch and reopening', async () => { + let call = 0; + vi.mocked(apiFetch).mockImplementation((url: string, options?: RequestInit) => { + const u = String(url); + if (u.includes('/update-readiness')) { + call += 1; + if (call === 1) { + // Hangs until the cleanup abort rejects it. + return new Promise((_, reject) => { + options?.signal?.addEventListener('abort', () => + reject(new DOMException('Aborted', 'AbortError'))); + }); + } + return Promise.resolve(new Response(JSON.stringify(report('ready')), { status: 200 })); + } + return Promise.resolve(new Response(JSON.stringify({ latestAt: null }), { status: 200 })); + }); + + const props = { stackName: 'web', onCancel: vi.fn(), onProceed: vi.fn() }; + const { rerender } = render(); + rerender(); + rerender(); + await waitFor(() => expect(screen.getByTestId('readiness-verdict')).toHaveAttribute('data-verdict', 'ready')); + }); + + it('marks a remote 502 as unreachable and unknown', async () => { + routeApi({ readiness: () => Promise.resolve(new Response('Bad Gateway', { status: 502 })) }); + setup(); + await waitFor(() => expect(screen.getByTestId('readiness-verdict')).toHaveAttribute('data-verdict', 'unknown')); + expect(screen.getByText(/may be unreachable/)).toBeInTheDocument(); + }); + + it('invokes onProceed without a snapshot when the checkbox is unchecked', async () => { + routeApi(); + const props = setup(); + await waitFor(() => expect(screen.getByTestId('readiness-verdict')).toBeInTheDocument()); + fireEvent.click(screen.getByTestId('readiness-proceed')); + await waitFor(() => expect(props.onProceed).toHaveBeenCalledTimes(1)); + const snapshotPosts = vi.mocked(apiFetch).mock.calls.filter( + c => String(c[0]) === '/fleet/snapshots' && (c[1] as { method?: string } | undefined)?.method === 'POST', + ); + expect(snapshotPosts).toHaveLength(0); + }); + + it('creates a snapshot before proceeding when checked', async () => { + routeApi(); + const props = setup(); + await waitFor(() => expect(screen.getByTestId('readiness-verdict')).toBeInTheDocument()); + fireEvent.click(screen.getByLabelText('Create a fleet snapshot before updating')); + fireEvent.click(screen.getByTestId('readiness-proceed')); + await waitFor(() => expect(props.onProceed).toHaveBeenCalledTimes(1)); + const snapshotPosts = vi.mocked(apiFetch).mock.calls.filter( + c => String(c[0]) === '/fleet/snapshots' && (c[1] as { method?: string } | undefined)?.method === 'POST', + ); + expect(snapshotPosts).toHaveLength(1); + }); + + it('halts the update when the pre-update snapshot fails', async () => { + routeApi({ snapshot: () => Promise.resolve(new Response('{"error":"boom"}', { status: 500 })) }); + const props = setup(); + await waitFor(() => expect(screen.getByTestId('readiness-verdict')).toBeInTheDocument()); + fireEvent.click(screen.getByLabelText('Create a fleet snapshot before updating')); + fireEvent.click(screen.getByTestId('readiness-proceed')); + await waitFor(() => expect(toast.error).toHaveBeenCalled()); + expect(props.onProceed).not.toHaveBeenCalled(); + }); + + it('hides the snapshot checkbox and coverage row from non-admins', async () => { + authState.isAdmin = false; + routeApi(); + setup(); + await waitFor(() => expect(screen.getByTestId('readiness-verdict')).toBeInTheDocument()); + expect(screen.queryByLabelText('Create a fleet snapshot before updating')).not.toBeInTheDocument(); + expect(screen.queryByText('Fleet snapshot')).not.toBeInTheDocument(); + const coverageCalls = vi.mocked(apiFetch).mock.calls.filter(c => String(c[0]).includes('/snapshots/coverage')); + expect(coverageCalls).toHaveLength(0); + }); + + it('shows the hub snapshot coverage row for admins', async () => { + routeApi({ coverage: () => Promise.resolve(new Response(JSON.stringify({ latestAt: Date.now() - 60_000 }), { status: 200 })) }); + setup(); + await waitFor(() => expect(screen.getByText('Fleet snapshot')).toBeInTheDocument()); + expect(screen.getByText(/most recent fleet snapshot/)).toBeInTheDocument(); + }); + + it('wires Cancel', async () => { + routeApi(); + const props = setup(); + await waitFor(() => expect(screen.getByTestId('readiness-verdict')).toBeInTheDocument()); + fireEvent.click(screen.getByText('Cancel')); + expect(props.onCancel).toHaveBeenCalledTimes(1); + }); +}); diff --git a/frontend/src/context/DeployFeedbackContext.tsx b/frontend/src/context/DeployFeedbackContext.tsx index 605d1318..d6fb3f5f 100644 --- a/frontend/src/context/DeployFeedbackContext.tsx +++ b/frontend/src/context/DeployFeedbackContext.tsx @@ -1,4 +1,5 @@ -import React, { createContext, useContext, useState, useRef, useCallback } from 'react'; +import React, { createContext, useContext, useState, useRef, useCallback, useEffect } from 'react'; +import { apiFetch } from '../lib/api'; import { type ParsedLogRow, parseLogChunk } from '../components/log-rendering/composeLogParser'; import { useDeployFeedbackEnabled } from '../hooks/use-deploy-feedback-enabled'; @@ -44,14 +45,35 @@ export interface DeployPanelState { interface RunResult { ok: boolean; errorMessage?: string; + /** + * Health gate run id from the success response, when the backend started a + * post-update observation. Its presence is the feature signal: an older + * node never returns one, so no gate UI appears. + */ + healthGateId?: string | null; } +/** Post-update health gate state for the current deploy session. */ +export interface HealthGateUiState { + stackName: string; + gateId: string; + trigger: 'update' | 'deploy'; + status: 'observing' | 'passed' | 'failed' | 'unknown'; + reason: string | null; + windowSeconds: number | null; + startedAt: number | null; +} + +const GATE_POLL_INTERVAL_MS = 4_000; + interface DeployFeedbackContextValue { runWithLog: ( params: { stackName: string; action: ActionVerb }, run: (deployStarted: Promise, deploySessionId: string) => Promise ) => Promise; panelState: DeployPanelState; + /** Gate state for the current session, or null when no gate was started. */ + healthGate: HealthGateUiState | null; logRows: ParsedLogRow[]; /** * Epoch ms of the most recent activity for the current session: stamped when @@ -99,9 +121,24 @@ const DeployFeedbackContext = createContext(DEFAULT_PANEL_STATE); + const [healthGate, setHealthGate] = useState(null); const [logRows, setLogRows] = useState([]); const [lastOutputAt, setLastOutputAt] = useState(0); + // Poll timer for the current session's health gate; cleared on panel close + // and whenever a new session starts. + const gatePollRef = useRef | null>(null); + + const stopGatePolling = useCallback(() => { + if (gatePollRef.current !== null) { + clearInterval(gatePollRef.current); + gatePollRef.current = null; + } + }, []); + + // The provider is app-root today, but do not let the interval depend on it. + useEffect(() => stopGatePolling, [stopGatePolling]); + // Idempotent resolver for the current session's deployStarted gate. Set at the // start of each runWithLog call; called by onTerminalReady (stream connected), // onTerminalError (stream failed/dropped), or the connect-timeout fallback. @@ -163,9 +200,83 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode sessionIdRef.current += 1; settleStartRef.current = null; streamReadyRef.current = false; + // The gate keeps observing server-side; only this session's poll stops. + stopGatePolling(); + setHealthGate(null); setPanelState(DEFAULT_PANEL_STATE); setLogRows([]); - }, []); + }, [stopGatePolling]); + + // Poll the by-id gate endpoint until a terminal status. The id-scoped read + // means a superseded gate still resolves to its terminal unknown state + // instead of this session showing a newer run's result. Mirrors the backend + // gate's own degradation: repeated failures (or an absurdly long poll) + // resolve to an honest client-side unknown rather than observing forever. + const startGatePolling = useCallback((stackName: string, gateId: string, trigger: 'update' | 'deploy', mySession: number) => { + stopGatePolling(); + setHealthGate({ stackName, gateId, trigger, status: 'observing', reason: null, windowSeconds: null, startedAt: null }); + let strikes = 0; + // Single-flight: skip a tick while one request is outstanding so two + // overlapping responses cannot land out of order. + let inFlight = false; + // Latched once a terminal verdict is applied, so a slow earlier response + // returning 'observing' can never roll the UI back from passed/failed. + let settled = false; + const pollStartedAt = Date.now(); + const giveUp = (reason: string) => { + settled = true; + stopGatePolling(); + setHealthGate(prev => (prev && prev.gateId === gateId ? { ...prev, status: 'unknown', reason } : prev)); + }; + const tick = async () => { + if (sessionIdRef.current !== mySession) { + stopGatePolling(); + return; + } + if (inFlight || settled) return; + // Backstop far beyond the largest configurable window (600s). + if (Date.now() - pollStartedAt > 660_000) { + giveUp('the observation did not report a result in time'); + return; + } + inFlight = true; + try { + const res = await apiFetch(`/stacks/${stackName}/health-gate?gateId=${encodeURIComponent(gateId)}`); + const report = res.ok + ? await res.json() as { + id: string | null; + status: 'observing' | 'passed' | 'failed' | 'unknown' | 'never-run'; + reason: string | null; + windowSeconds: number | null; + startedAt: number | null; + } + : null; + if (sessionIdRef.current !== mySession || settled) return; + // A non-ok response or a missing run (node switched, stack removed, or + // an older node answering) is a strike, not a retry-forever condition. + if (!report || report.id !== gateId || report.status === 'never-run') { + strikes += 1; + if (!res.ok) console.warn('[DeployFeedback] health gate poll returned', res.status); + if (strikes >= 4) giveUp('the gate result could not be retrieved'); + return; + } + strikes = 0; + setHealthGate({ stackName, gateId, trigger, status: report.status, reason: report.reason, windowSeconds: report.windowSeconds, startedAt: report.startedAt }); + if (report.status !== 'observing') { + settled = true; + stopGatePolling(); + } + } catch (e) { + strikes += 1; + console.warn('[DeployFeedback] health gate poll failed:', e); + if (sessionIdRef.current === mySession && strikes >= 4) giveUp('the gate result could not be retrieved'); + } finally { + inFlight = false; + } + }; + void tick(); + gatePollRef.current = setInterval(() => { void tick(); }, GATE_POLL_INTERVAL_MS); + }, [stopGatePolling]); const runWithLog = useCallback( async ( @@ -190,6 +301,8 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode sessionIdRef.current += 1; const mySession = sessionIdRef.current; streamReadyRef.current = false; + stopGatePolling(); + setHealthGate(null); // idCounterRef is intentionally not reset; keys must remain globally unique across sessions. setLogRows([]); @@ -239,16 +352,19 @@ export function DeployFeedbackProvider({ children }: { children: React.ReactNode status: result.ok ? 'succeeded' : 'failed', errorMessage: result.ok ? undefined : result.errorMessage, })); + if (result.ok && result.healthGateId && (params.action === 'update' || params.action === 'deploy')) { + startGatePolling(params.stackName, result.healthGateId, params.action, mySession); + } } return result; }, - [isEnabled] + [isEnabled, startGatePolling, stopGatePolling] ); return ( {children} diff --git a/frontend/src/lib/capabilities.ts b/frontend/src/lib/capabilities.ts index c00b0be0..0fdcc136 100644 --- a/frontend/src/lib/capabilities.ts +++ b/frontend/src/lib/capabilities.ts @@ -24,6 +24,7 @@ export const CAPABILITIES = [ 'self-update', 'vulnerability-scanning', 'compose-doctor', + 'update-guard', ] as const; export type Capability = (typeof CAPABILITIES)[number];