diff --git a/backend/src/__tests__/apply-fleet-snapshot-files.test.ts b/backend/src/__tests__/apply-fleet-snapshot-files.test.ts new file mode 100644 index 00000000..4101476c --- /dev/null +++ b/backend/src/__tests__/apply-fleet-snapshot-files.test.ts @@ -0,0 +1,159 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { applyFleetSnapshotFiles, selectFleetSnapshotApplyFiles, type FleetSnapshotApplyFile } from '../helpers/applyFleetSnapshotFiles'; +import { StackOpLockService } from '../services/StackOpLockService'; + +const mocks = vi.hoisted(() => ({ + hasComposeFile: vi.fn(), + saveStackContent: vi.fn(), + saveEnvContent: vi.fn(), + getBaseDir: vi.fn(() => '/tmp/compose'), + captureCurrentBackup: vi.fn(), + invalidateNodeCaches: vi.fn(), +})); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBaseDir: mocks.getBaseDir, + hasComposeFile: mocks.hasComposeFile, + saveStackContent: mocks.saveStackContent, + saveEnvContent: mocks.saveEnvContent, + }), + }, +})); + +vi.mock('../helpers/cacheInvalidation', () => ({ + invalidateNodeCaches: mocks.invalidateNodeCaches, +})); + +vi.mock('../services/StackUpdateRecoveryService', () => ({ + StackUpdateRecoveryService: { + getInstance: () => ({ + captureCurrentBackup: mocks.captureCurrentBackup, + }), + }, +})); + +const FILES: FleetSnapshotApplyFile[] = [ + { filename: 'compose.yaml', content: 'services:\n snap: {}\n' }, + { filename: '.env', content: 'SNAP=1\n' }, +]; + +function applyWeb(overrides: Partial<{ + stackName: string; + files: FleetSnapshotApplyFile[]; + actor: string; +}> = {}) { + return applyFleetSnapshotFiles({ + nodeId: 1, + stackName: 'web', + files: FILES, + actor: 'system:fleet-snapshot', + ...overrides, + }); +} + +describe('applyFleetSnapshotFiles', () => { + beforeEach(() => { + StackOpLockService.resetForTests(); + mocks.hasComposeFile.mockReset().mockResolvedValue(true); + mocks.saveStackContent.mockReset().mockResolvedValue(undefined); + mocks.saveEnvContent.mockReset().mockResolvedValue(undefined); + mocks.captureCurrentBackup.mockReset().mockResolvedValue({ id: 'gen-pre' }); + mocks.invalidateNodeCaches.mockReset(); + }); + + afterEach(() => { + StackOpLockService.resetForTests(); + }); + + it('captures a recovery generation before writing when the stack already exists', async () => { + const order: string[] = []; + mocks.captureCurrentBackup.mockImplementation(async () => { + order.push('capture'); + return { id: 'gen-pre' }; + }); + mocks.saveStackContent.mockImplementation(async () => { + order.push('compose'); + }); + mocks.saveEnvContent.mockImplementation(async () => { + order.push('env'); + }); + + const result = await applyWeb(); + + expect(result.capturedGenerationId).toBe('gen-pre'); + expect(order).toEqual(['capture', 'compose', 'env']); + expect(mocks.invalidateNodeCaches).toHaveBeenCalledWith(1); + expect(mocks.captureCurrentBackup).toHaveBeenCalledWith({ + nodeId: 1, + stackName: 'web', + createdBy: 'system:fleet-snapshot', + }); + }); + + it('skips capture for a new stack with no compose file', async () => { + mocks.hasComposeFile.mockResolvedValue(false); + + const result = await applyWeb({ stackName: 'fresh' }); + + expect(result.capturedGenerationId).toBeNull(); + expect(mocks.captureCurrentBackup).not.toHaveBeenCalled(); + expect(mocks.saveStackContent).toHaveBeenCalledWith('fresh', FILES[0].content); + expect(mocks.saveEnvContent).toHaveBeenCalledWith('fresh', FILES[1].content); + }); + + it('aborts without writing when capture fails', async () => { + mocks.captureCurrentBackup.mockRejectedValue(Object.assign(new Error('capture failed'), { code: 'CAPTURE_FAILED' })); + + await expect(applyWeb()).rejects.toThrow('capture failed'); + + expect(mocks.saveStackContent).not.toHaveBeenCalled(); + expect(mocks.saveEnvContent).not.toHaveBeenCalled(); + expect(mocks.invalidateNodeCaches).not.toHaveBeenCalled(); + }); + + it('refuses to mutate when another stack operation holds the lock', async () => { + StackOpLockService.getInstance().tryAcquire(1, 'web', 'deploy', 'other'); + + await expect(applyWeb()).rejects.toMatchObject({ code: 'stack_op_in_progress' }); + + expect(mocks.captureCurrentBackup).not.toHaveBeenCalled(); + expect(mocks.saveStackContent).not.toHaveBeenCalled(); + }); + + it('leaves the captured generation in place when a later file write fails', async () => { + mocks.saveEnvContent.mockRejectedValue(new Error('disk full')); + + await expect(applyWeb()).rejects.toThrow('disk full'); + + expect(mocks.captureCurrentBackup).toHaveBeenCalledTimes(1); + expect(mocks.saveStackContent).toHaveBeenCalledWith('web', FILES[0].content); + expect(mocks.invalidateNodeCaches).not.toHaveBeenCalled(); + }); + + it('rejects an empty apply file list before locking', async () => { + await expect(applyWeb({ files: [] })).rejects.toMatchObject({ code: 'INVALID_SNAPSHOT_FILES' }); + + expect(mocks.captureCurrentBackup).not.toHaveBeenCalled(); + expect(mocks.saveStackContent).not.toHaveBeenCalled(); + }); + + it('rejects an invalid stack name before locking or writing', async () => { + await expect(applyWeb({ stackName: '../escape' })).rejects.toMatchObject({ code: 'INVALID_STACK_NAME' }); + + expect(mocks.captureCurrentBackup).not.toHaveBeenCalled(); + expect(mocks.saveStackContent).not.toHaveBeenCalled(); + }); + + it('drops snapshot filenames other than compose.yaml and .env', () => { + expect(selectFleetSnapshotApplyFiles([ + { filename: 'compose.yaml', content: 'a' }, + { filename: 'notes.txt', content: 'nope' }, + { filename: '.env', content: 'b' }, + ])).toEqual([ + { filename: 'compose.yaml', content: 'a' }, + { filename: '.env', content: 'b' }, + ]); + }); +}); diff --git a/backend/src/__tests__/atomic-deploy-hardening.test.ts b/backend/src/__tests__/atomic-deploy-hardening.test.ts index 9008e501..1683a9e2 100644 --- a/backend/src/__tests__/atomic-deploy-hardening.test.ts +++ b/backend/src/__tests__/atomic-deploy-hardening.test.ts @@ -83,7 +83,7 @@ afterAll(() => { }); beforeEach(async () => { - mockDeployStack.mockReset(); + mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null }); mockGetBackupInfo.mockReset().mockResolvedValue({ exists: true, timestamp: Date.now() }); mockRestoreStackFiles.mockReset().mockResolvedValue(undefined); mockSnapshotStackFiles.mockReset().mockResolvedValue(async () => {}); @@ -96,7 +96,7 @@ afterEach(() => vi.restoreAllMocks()); describe('Rollback holds the stack lifecycle lock (H-1)', () => { it('blocks deploy while a rollback is in flight on the same stack', async () => { mockTier('paid'); - const gate = deferred(); + const gate = deferred<{ recoveryId: string | null }>(); mockDeployStack.mockImplementationOnce(() => gate.promise); const rollback = request(app) @@ -113,14 +113,14 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => { expect(deploy.body.code).toBe('stack_op_in_progress'); expect(deploy.body.inProgress.action).toBe('rollback'); - gate.resolve(); + gate.resolve({ recoveryId: null }); const rollbackRes = await rollback; expect(rollbackRes.status).toBe(200); }); it('returns 409 when a rollback lands while a deploy is in flight', async () => { mockTier('paid'); - const gate = deferred(); + const gate = deferred<{ recoveryId: string | null }>(); mockDeployStack.mockImplementationOnce(() => gate.promise); const deploy = request(app) @@ -136,13 +136,13 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => { expect(rollback.status).toBe(409); expect(rollback.body.inProgress.action).toBe('deploy'); - gate.resolve(); + gate.resolve({ recoveryId: null }); await deploy; }); it('releases the lock after a successful rollback', async () => { mockTier('paid'); - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); const first = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(first.status).toBe(200); @@ -157,7 +157,7 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => { const first = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(first.status).toBe(500); - mockDeployStack.mockResolvedValueOnce(undefined); + mockDeployStack.mockResolvedValueOnce({ recoveryId: null }); const second = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(second.status).toBe(200); }); @@ -166,7 +166,7 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => { describe('Rollback notifications (M-2)', () => { it('dispatches a success notification when a rollback completes', async () => { mockTier('paid'); - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); const { NotificationService } = await import('../services/NotificationService'); const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); @@ -214,7 +214,7 @@ describe('Rollback returns 404 when no backup exists', () => { // The 404 is an early return inside the try; the finally must still release // the lock so the stack is not wedged at 409 afterwards. mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: Date.now() }); - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); const next = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(next.status).toBe(200); }); @@ -223,7 +223,7 @@ describe('Rollback returns 404 when no backup exists', () => { describe('Developer Mode logging matrix', () => { it('only emits rollback diagnostic logs when Developer Mode is enabled', async () => { mockTier('paid'); - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); const { DatabaseService } = await import('../services/DatabaseService'); const db = DatabaseService.getInstance(); const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); @@ -246,7 +246,7 @@ describe('Deploy safety is available on every tier', () => { it('allows rollback on community', async () => { mockTier('community'); mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: 1700000000000 }); - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(res.status).toBe(200); expect(mockDeployStack).toHaveBeenCalled(); @@ -257,7 +257,7 @@ describe('Deploy safety is available on every tier', () => { mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: 1700000000000 }); const res = await request(app).get('/api/stacks/web/backup').set('Cookie', authCookie); expect(res.status).toBe(200); - expect(res.body).toEqual({ exists: true, timestamp: 1700000000000 }); + expect(res.body).toMatchObject({ exists: true, timestamp: 1700000000000 }); }); it('returns backup metadata on paid', async () => { @@ -265,6 +265,6 @@ describe('Deploy safety is available on every tier', () => { mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: 1700000000000 }); const res = await request(app).get('/api/stacks/web/backup').set('Cookie', authCookie); expect(res.status).toBe(200); - expect(res.body).toEqual({ exists: true, timestamp: 1700000000000 }); + expect(res.body).toMatchObject({ exists: true, timestamp: 1700000000000 }); }); }); diff --git a/backend/src/__tests__/blueprints-compose-apply.test.ts b/backend/src/__tests__/blueprints-compose-apply.test.ts index 8baf72f6..55f4b217 100644 --- a/backend/src/__tests__/blueprints-compose-apply.test.ts +++ b/backend/src/__tests__/blueprints-compose-apply.test.ts @@ -75,7 +75,7 @@ describe('Blueprint compose apply (real filesystem)', () => { const composeContent = 'services:\n web:\n image: traefik:v3\n'; const markerContent = JSON.stringify({ blueprintId: 1, revision: 1, lastApplied: Date.now() }, null, 2); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const outcome = await BlueprintService.getInstance().applyLocalUnderLock( nodeId, @@ -120,6 +120,7 @@ describe('Blueprint compose apply (real filesystem)', () => { path.join(stackDir, 'docker-compose.yaml'), path.join(stackDir, 'docker-compose.yml'), ); + return { recoveryId: null }; }); const outcome = await BlueprintService.getInstance().applyLocalUnderLock( @@ -197,7 +198,7 @@ describe('Blueprint compose apply (real filesystem)', () => { const original = 'services:\n mine:\n image: nginx:alpine\n'; await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); await expect( BlueprintService.getInstance().applyLocalUnderLock( diff --git a/backend/src/__tests__/blueprints.test.ts b/backend/src/__tests__/blueprints.test.ts index 0dd148dc..8eadc069 100644 --- a/backend/src/__tests__/blueprints.test.ts +++ b/backend/src/__tests__/blueprints.test.ts @@ -410,7 +410,7 @@ describe('BlueprintService per-stack lock', () => { vi.spyOn(FileSystemService.prototype, 'createStack').mockResolvedValue(undefined); const writeSpy = vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockResolvedValue(undefined); const cleanupSpy = vi.spyOn(FileSystemService.prototype, 'removeAlternateRootComposeFiles').mockResolvedValue(undefined); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const outcome = await BlueprintService.getInstance().deployToNode(bp, node); diff --git a/backend/src/__tests__/community-deploy-policy-route.test.ts b/backend/src/__tests__/community-deploy-policy-route.test.ts index 436ca765..039bd2f1 100644 --- a/backend/src/__tests__/community-deploy-policy-route.test.ts +++ b/backend/src/__tests__/community-deploy-policy-route.test.ts @@ -49,7 +49,7 @@ beforeAll(async () => { const { ComposeService } = await import('../services/ComposeService'); listImagesSpy = vi.spyOn(ComposeService.prototype, 'listStackImages').mockResolvedValue(['nginx:bad']); - deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined); + deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const TrivyService = (await import('../services/TrivyService')).default; const trivy = TrivyService.getInstance(); diff --git a/backend/src/__tests__/compose-project-context.test.ts b/backend/src/__tests__/compose-project-context.test.ts new file mode 100644 index 00000000..45c38605 --- /dev/null +++ b/backend/src/__tests__/compose-project-context.test.ts @@ -0,0 +1,112 @@ +/** + * Generation capture must not rewrite the legacy single-slot backup. A later + * capture failure has to leave a pre-migration recovery point byte-identical. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import path from 'path'; +import os from 'os'; +import { promises as fsPromises } from 'fs'; + +const mockState = { composeDir: '', composeDirs: new Map() }; + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getComposeDir: (nodeId?: number) => mockState.composeDirs.get(nodeId ?? 1) ?? mockState.composeDir, + getDefaultNodeId: () => 1, + }), + }, +})); + +vi.mock('../utils/debug', () => ({ + isDebugEnabled: () => false, +})); + +vi.mock('../services/rollbackInventory', () => ({ + resolveRollbackInventory: vi.fn(), +})); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getStackProjectEnvFiles: () => [], + }), + }, +})); + +import { FileSystemService } from '../services/FileSystemService'; +import { RollbackGenerationStore } from '../services/RollbackGenerationStore'; +import { resolveRollbackInventory } from '../services/rollbackInventory'; +import { resolveComposeProjectContext } from '../services/composeProjectContext'; + +describe('composeProjectContext backupFromContext', () => { + let composeDir: string; + let dataDir: string; + let originalDataDir: string | undefined; + + beforeEach(async () => { + composeDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-compose-')); + dataDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-data-')); + mockState.composeDir = composeDir; + mockState.composeDirs = new Map([[1, composeDir]]); + originalDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = dataDir; + }); + + afterEach(async () => { + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + await fsPromises.rm(composeDir, { recursive: true, force: true }); + await fsPromises.rm(dataDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it('preserves a seeded legacy backup when generation capture fails', async () => { + const stackName = 'legacy-slot'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'ORIGINAL\n', 'utf8'); + + const fsSvc = FileSystemService.getInstance(1); + await fsSvc.backupStackFiles(stackName); + const backupPath = path.join(dataDir, 'backups', '1', stackName, 'compose.yaml'); + const originalBackup = await fsPromises.readFile(backupPath); + + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'MUTATED\n', 'utf8'); + + vi.mocked(resolveRollbackInventory).mockResolvedValue({ + entries: [{ + relativePath: 'compose.yaml', + dependencyKind: 'compose-root', + provenance: 'authored', + sensitivity: 'low', + absolutePath: path.join(stackDir, 'compose.yaml'), + }], + invocation: { + composeArgsPrefix: [], + projectDirectory: null, + projectName: stackName, + explicitComposeFiles: ['compose.yaml'], + meshEnabled: false, + meshOverrideRelativePath: null, + }, + git: null, + appliedDeploySpec: null, + lastAppliedContentHash: null, + manifestState: null, + manifestGeneration: null, + exactCoverage: true, + coverageRefusal: null, + }); + vi.spyOn(RollbackGenerationStore, 'captureGeneration').mockRejectedValue( + new Error('injected capture failure'), + ); + + const ctx = await resolveComposeProjectContext(1, stackName); + await expect(ctx.backupFromContext('update')).rejects.toThrow(/injected capture failure/); + + expect(await fsPromises.readFile(backupPath)).toEqual(originalBackup); + await fsSvc.restoreStackFiles(stackName); + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe('ORIGINAL\n'); + }); +}); diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts index 661f6afc..fd012439 100644 --- a/backend/src/__tests__/compose-service.test.ts +++ b/backend/src/__tests__/compose-service.test.ts @@ -735,7 +735,7 @@ describe('ComposeService - deployStack', () => { const promise = ComposeService.getInstance(1).deployStack('my-stack'); await vi.advanceTimersByTimeAsync(3100); - await expect(promise).resolves.toBeUndefined(); + await expect(promise).resolves.toEqual({ recoveryId: null }); expect(mockGetLegacyOrphanContainersByStack).toHaveBeenCalledWith('my-stack'); }); @@ -829,7 +829,7 @@ describe('ComposeService - deployStack', () => { ); }); - it('creates backup when atomic=true', async () => { + it('captures recovery generation when atomic=true', async () => { setupAutoCloseSpawn(); mockListContainers.mockResolvedValue([]); @@ -837,9 +837,16 @@ describe('ComposeService - deployStack', () => { const promise = svc.deployStack('my-stack', undefined, true); await vi.advanceTimersByTimeAsync(3100); - await promise; + const result = await promise; - expect(mockBackupStackFiles).toHaveBeenCalledWith('my-stack'); + expect(result).toEqual({ recoveryId: 'recovery-1' }); + expect(mockCaptureCandidate).toHaveBeenCalledWith(expect.objectContaining({ + stackName: 'my-stack', + operationKind: 'deployment', + createdBy: 'atomic-deploy', + })); + expect(mockHandoff).toHaveBeenCalled(); + expect(mockMarkImmediateVerified).toHaveBeenCalledWith('recovery-1'); }); it('blocks deploy before backup when missing external networks need a prompt', async () => { @@ -861,7 +868,7 @@ describe('ComposeService - deployStack', () => { const svc = ComposeService.getInstance(1); await expect(svc.deployStack('my-stack', undefined, true)).rejects.toBeInstanceOf(MissingExternalNetworksError); - expect(mockBackupStackFiles).not.toHaveBeenCalled(); + expect(mockCaptureCandidate).not.toHaveBeenCalled(); expect(mockSpawn).not.toHaveBeenCalled(); expect(mockAddNotificationHistory).not.toHaveBeenCalled(); }); @@ -913,7 +920,7 @@ describe('ComposeService - deployStack', () => { stack_name: 'my-stack', }), ); - expect(mockBackupStackFiles).toHaveBeenCalled(); + expect(mockCaptureCandidate).toHaveBeenCalled(); }); it('does not wrap a missing-external prompt in ComposeRollbackError when atomic', async () => { @@ -938,14 +945,12 @@ describe('ComposeService - deployStack', () => { expect(error?.name).toBe('MissingExternalNetworksError'); }); - it('aborts atomic deploy before docker side effects when backup fails', async () => { - mockBackupStackFiles.mockRejectedValueOnce(new Error('disk full')); + it('aborts atomic deploy before docker side effects when capture fails', async () => { + mockCaptureCandidate.mockRejectedValueOnce(new Error('disk full')); const svc = ComposeService.getInstance(1); - await expect(svc.deployStack('my-stack', undefined, true)).rejects.toThrow( - 'Atomic deployment backup failed', - ); + await expect(svc.deployStack('my-stack', undefined, true)).rejects.toThrow('disk full'); expect(mockSpawn).not.toHaveBeenCalled(); expect(mockGetLegacyOrphanContainersByStack).not.toHaveBeenCalled(); }); @@ -990,7 +995,7 @@ describe('ComposeService - deployStack', () => { expect(error).not.toBeNull(); expect(error!.message).toContain('CONTAINER_CRASHED'); expect(getComposeRollbackInfo(error)).toEqual({ attempted: true, rolledBack: true }); - expect(mockRestoreStackFiles).toHaveBeenCalledWith('my-stack'); + expect(mockCompensateWithCandidate).toHaveBeenCalledWith('recovery-1', expect.any(Function)); }); it('reports rollback failure when atomic restore fails', async () => { @@ -1001,7 +1006,7 @@ describe('ComposeService - deployStack', () => { Labels: { 'com.docker.compose.project': 'my-stack' }, }]); mockContainerInspect.mockResolvedValue({ State: { ExitCode: 1 } }); - mockRestoreStackFiles.mockRejectedValueOnce(new Error('restore denied')); + mockCompensateWithCandidate.mockResolvedValueOnce(false); const svc = ComposeService.getInstance(1); const result = svc.deployStack('my-stack', undefined, true).then(() => null, (e: Error) => e); @@ -1012,6 +1017,28 @@ describe('ComposeService - deployStack', () => { expect(getComposeRollbackInfo(error)).toEqual({ attempted: true, rolledBack: false }); }); + it('wraps a missing hold tag as rolledBack=false without replacing the original error', async () => { + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([{ + Id: 'crashed-c1', + State: 'exited', + Labels: { 'com.docker.compose.project': 'my-stack' }, + }]); + mockContainerInspect.mockResolvedValue({ State: { ExitCode: 1 } }); + mockCompensateWithCandidate.mockRejectedValueOnce( + Object.assign(new Error('Held recovery image is missing'), { code: 'HELD_IMAGE_MISSING' }), + ); + + const svc = ComposeService.getInstance(1); + const result = svc.deployStack('my-stack', undefined, true).then(() => null, (e: Error) => e); + + await vi.runAllTimersAsync(); + const error = await result; + expect(error).not.toBeNull(); + expect(error!.message).toContain('CONTAINER_CRASHED'); + expect(getComposeRollbackInfo(error)).toEqual({ attempted: true, rolledBack: false }); + }); + it('does not roll back when atomic=false', async () => { setupAutoCloseSpawn(); mockListContainers.mockResolvedValue([{ diff --git a/backend/src/__tests__/deployed-stack-deletion-service.test.ts b/backend/src/__tests__/deployed-stack-deletion-service.test.ts index c156ed15..b352f631 100644 --- a/backend/src/__tests__/deployed-stack-deletion-service.test.ts +++ b/backend/src/__tests__/deployed-stack-deletion-service.test.ts @@ -50,6 +50,8 @@ describe('DeployedStackDeletionService ready transaction', () => { phase: 'immediate_verified', is_current: 1, backup_slot_id: null, + content_path: null, + operation_kind: null, override_path: null, services_json: '[]', health_gate_id: null, diff --git a/backend/src/__tests__/docker-integration/exact-prior-image-rollback.test.ts b/backend/src/__tests__/docker-integration/exact-prior-image-rollback.test.ts new file mode 100644 index 00000000..a86d42c7 --- /dev/null +++ b/backend/src/__tests__/docker-integration/exact-prior-image-rollback.test.ts @@ -0,0 +1,150 @@ +/** + * Docker-backed: recovery override must recreate containers whose inspected + * Image ID equals the captured prior image ID (moving-tag case). + * Skipped when Docker is unavailable. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { execFileSync } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb'; + +function dockerAvailable(): boolean { + try { + execFileSync('docker', ['info'], { + stdio: 'ignore', + timeout: 8_000, + windowsHide: true, + }); + return true; + } catch { + return false; + } +} + +const hasDocker = dockerAvailable(); +const STACK = 'exactimg'; + +function compose(args: string[], cwd: string): string { + return execFileSync('docker', ['compose', '-p', STACK, ...args], { + cwd, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +function inspectImageId(containerId: string): string { + const raw = execFileSync('docker', ['inspect', containerId, '--format', '{{.Image}}'], { + encoding: 'utf8', + }).trim(); + expect(raw.length).toBeGreaterThan(0); + return raw; +} + +describe.skipIf(!hasDocker)('exact prior-image rollback after post-handoff failure', () => { + let tmpDir: string; + let composeDir: string; + let stackDir: string; + let nodeId: number; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + composeDir = process.env.COMPOSE_DIR!; + stackDir = path.join(composeDir, STACK); + fs.mkdirSync(stackDir, { recursive: true }); + + const { DatabaseService } = await import('../../services/DatabaseService'); + const db = DatabaseService.getInstance(); + const local = db.getDefaultNode(); + if (!local?.id) throw new Error('Test DB has no default local node'); + nodeId = local.id; + db.updateGlobalSetting('prune_on_update', '0'); + + execFileSync('docker', ['pull', 'busybox:1.36.1'], { stdio: 'ignore' }); + execFileSync('docker', ['pull', 'busybox:1.36.0'], { stdio: 'ignore' }); + + fs.writeFileSync( + path.join(stackDir, 'compose.yaml'), + [ + 'services:', + ' web:', + ' image: busybox:1.36.1', + ' command: ["sleep", "3600"]', + '', + ].join('\n'), + 'utf8', + ); + + compose(['up', '-d', '--pull', 'never'], stackDir); + }, 300_000); + + afterAll(async () => { + try { + if (stackDir && fs.existsSync(stackDir)) { + compose(['down', '--remove-orphans'], stackDir); + } + } catch { + // Best-effort cleanup. + } + if (tmpDir) cleanupTestDb(tmpDir); + }, 120_000); + + it('restores a container whose inspected Image equals the captured prior image ID', async () => { + const beforeId = compose(['ps', '-q'], stackDir).trim(); + expect(beforeId.length).toBeGreaterThan(0); + const priorImageId = inspectImageId(beforeId); + + const { StackUpdateRecoveryService } = await import('../../services/StackUpdateRecoveryService'); + StackUpdateRecoveryService.resetForTests(); + const recoverySvc = StackUpdateRecoveryService.getInstance(); + + // Capture while the prior runtime is still healthy. + const candidate = await recoverySvc.captureCandidate({ + nodeId, + stackName: STACK, + createdBy: null, + operationKind: 'update', + }); + expect(candidate.override_path).toBeTruthy(); + expect(recoverySvc.markAcquired(candidate.id)).toBe(true); + expect(recoverySvc.handoff(candidate.id, nodeId, STACK)).toBe(true); + + // Simulate a post-handoff mutation that moves the tag / image identity. + fs.writeFileSync( + path.join(stackDir, 'compose.yaml'), + [ + 'services:', + ' web:', + ' image: busybox:1.36.0', + ' command: ["sleep", "3600"]', + '', + ].join('\n'), + 'utf8', + ); + compose(['up', '-d', '--pull', 'never', '--force-recreate'], stackDir); + const midId = compose(['ps', '-q'], stackDir).trim(); + const midImageId = inspectImageId(midId); + expect(midImageId).not.toBe(priorImageId); + + const { ComposeService } = await import('../../services/ComposeService'); + const rolledBack = await recoverySvc.compensateWithCandidate( + candidate.id, + (overridePath, invocation) => ComposeService.getInstance(nodeId).composeUpWithRecoveryOverride( + STACK, + overridePath, + undefined, + invocation, + ), + ); + expect(rolledBack).toBe(true); + + const afterId = compose(['ps', '-q'], stackDir).trim(); + expect(afterId.length).toBeGreaterThan(0); + expect(inspectImageId(afterId)).toBe(priorImageId); + + const afterInspect = JSON.parse( + execFileSync('docker', ['inspect', afterId], { encoding: 'utf8' }), + ) as Array<{ State: { Running: boolean } }>; + expect(afterInspect[0].State.Running).toBe(true); + }, 300_000); +}); diff --git a/backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts b/backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts index 696db72c..a11a7713 100644 --- a/backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts +++ b/backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts @@ -10,7 +10,11 @@ import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb'; function dockerAvailable(): boolean { try { - execFileSync('docker', ['info'], { stdio: 'ignore' }); + execFileSync('docker', ['info'], { + stdio: 'ignore', + timeout: 8_000, + windowsHide: true, + }); return true; } catch { return false; diff --git a/backend/src/__tests__/failure-classifier.test.ts b/backend/src/__tests__/failure-classifier.test.ts index d771738a..73535b75 100644 --- a/backend/src/__tests__/failure-classifier.test.ts +++ b/backend/src/__tests__/failure-classifier.test.ts @@ -109,6 +109,21 @@ describe('classifyFailure', () => { message: 'something completely unexpected happened', reason: 'unknown', }, + { + name: 'mixed replica images capture refusal', + message: 'Service "web" has mixed replica images; refusing recovery capture that cannot restore exact prior identity', + reason: 'mixed_replica_images', + }, + { + name: 'host-absolute include coverage refusal', + message: 'Host-absolute include path cannot be captured for exact rollback', + reason: 'rollback_coverage_unavailable', + }, + { + name: 'generic rollback coverage unavailable', + message: 'Exact authored-project rollback coverage is unavailable for this stack', + reason: 'rollback_coverage_unavailable', + }, ]; it.each(cases)('classifies $name as $reason', ({ message, reason }) => { diff --git a/backend/src/__tests__/fleet-snapshot-routes.test.ts b/backend/src/__tests__/fleet-snapshot-routes.test.ts index 3a3db86d..f44ac149 100644 --- a/backend/src/__tests__/fleet-snapshot-routes.test.ts +++ b/backend/src/__tests__/fleet-snapshot-routes.test.ts @@ -8,8 +8,11 @@ import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest import request from 'supertest'; import fs from 'fs'; import path from 'path'; +import { createHash, randomUUID } from 'crypto'; import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; import * as policyGate from '../helpers/policyGate'; +import { RollbackGenerationStore } from '../services/RollbackGenerationStore'; +import type { ResolvedRollbackInventory, RollbackGenerationManifest } from '../types/rollbackGeneration'; let tmpDir: string; let app: import('express').Express; @@ -18,6 +21,9 @@ let CryptoService: typeof import('../services/CryptoService').CryptoService; let ComposeService: typeof import('../services/ComposeService').ComposeService; let LicenseService: typeof import('../services/LicenseService').LicenseService; let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry; +let FileSystemService: typeof import('../services/FileSystemService').FileSystemService; +let StackUpdateRecoveryService: typeof import('../services/StackUpdateRecoveryService').StackUpdateRecoveryService; +let StackOpLockService: typeof import('../services/StackOpLockService').StackOpLockService; let adminCookie: string; let viewerCookie: string; let snapshotId: number; @@ -39,6 +45,9 @@ beforeAll(async () => { ({ ComposeService } = await import('../services/ComposeService')); ({ LicenseService } = await import('../services/LicenseService')); ({ NodeRegistry } = await import('../services/NodeRegistry')); + ({ FileSystemService } = await import('../services/FileSystemService')); + ({ StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService')); + ({ StackOpLockService } = await import('../services/StackOpLockService')); ({ app } = await import('../index')); adminCookie = await loginAsTestAdmin(app); @@ -289,7 +298,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => { fs.writeFileSync(composePath('corrupt-web'), beforeCompose); fs.writeFileSync(envPath('corrupt-web'), beforeEnv); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const res = await request(app) .post(`/api/fleet/snapshots/${id}/restore`) .set('Cookie', adminCookie) @@ -320,7 +329,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => { fs.writeFileSync(composePath('mixed-web'), beforeCompose); fs.writeFileSync(envPath('mixed-web'), beforeEnv); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const res = await request(app) .post(`/api/fleet/snapshots/${id}/restore`) .set('Cookie', adminCookie) @@ -346,7 +355,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => { const beforeCompose = 'services:\n keep: {}\n'; fs.writeFileSync(composePath('delim-web'), beforeCompose); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const res = await request(app) .post(`/api/fleet/snapshots/${id}/restore`) .set('Cookie', adminCookie) @@ -387,7 +396,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => { it('redeploys after restore when requested', async () => { vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const db = DatabaseService.getInstance(); const id = db.createSnapshot('restore-redeploy', 'admin', 1, 1, '[]', '[]'); db.insertSnapshotFiles(id, [ @@ -749,7 +758,7 @@ describe('Restore-all', () => { it('isolates corrupt decrypt stacks before any mutation with notes and redeploy requested', async () => { vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const db = DatabaseService.getInstance(); const id = db.createSnapshot('restore-all-corrupt', 'admin', 1, 2, '[]', '[]'); const good = CryptoService.getInstance().encrypt('services:\n app: {}\n'); @@ -791,7 +800,7 @@ describe('Restore-all', () => { it('isolates delimiter-byte corruption before restore-all mutation', async () => { vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const db = DatabaseService.getInstance(); const id = db.createSnapshot('restore-all-delim', 'admin', 1, 2, '[]', '[]'); const good = CryptoService.getInstance().encrypt('services:\n app: {}\n'); @@ -801,13 +810,13 @@ describe('Restore-all', () => { const damaged = `enc:${iv} ${tag}:${ct}`; db.getDb().prepare( 'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)', - ).run(id, LOCAL_NODE_ID, 'local', 'healthy', 'compose.yaml', good); + ).run(id, LOCAL_NODE_ID, 'local', 'delim-healthy', 'compose.yaml', good); db.getDb().prepare( 'INSERT INTO fleet_snapshot_files (snapshot_id, node_id, node_name, stack_name, filename, content) VALUES (?, ?, ?, ?, ?, ?)', - ).run(id, LOCAL_NODE_ID, 'local', 'corrupt', 'compose.yaml', damaged); - seedStackDir('healthy'); - seedStackDir('corrupt'); - fs.writeFileSync(composePath('corrupt'), 'services:\n keep: {}\n'); + ).run(id, LOCAL_NODE_ID, 'local', 'delim-corrupt', 'compose.yaml', damaged); + seedStackDir('delim-healthy'); + seedStackDir('delim-corrupt'); + fs.writeFileSync(composePath('delim-corrupt'), 'services:\n keep: {}\n'); const res = await request(app) .post(`/api/fleet/snapshots/${id}/restore-all`) @@ -817,16 +826,16 @@ describe('Restore-all', () => { expect(res.body.restored).toBe(1); expect(res.body.failed).toBe(1); const corrupt = (res.body.results as Array<{ stackName: string; success: boolean; error?: string }>) - .find(r => r.stackName === 'corrupt'); + .find(r => r.stackName === 'delim-corrupt'); expect(corrupt?.success).toBe(false); - expect(fs.readFileSync(composePath('corrupt'), 'utf-8')).toContain('keep: {}'); + expect(fs.readFileSync(composePath('delim-corrupt'), 'utf-8')).toContain('keep: {}'); expect(deploySpy).toHaveBeenCalledTimes(1); - expect(deploySpy.mock.calls.every(call => call[0] !== 'corrupt')).toBe(true); + expect(deploySpy.mock.calls.every(call => call[0] !== 'delim-corrupt')).toBe(true); }); it('redeploys each restored stack when requested', async () => { vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const db = DatabaseService.getInstance(); const id = db.createSnapshot('restore-all-redeploy', 'admin', 1, 1, '[]', '[]'); db.insertSnapshotFiles(id, [ @@ -847,7 +856,7 @@ describe('Restore-all', () => { }); it('records a policy-blocked redeploy as a per-stack failure and still restores the rest', async () => { - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); vi.spyOn(policyGate, 'assertPolicyGateAllows').mockImplementation(async (stackName: string) => { if (stackName === 'blocked-web') throw new Error('Policy "block-criticals" blocked deploy: 1 image(s) exceed high'); }); @@ -876,3 +885,456 @@ describe('Restore-all', () => { expect(deploySpy.mock.calls.map((c) => c[0])).not.toContain('blocked-web'); }); }); + +describe('Snapshot restore: recovery generation contract', () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + StackOpLockService.resetForTests(); + DatabaseService.getInstance().getDb().prepare('DELETE FROM stack_update_recovery_generations').run(); + }); + + function seedExistingStack(stack: string, compose: string, extraName?: string, extraContent?: string): void { + seedStackDir(stack); + fs.writeFileSync(composePath(stack), compose); + if (extraName && extraContent !== undefined) { + fs.writeFileSync(path.join(process.env.COMPOSE_DIR as string, stack, extraName), extraContent); + } + } + + function insertSnapshot( + label: string, + files: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }>, + stackCount = 1, + ): number { + const db = DatabaseService.getInstance(); + const id = db.createSnapshot(label, 'admin', 1, stackCount, '[]', '[]'); + db.insertSnapshotFiles(id, files); + return id; + } + + function addRemoteNode(name: string): number { + return DatabaseService.getInstance().addNode({ + name, + type: 'remote', + api_url: 'http://remote:1852', + api_token: 'tok', + compose_dir: '/app/compose', + is_default: false, + }); + } + + function stubRemoteProxy(): void { + vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ + apiUrl: 'http://remote:1852', + apiToken: 'tok', + }); + } + + function sha256Text(text: string): string { + return createHash('sha256').update(text, 'utf8').digest('hex'); + } + + function liveInventory(stack: string): ResolvedRollbackInventory { + const stackDir = path.join(process.env.COMPOSE_DIR as string, stack); + const entries: ResolvedRollbackInventory['entries'] = [{ + relativePath: 'compose.yaml', + dependencyKind: 'compose-root', + provenance: 'authored', + sensitivity: 'low', + absolutePath: path.join(stackDir, 'compose.yaml'), + }]; + if (fs.existsSync(envPath(stack))) { + entries.push({ + relativePath: '.env', + dependencyKind: 'project-env', + provenance: 'authored', + sensitivity: 'low', + absolutePath: envPath(stack), + }); + } + return { + entries, + invocation: { + composeArgsPrefix: [], + projectDirectory: null, + projectName: stack, + explicitComposeFiles: ['compose.yaml'], + meshEnabled: false, + meshOverrideRelativePath: null, + }, + git: null, + appliedDeploySpec: null, + lastAppliedContentHash: null, + manifestState: null, + manifestGeneration: null, + exactCoverage: true, + coverageRefusal: null, + }; + } + + async function persistPreRestoreGeneration(stack: string): Promise { + const id = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: LOCAL_NODE_ID, + stackName: stack, + generationId: id, + inventory: liveInventory(stack), + operationKind: 'manual_backup', + }); + const now = Date.now(); + DatabaseService.getInstance().insertStackUpdateRecoveryGeneration({ + id, + node_id: LOCAL_NODE_ID, + stack_name: stack, + status: 'active', + phase: 'immediate_verified', + is_current: 1, + backup_slot_id: id, + content_path: id, + operation_kind: 'manual_backup', + override_path: null, + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: now, + updated_at: now, + created_by: 'system:fleet-snapshot', + artifacts_retired: 0, + released_at: null, + released_by: null, + }); + return id; + } + + function spyCaptureRecordingLive(stack: string): { composeAtCapture: string; envAtCapture: string } { + const captured = { composeAtCapture: '', envAtCapture: '' }; + vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup').mockImplementation(async () => { + captured.composeAtCapture = fs.readFileSync(composePath(stack), 'utf-8'); + captured.envAtCapture = fs.existsSync(envPath(stack)) ? fs.readFileSync(envPath(stack), 'utf-8') : ''; + const id = await persistPreRestoreGeneration(stack); + return { id } as never; + }); + return captured; + } + + function expectCurrentGenerationMatches(stack: string, compose: string, env: string): void { + const current = StackUpdateRecoveryService.getInstance().getCurrent(LOCAL_NODE_ID, stack); + expect(current?.id).toBeTruthy(); + const genDir = RollbackGenerationStore.getGenerationDir(LOCAL_NODE_ID, stack, current!.id); + const manifest = JSON.parse( + fs.readFileSync(path.join(genDir, 'generation.json'), 'utf-8'), + ) as RollbackGenerationManifest; + expect(manifest.entries.find((e) => e.relativePath === 'compose.yaml')?.contentSha256).toBe(sha256Text(compose)); + expect(manifest.entries.find((e) => e.relativePath === '.env')?.contentSha256).toBe(sha256Text(env)); + } + + it('captures the pre-restore multi-file project before overwriting compose.yaml', async () => { + const id = insertSnapshot('restore-gen-local', [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'preimage-web', filename: 'compose.yaml', content: 'services:\n snap: {}\n' }, + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'preimage-web', filename: '.env', content: 'SNAP=1\n' }, + ]); + const preCompose = 'services:\n old: {}\n'; + const preEnv = 'OLD=1\n'; + seedExistingStack('preimage-web', preCompose, 'extra.yml', 'x: 1\n'); + fs.writeFileSync(envPath('preimage-web'), preEnv); + const captured = spyCaptureRecordingLive('preimage-web'); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'preimage-web' }); + + expect(res.status).toBe(200); + expect(captured.composeAtCapture).toBe(preCompose); + expect(captured.envAtCapture).toBe(preEnv); + expect(fs.readFileSync(composePath('preimage-web'), 'utf-8')).toContain('snap: {}'); + expect(fs.readFileSync(envPath('preimage-web'), 'utf-8')).toContain('SNAP=1'); + expect(fs.readFileSync(path.join(process.env.COMPOSE_DIR as string, 'preimage-web', 'extra.yml'), 'utf-8')).toBe('x: 1\n'); + expectCurrentGenerationMatches('preimage-web', preCompose, preEnv); + }); + + it('does not mutate live files when capture fails', async () => { + const id = insertSnapshot('restore-gen-fail', [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'fail-web', filename: 'compose.yaml', content: 'services:\n snap: {}\n' }, + ]); + const preCompose = 'services:\n keep: {}\n'; + seedExistingStack('fail-web', preCompose); + vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup') + .mockRejectedValue(new Error('generation capture failed')); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'fail-web' }); + + expect(res.status).toBe(500); + expect(fs.readFileSync(composePath('fail-web'), 'utf-8')).toBe(preCompose); + }); + + it('returns 409 and does not write when the stack operation lock is held', async () => { + const id = insertSnapshot('restore-gen-lock', [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'lock-web', filename: 'compose.yaml', content: 'services:\n snap: {}\n' }, + ]); + const preCompose = 'services:\n keep: {}\n'; + seedExistingStack('lock-web', preCompose); + const captureSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup') + .mockResolvedValue({ id: 'gen-lock-web' } as never); + StackOpLockService.getInstance().tryAcquire(LOCAL_NODE_ID, 'lock-web', 'deploy', 'other'); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'lock-web' }); + + expect(res.status).toBe(409); + expect(res.body.code).toBe('stack_op_in_progress'); + expect(captureSpy).not.toHaveBeenCalled(); + expect(fs.readFileSync(composePath('lock-web'), 'utf-8')).toBe(preCompose); + }); + + it('keeps the captured generation when a later snapshot file write fails', async () => { + const id = insertSnapshot('restore-gen-partial', [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'partial-web', filename: 'compose.yaml', content: 'services:\n snap: {}\n' }, + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'partial-web', filename: '.env', content: 'SNAP=1\n' }, + ]); + const preCompose = 'services:\n old: {}\n'; + const preEnv = 'OLD=1\n'; + seedExistingStack('partial-web', preCompose); + fs.writeFileSync(envPath('partial-web'), preEnv); + const captured = spyCaptureRecordingLive('partial-web'); + vi.spyOn(FileSystemService.prototype, 'saveEnvContent').mockRejectedValue(new Error('disk full')); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'partial-web' }); + + expect(res.status).toBe(500); + expect(captured.composeAtCapture).toBe(preCompose); + expect(captured.envAtCapture).toBe(preEnv); + expect(fs.readFileSync(composePath('partial-web'), 'utf-8')).toContain('snap: {}'); + expect(fs.readFileSync(envPath('partial-web'), 'utf-8')).toBe(preEnv); + expectCurrentGenerationMatches('partial-web', preCompose, preEnv); + }); + + it('does not capture a generation when restoring a stack that does not exist yet', async () => { + const id = insertSnapshot('restore-gen-new', [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'brand-new', filename: 'compose.yaml', content: 'services:\n snap: {}\n' }, + ]); + seedStackDir('brand-new'); + const captureSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup') + .mockResolvedValue({ id: 'should-not-run' } as never); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: LOCAL_NODE_ID, stackName: 'brand-new' }); + + expect(res.status).toBe(200); + expect(captureSpy).not.toHaveBeenCalled(); + expect(fs.readFileSync(composePath('brand-new'), 'utf-8')).toContain('snap: {}'); + }); + + it('restore-all captures existing stacks and still restores siblings when one capture fails', async () => { + const id = insertSnapshot('restore-all-gen', [ + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'all-keep', filename: 'compose.yaml', content: 'services:\n keep: {}\n' }, + { nodeId: LOCAL_NODE_ID, nodeName: 'local', stackName: 'all-fail', filename: 'compose.yaml', content: 'services:\n fail: {}\n' }, + ], 2); + seedExistingStack('all-keep', 'services:\n old-keep: {}\n'); + fs.writeFileSync(envPath('all-keep'), 'KEEP=1\n'); + seedExistingStack('all-fail', 'services:\n old-fail: {}\n'); + fs.writeFileSync(envPath('all-fail'), 'FAIL=1\n'); + vi.spyOn(StackUpdateRecoveryService.getInstance(), 'captureCurrentBackup') + .mockImplementation(async (input) => { + if (input.stackName === 'all-fail') throw new Error('generation capture failed'); + const id = await persistPreRestoreGeneration(input.stackName); + return { id } as never; + }); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore-all`) + .set('Cookie', adminCookie) + .send({}); + + expect(res.status).toBe(200); + expect(res.body.restored).toBe(1); + expect(res.body.failed).toBe(1); + const failed = (res.body.results as Array<{ stackName: string; success: boolean }>).find(r => r.stackName === 'all-fail'); + expect(failed?.success).toBe(false); + expect(fs.readFileSync(composePath('all-keep'), 'utf-8')).toContain('keep: {}'); + expect(fs.readFileSync(composePath('all-fail'), 'utf-8')).toContain('old-fail: {}'); + expect(fs.readFileSync(envPath('all-fail'), 'utf-8')).toBe('FAIL=1\n'); + expectCurrentGenerationMatches('all-keep', 'services:\n old-keep: {}\n', 'KEEP=1\n'); + expect(StackUpdateRecoveryService.getInstance().getCurrent(LOCAL_NODE_ID, 'all-fail')).toBeUndefined(); + }); + + it('restores a remote stack through one node-local apply request', async () => { + const remoteId = addRemoteNode('remote-gen'); + const id = insertSnapshot('restore-gen-remote', [ + { nodeId: remoteId, nodeName: 'remote-gen', stackName: 'rweb', filename: 'compose.yaml', content: 'services: {}\n' }, + { nodeId: remoteId, nodeName: 'remote-gen', stackName: 'rweb', filename: '.env', content: 'SNAP=1\n' }, + ]); + stubRemoteProxy(); + const calls: Array<{ url: string; method?: string; body?: string }> = []; + vi.stubGlobal('fetch', vi.fn(async (url: string, opts?: { method?: string; body?: string }) => { + calls.push({ url, method: opts?.method, body: opts?.body }); + return { ok: true, status: 200, text: async () => '' } as unknown as Response; + })); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: remoteId, stackName: 'rweb' }); + + expect(res.status).toBe(200); + const applyCalls = calls.filter(c => c.url.includes('/fleet-snapshot-apply')); + expect(applyCalls).toHaveLength(1); + expect(applyCalls[0].method).toBe('POST'); + expect(applyCalls[0].body).toContain('compose.yaml'); + expect(applyCalls[0].body).toContain('.env'); + expect(calls.some(c => c.method === 'PUT' && /\/api\/stacks\/[^/]+$/.test(c.url))).toBe(false); + expect(calls.some(c => c.method === 'PUT' && /\/env$/.test(c.url))).toBe(false); + }); + + it('fails a remote restore without writing when the node-local apply returns 409', async () => { + const remoteId = addRemoteNode('remote-lock'); + const id = insertSnapshot('restore-gen-remote-lock', [ + { nodeId: remoteId, nodeName: 'remote-lock', stackName: 'rlock', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + stubRemoteProxy(); + vi.stubGlobal('fetch', vi.fn(async () => ({ + ok: false, + status: 409, + text: async () => JSON.stringify({ error: 'locked', code: 'stack_op_in_progress' }), + } as unknown as Response))); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: remoteId, stackName: 'rlock' }); + + expect(res.status).toBe(409); + expect(res.body.code).toBe('stack_op_in_progress'); + }); + + it('does not invent stack_op_in_progress from a bare remote 409', async () => { + const remoteId = addRemoteNode('remote-bare-409'); + const id = insertSnapshot('restore-gen-remote-bare-409', [ + { nodeId: remoteId, nodeName: 'remote-bare-409', stackName: 'rbare', filename: 'compose.yaml', content: 'services: {}\n' }, + ]); + stubRemoteProxy(); + vi.stubGlobal('fetch', vi.fn(async () => ({ + ok: false, + status: 409, + text: async () => JSON.stringify({ error: 'conflict' }), + } as unknown as Response))); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore`) + .set('Cookie', adminCookie) + .send({ nodeId: remoteId, stackName: 'rbare' }); + + expect(res.status).toBe(500); + expect(res.body.code).toBeUndefined(); + }); + + it('restore-all sends one node-local apply per remote stack', async () => { + const remoteId = addRemoteNode('remote-all'); + const id = insertSnapshot('restore-all-gen-remote', [ + { nodeId: remoteId, nodeName: 'remote-all', stackName: 'ra', filename: 'compose.yaml', content: 'services:\n a: {}\n' }, + { nodeId: remoteId, nodeName: 'remote-all', stackName: 'rb', filename: 'compose.yaml', content: 'services:\n b: {}\n' }, + ], 2); + stubRemoteProxy(); + const urls: string[] = []; + vi.stubGlobal('fetch', vi.fn(async (url: string) => { + urls.push(url); + return { ok: true, status: 200, text: async () => '' } as unknown as Response; + })); + + const res = await request(app) + .post(`/api/fleet/snapshots/${id}/restore-all`) + .set('Cookie', adminCookie) + .send({}); + + expect(res.status).toBe(200); + expect(res.body.restored).toBe(2); + expect(urls.filter(u => u.includes('/fleet-snapshot-apply'))).toHaveLength(2); + expect(urls.some(u => /\/api\/stacks\/[^/]+$/.test(u))).toBe(false); + }); + + it('POST /api/stacks/:stackName/fleet-snapshot-apply returns 403 for a viewer', async () => { + const res = await request(app) + .post('/api/stacks/preimage-web/fleet-snapshot-apply') + .set('Cookie', viewerCookie) + .send({ files: [{ filename: 'compose.yaml', content: 'services: {}\n' }] }); + expect(res.status).toBe(403); + }); + + it('POST /api/stacks/:stackName/fleet-snapshot-apply captures an existing stack before writing', async () => { + const preCompose = 'services:\n old: {}\n'; + const preEnv = 'OLD=1\n'; + seedExistingStack('apply-existing', preCompose); + fs.writeFileSync(envPath('apply-existing'), preEnv); + spyCaptureRecordingLive('apply-existing'); + + const res = await request(app) + .post('/api/stacks/apply-existing/fleet-snapshot-apply') + .set('Cookie', adminCookie) + .send({ + files: [ + { filename: 'compose.yaml', content: 'services:\n snap: {}\n' }, + { filename: '.env', content: 'SNAP=1\n' }, + ], + }); + + expect(res.status).toBe(200); + expect(res.body.capturedGenerationId).toBe( + StackUpdateRecoveryService.getInstance().getCurrent(LOCAL_NODE_ID, 'apply-existing')?.id, + ); + expect(fs.readFileSync(composePath('apply-existing'), 'utf-8')).toContain('snap: {}'); + expect(fs.readFileSync(envPath('apply-existing'), 'utf-8')).toBe('SNAP=1\n'); + expectCurrentGenerationMatches('apply-existing', preCompose, preEnv); + }); + + it('POST /api/stacks/:stackName/fleet-snapshot-apply returns 400 when no restoreable files remain', async () => { + seedStackDir('apply-empty'); + const res = await request(app) + .post('/api/stacks/apply-empty/fleet-snapshot-apply') + .set('Cookie', adminCookie) + .send({ files: [{ filename: 'notes.txt', content: 'nope\n' }] }); + expect(res.status).toBe(400); + expect(fs.existsSync(composePath('apply-empty'))).toBe(false); + }); + + it('POST /api/stacks/:stackName/fleet-snapshot-apply ignores extra filenames', async () => { + seedStackDir('apply-extra'); + const res = await request(app) + .post('/api/stacks/apply-extra/fleet-snapshot-apply') + .set('Cookie', adminCookie) + .send({ + files: [ + { filename: 'compose.yaml', content: 'services:\n snap: {}\n' }, + { filename: 'notes.txt', content: 'should-not-write\n' }, + { filename: '.env', content: 'SNAP=1\n' }, + ], + }); + expect(res.status).toBe(200); + expect(fs.readFileSync(composePath('apply-extra'), 'utf-8')).toContain('snap: {}'); + expect(fs.readFileSync(envPath('apply-extra'), 'utf-8')).toBe('SNAP=1\n'); + expect(fs.existsSync(path.join(process.env.COMPOSE_DIR as string, 'apply-extra', 'notes.txt'))).toBe(false); + }); + + it('POST /api/stacks/:stackName/fleet-snapshot-apply accepts a combined body over 100 KB', async () => { + seedStackDir('apply-large'); + const compose = `services:\n snap:\n image: nginx\n labels:\n note: "${'x'.repeat(150 * 1024)}"\n`; + const res = await request(app) + .post('/api/stacks/apply-large/fleet-snapshot-apply') + .set('Cookie', adminCookie) + .send({ files: [{ filename: 'compose.yaml', content: compose }] }); + expect(res.status).toBe(200); + expect(fs.readFileSync(composePath('apply-large'), 'utf-8')).toBe(compose); + }); +}); diff --git a/backend/src/__tests__/git-source-apply-recovery.test.ts b/backend/src/__tests__/git-source-apply-recovery.test.ts new file mode 100644 index 00000000..452cc714 --- /dev/null +++ b/backend/src/__tests__/git-source-apply-recovery.test.ts @@ -0,0 +1,283 @@ +/** + * R1: Git apply promote succeeds, deploy fails → applied true, generation current, + * compensateWithCandidate is not called. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockCaptureCandidate = vi.fn(); +const mockAbandon = vi.fn(); +const mockMarkAcquired = vi.fn().mockReturnValue(true); +const mockHandoff = vi.fn().mockReturnValue(true); +const mockMarkReconciling = vi.fn().mockReturnValue(true); +const mockMarkImmediateVerified = vi.fn().mockReturnValue(true); +const mockGet = vi.fn(); +const mockCompensate = vi.fn(); + +vi.mock('../services/StackUpdateRecoveryService', () => ({ + StackUpdateRecoveryService: { + getInstance: () => ({ + captureCandidate: mockCaptureCandidate, + abandon: mockAbandon, + markAcquired: mockMarkAcquired, + handoff: mockHandoff, + markReconciling: mockMarkReconciling, + markImmediateVerified: mockMarkImmediateVerified, + get: mockGet, + compensateWithCandidate: mockCompensate, + }), + }, +})); + +const mockDeployStack = vi.fn(); +vi.mock('../services/ComposeService', () => ({ + ComposeService: { + getInstance: () => ({ + deployStack: mockDeployStack, + }), + }, +})); + +vi.mock('../services/StackOpLockService', () => ({ + StackOpLockService: { + getInstance: () => ({ + runExclusive: async ( + _n: number, + _s: string, + _a: string, + _who: string, + fn: () => Promise, + ) => { + const result = await fn(); + return { ran: true, result }; + }, + }), + }, +})); + +vi.mock('../helpers/policyGate', () => ({ + assertPolicyGateAllows: vi.fn().mockResolvedValue(undefined), + buildSystemPolicyGateOptions: vi.fn().mockReturnValue({}), +})); + +vi.mock('../services/HealthGateService', () => ({ + HealthGateService: { + getInstance: () => ({ + beginStack: vi.fn(), + }), + }, +})); + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getDefaultNodeId: () => 1, + }), + }, +})); + +const mockPromoteGeneration = vi.fn().mockResolvedValue(undefined); +vi.mock('../services/GitProjectManifestService', () => ({ + GitProjectManifestService: { + getInstance: () => ({ + readManifest: vi.fn().mockResolvedValue(null), + buildManifest: vi.fn().mockReturnValue({ + manifestVersion: 1, + state: 'active', + inputs: [], + refusals: [], + generation: { candidateDir: 'c', appliedDir: 'a', previousDir: null }, + }), + promoteGeneration: mockPromoteGeneration, + boundsConfig: vi.fn().mockReturnValue({}), + hashStackFile: vi.fn(), + verifyContextOnDisk: vi.fn().mockResolvedValue([]), + writeManifest: vi.fn(), + buildMigratedManifest: vi.fn(), + }), + }, +})); + +vi.mock('../utils/authoredComposeArgs', () => ({ + authoredComposeFileArgs: vi.fn().mockResolvedValue(['-f', 'compose.yaml']), + authoredComposeEnvFileArgs: vi.fn().mockResolvedValue([]), +})); + +const mockGetGitSource = vi.fn(); +const mockMarkGitSourceApplied = vi.fn(); +const mockSetGitSourceAppliedSpec = vi.fn(); +const mockSetGitSourceManifestState = vi.fn(); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getGitSource: mockGetGitSource, + markGitSourceApplied: mockMarkGitSourceApplied, + setGitSourceAppliedSpec: mockSetGitSourceAppliedSpec, + setGitSourceManifestState: mockSetGitSourceManifestState, + }), + }, +})); + +vi.mock('../services/CryptoService', () => ({ + CryptoService: { + getInstance: () => ({ + decrypt: (v: string) => v, + encrypt: (v: string) => v, + }), + }, +})); + +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + promises: { + ...actual.promises, + access: vi.fn().mockResolvedValue(undefined), + }, + }; +}); + +describe('git-source apply recovery (R1)', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockCaptureCandidate.mockResolvedValue({ + id: 'rec-1', + node_id: 1, + stack_name: 'app', + status: 'candidate', + phase: 'captured', + is_current: 0, + }); + mockGet.mockReturnValue({ + id: 'rec-1', + is_current: 1, + status: 'active', + phase: 'reconciling', + }); + mockDeployStack.mockRejectedValue(new Error('compose up failed')); + mockGetGitSource.mockReturnValue({ + stack_name: 'app', + repo_url: 'https://example.com/repo.git', + branch: 'main', + pending_commit_sha: 'abc1234deadbeef', + pending_compose_content: JSON.stringify({ + v: 3, + files: { 'compose.yaml': 'services:\n web:\n image: nginx\n' }, + contextDir: null, + candidateRelPath: 'generations/cand', + inventory: { + inputs: [], + refusals: [], + buildContexts: [], + }, + }), + pending_env_content: null, + sync_env: false, + compose_paths: ['compose.yaml'], + context_dir: null, + auto_deploy_on_apply: true, + applied_deploy_spec: null, + }); + }); + + it('keeps applied=true and generation current without compensate when deploy fails', async () => { + const { GitSourceService } = await import('../services/GitSourceService'); + // Avoid withStackLock contention by calling applyLocked through apply + // after stubbing the lock if present. + const svc = GitSourceService.getInstance(); + const withLock = vi.spyOn( + svc as unknown as { withStackLock: (name: string, fn: () => Promise) => Promise }, + 'withStackLock', + ); + withLock.mockImplementation(async (_name, fn) => fn()); + + // validateCandidate is used on the v3 path; stub it open. + vi.spyOn( + svc as unknown as { + validateCandidate: (...args: unknown[]) => Promise<{ ok: boolean }>; + }, + 'validateCandidate', + ).mockResolvedValue({ ok: true }); + + vi.spyOn( + svc as unknown as { + decodePendingCompose: (raw: string) => unknown; + }, + 'decodePendingCompose', + ).mockReturnValue({ + files: { 'compose.yaml': 'services:\n web:\n image: nginx\n' }, + contextDir: null, + candidateRelPath: 'generations/cand', + inventory: { inputs: [], refusals: [], buildContexts: [] }, + }); + + vi.spyOn( + svc as unknown as { + deriveAppliedSpec: (...args: unknown[]) => unknown; + }, + 'deriveAppliedSpec', + ).mockReturnValue({ files: ['compose.yaml'], contextDir: null }); + + vi.spyOn( + svc as unknown as { + hashContent: (...args: unknown[]) => string; + }, + 'hashContent', + ).mockReturnValue('hash'); + + const result = await svc.apply('app', 'abc1234deadbeef', { deploy: true, actor: 'tester' }); + + expect(result.applied).toBe(true); + expect(result.deployed).toBe(false); + expect(result.deployError).toBeTruthy(); + expect(result.recoveryId).toBe('rec-1'); + expect(mockPromoteGeneration).toHaveBeenCalled(); + expect(mockCaptureCandidate).toHaveBeenCalledWith( + expect.objectContaining({ operationKind: 'git_apply', stackName: 'app' }), + ); + expect(mockHandoff).toHaveBeenCalled(); + expect(mockCompensate).not.toHaveBeenCalled(); + expect(mockAbandon).not.toHaveBeenCalled(); + }); + + it('refuses to promote when recovery capture fails', async () => { + mockCaptureCandidate.mockRejectedValue(new Error('Exact authored-project rollback coverage is unavailable')); + mockDeployStack.mockResolvedValue({ recoveryId: null }); + + const { GitSourceService, GitSourceError } = await import('../services/GitSourceService'); + const svc = GitSourceService.getInstance(); + vi.spyOn( + svc as unknown as { withStackLock: (name: string, fn: () => Promise) => Promise }, + 'withStackLock', + ).mockImplementation(async (_name, fn) => fn()); + vi.spyOn( + svc as unknown as { validateCandidate: (...args: unknown[]) => Promise<{ ok: boolean }> }, + 'validateCandidate', + ).mockResolvedValue({ ok: true }); + vi.spyOn( + svc as unknown as { decodePendingCompose: (raw: string) => unknown }, + 'decodePendingCompose', + ).mockReturnValue({ + files: { 'compose.yaml': 'services:\n web:\n image: nginx\n' }, + contextDir: null, + candidateRelPath: 'generations/cand', + inventory: { inputs: [], refusals: [], buildContexts: [] }, + }); + vi.spyOn( + svc as unknown as { deriveAppliedSpec: (...args: unknown[]) => unknown }, + 'deriveAppliedSpec', + ).mockReturnValue({ files: ['compose.yaml'], contextDir: null }); + vi.spyOn( + svc as unknown as { hashContent: (...args: unknown[]) => string }, + 'hashContent', + ).mockReturnValue('hash'); + + await expect(svc.apply('app', 'abc1234deadbeef', { deploy: true, actor: 'tester' })).rejects.toBeInstanceOf(GitSourceError); + expect(mockPromoteGeneration).not.toHaveBeenCalled(); + expect(mockCaptureCandidate).toHaveBeenCalled(); + expect(mockMarkGitSourceApplied).not.toHaveBeenCalled(); + expect(mockHandoff).not.toHaveBeenCalled(); + expect(mockAbandon).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index b8f0c777..d2a8745b 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -32,6 +32,44 @@ vi.mock('isomorphic-git', () => { vi.mock('isomorphic-git/http/node', () => ({ default: {} })); + +const { + mockCaptureCandidate, + mockRecoveryAbandon, + mockRecoveryMarkAcquired, + mockRecoveryHandoff, + mockRecoveryMarkReconciling, + mockRecoveryMarkImmediateVerified, + mockRecoveryGet, + mockRecoveryLinkGateOrRetain, +} = vi.hoisted(() => ({ + mockCaptureCandidate: vi.fn(async () => ({ id: 'rec-test-1' })), + mockRecoveryAbandon: vi.fn(async () => true), + mockRecoveryMarkAcquired: vi.fn(() => true), + mockRecoveryHandoff: vi.fn(() => true), + mockRecoveryMarkReconciling: vi.fn(() => true), + mockRecoveryMarkImmediateVerified: vi.fn(() => true), + mockRecoveryGet: vi.fn(() => ({ id: 'rec-test-1', is_current: 1 })), + mockRecoveryLinkGateOrRetain: vi.fn(), +})); + +vi.mock('../services/StackUpdateRecoveryService', () => ({ + StackUpdateRecoveryService: { + getInstance: () => ({ + captureCandidate: mockCaptureCandidate, + abandon: mockRecoveryAbandon, + markAcquired: mockRecoveryMarkAcquired, + handoff: mockRecoveryHandoff, + markReconciling: mockRecoveryMarkReconciling, + markImmediateVerified: mockRecoveryMarkImmediateVerified, + get: mockRecoveryGet, + linkGateOrRetain: mockRecoveryLinkGateOrRetain, + compensateWithCandidate: vi.fn(async () => true), + }), + }, +})); + + let tmpDir: string; let GitSourceService: typeof import('../services/GitSourceService').GitSourceService; let GitSourceError: typeof import('../services/GitSourceService').GitSourceError; @@ -50,6 +88,21 @@ afterAll(() => { beforeEach(() => { mockGitClone.mockReset(); mockGitLog.mockReset(); + mockCaptureCandidate.mockReset(); + mockCaptureCandidate.mockImplementation(async () => ({ id: 'rec-test-1' })); + mockRecoveryAbandon.mockReset(); + mockRecoveryAbandon.mockResolvedValue(true); + mockRecoveryMarkAcquired.mockReset(); + mockRecoveryMarkAcquired.mockReturnValue(true); + mockRecoveryHandoff.mockReset(); + mockRecoveryHandoff.mockReturnValue(true); + mockRecoveryMarkReconciling.mockReset(); + mockRecoveryMarkReconciling.mockReturnValue(true); + mockRecoveryMarkImmediateVerified.mockReset(); + mockRecoveryMarkImmediateVerified.mockReturnValue(true); + mockRecoveryGet.mockReset(); + mockRecoveryLinkGateOrRetain.mockReset(); + mockRecoveryGet.mockReturnValue({ id: 'rec-test-1', is_current: 1 }); // Wipe persisted git sources between tests const db = DatabaseService.getInstance(); @@ -894,6 +947,47 @@ describe('GitSourceService.handleWebhookPull debounce', () => { expect(result.message).toMatch(/validation failed/i); runSpy.mockRestore(); }); + + it('routes webhook auto-apply through the shared stack-operation lock', async () => { + const sha = 'ffff666ffff666ffff666ffff666ffff666ffff6'; + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + await svc.upsert({ + stackName: 'webhook-shared-lock', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: true, + autoDeployOnApply: false, + }); + mockGitClone.mockClear(); + + const { StackOpLockService } = await import('../services/StackOpLockService'); + const runExclusive = vi.spyOn(StackOpLockService.getInstance(), 'runExclusive') + .mockResolvedValue({ + ran: false, + existing: { action: 'update', actor: 'user:admin', startedAt: Date.now() }, + } as never); + + const result = await svc.handleWebhookPull('webhook-shared-lock'); + expect(result.status).toBe('error'); + expect(result.message).toMatch(/already in progress/i); + expect(runExclusive).toHaveBeenCalledWith( + expect.any(Number), + 'webhook-shared-lock', + 'git_apply', + 'system:webhook', + expect.any(Function), + ); + + runExclusive.mockRestore(); + validateSpy.mockRestore(); + }); }); describe('GitSourceService per-stack mutex', () => { @@ -1359,7 +1453,7 @@ describe('GitSourceService.apply', () => { const { ComposeService } = await import('../services/ComposeService'); const { HealthGateService } = await import('../services/HealthGateService'); const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-git'); const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; @@ -1371,6 +1465,7 @@ describe('GitSourceService.apply', () => { actor: 'system:git-source', }); expect(beginSpy).toHaveBeenCalledWith(nodeId, 'apply-deploy-gate', 'deploy', 'system:git-source'); + expect(mockRecoveryLinkGateOrRetain).toHaveBeenCalledWith('rec-test-1', 'gate-git'); } finally { validateSpy.mockRestore(); saveSpy.mockRestore(); @@ -1470,7 +1565,7 @@ describe('GitSourceService.apply', () => { const TrivyService = (await import('../services/TrivyService')).default; const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); const listImagesSpy = vi.spyOn(ComposeService.prototype, 'listStackImages').mockResolvedValue(['nginx:bad']); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); const trivy = TrivyService.getInstance(); const trivyAvailableSpy = vi.spyOn(trivy, 'isTrivyAvailable').mockReturnValue(true); const scanSpy = vi.spyOn(trivy, 'scanImagePreflight').mockResolvedValue({ @@ -2178,6 +2273,50 @@ describe('GitSourceService legacy pending apply (migration path)', () => { await cleanupStackDir('legacy-apply'); } }); + + it('rejects materialize when deleting a stale override fails', async () => { + const svc = GitSourceService.getInstance(); + const { FileSystemService } = await import('../services/FileSystemService'); + const fsSvc = FileSystemService.getInstance(); + const stackName = 'legacy-stale-del'; + await fsSvc.createStack(stackName); + await fsSvc.saveStackContent(stackName, 'services:\n web:\n image: nginx\n'); + await fsSvc.writeStackFile(stackName, 'compose.override.yaml', 'services:\n web:\n environment: [X=1]\n'); + + const deleteSpy = vi.spyOn(FileSystemService.prototype, 'deleteStackPath') + .mockRejectedValue(Object.assign(new Error('permission denied'), { code: 'EACCES' })); + + const materialize = (svc as unknown as { + materialize: ( + stackName: string, + files: Array<{ path: string; content: string }>, + contextDir: string | null, + syncEnv: boolean, + envContent: string | null, + prevSpec: { files: string[]; contextDir: string | null } | null, + ) => Promise; + }).materialize.bind(svc); + + try { + await expect( + materialize( + stackName, + [{ path: 'compose.yaml', content: 'services:\n web:\n image: alpine\n' }], + null, + false, + null, + { files: ['compose.yaml', 'compose.override.yaml'], contextDir: null }, + ), + ).rejects.toThrow(/permission denied/); + expect(deleteSpy).toHaveBeenCalledWith(stackName, 'compose.override.yaml'); + // Stale override must still be present; apply must not report success over a hybrid. + const override = await fsSvc.readStackFile(stackName, 'compose.override.yaml'); + expect(override.content).toContain('X=1'); + } finally { + deleteSpy.mockRestore(); + await cleanupStackDir(stackName); + } + }); }); describe('GitSourceService sync-env stacks with a repo .env (audit C-2)', () => { diff --git a/backend/src/__tests__/operational-permission-matrix.test.ts b/backend/src/__tests__/operational-permission-matrix.test.ts index a876ba04..e3046919 100644 --- a/backend/src/__tests__/operational-permission-matrix.test.ts +++ b/backend/src/__tests__/operational-permission-matrix.test.ts @@ -43,6 +43,7 @@ describe('named stack route permission inventory', () => { ['PUT', '/stacks/web/env', 'stack:edit'], ['PUT', '/stacks/web/dossier', 'stack:edit'], ['PUT', '/stacks/web/labels', 'stack:edit'], + ['POST', '/stacks/web/fleet-snapshot-apply', 'stack:edit'], ['POST', '/stacks/web/deploy', 'stack:deploy'], ['POST', '/stacks/web/stop', 'stack:deploy'], ['POST', '/stacks/web/services/api/update', 'stack:deploy'], diff --git a/backend/src/__tests__/recovery-captured-invocation.test.ts b/backend/src/__tests__/recovery-captured-invocation.test.ts new file mode 100644 index 00000000..b62d86a6 --- /dev/null +++ b/backend/src/__tests__/recovery-captured-invocation.test.ts @@ -0,0 +1,102 @@ +/** + * Recovery must execute the generation-captured Compose invocation, not the + * live database-derived file / env selection after capture. + */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import path from 'path'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +describe('captured invocation on recovery Compose args', () => { + let tmpDir: string; + let composeDir: string; + let nodeId: number; + + beforeEach(async () => { + tmpDir = await setupTestDb(); + composeDir = process.env.COMPOSE_DIR!; + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + const local = db.getDefaultNode(); + if (!local?.id) throw new Error('missing default node'); + nodeId = local.id; + }); + + afterEach(() => { + if (tmpDir) cleanupTestDb(tmpDir); + }); + + it('uses captured -f order and env-file when live settings diverge', async () => { + const stackName = 'inv-capture'; + const stackDir = path.join(composeDir, stackName); + const fsPromises = await import('fs/promises'); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n a: {}\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'compose.prod.yaml'), 'services:\n b: {}\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'prod.env'), 'A=1\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'dev.env'), 'A=2\n', 'utf8'); + + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + db.upsertGitSource({ + stack_name: stackName, + repo_url: 'https://example.com/repo.git', + branch: 'main', + compose_path: 'compose.yaml', + compose_paths: ['compose.yaml', 'compose.prod.yaml'], + context_dir: null, + sync_env: false, + env_path: null, + auth_type: 'none', + encrypted_token: null, + auto_apply_on_webhook: false, + auto_deploy_on_apply: false, + last_applied_commit_sha: 'abc', + last_applied_content_hash: 'h', + pending_commit_sha: null, + pending_compose_content: null, + pending_env_content: null, + pending_fetched_at: null, + last_debounce_at: null, + }); + db.getDb().prepare( + `UPDATE stack_git_sources SET applied_deploy_spec = ? WHERE stack_name = ?`, + ).run( + JSON.stringify({ files: ['compose.yaml'], contextDir: null }), + stackName, + ); + db.setStackProjectEnvFiles(nodeId, stackName, ['dev.env']); + + const { ComposeService } = await import('../services/ComposeService'); + const svc = ComposeService.getInstance(nodeId); + const captured = { + composeArgsPrefix: [ + '-f', 'compose.yaml', + '-f', 'compose.prod.yaml', + '-p', stackName, + '--env-file', path.join(stackDir, 'prod.env'), + ], + projectDirectory: null, + projectName: stackName, + explicitComposeFiles: ['compose.yaml', 'compose.prod.yaml'], + }; + + const args = await svc.buildComposeArgsWithRecoveryOverride( + stackName, + ['up', '-d'], + path.join(stackDir, '.sencho-recovery-aaaaaaaaaaaa.yml'), + captured, + ); + + expect(args).toEqual([ + 'compose', + '-f', 'compose.yaml', + '-f', 'compose.prod.yaml', + '-p', stackName, + '--env-file', path.resolve(stackDir, 'prod.env'), + '-f', path.join(stackDir, '.sencho-recovery-aaaaaaaaaaaa.yml'), + 'up', '-d', + ]); + // Live settings would prefer only compose.yaml + dev.env; captured must win. + expect(args.join('\0')).not.toContain('dev.env'); + }); +}); diff --git a/backend/src/__tests__/recovery-services-json-fifth-audit.test.ts b/backend/src/__tests__/recovery-services-json-fifth-audit.test.ts new file mode 100644 index 00000000..1a4aa895 --- /dev/null +++ b/backend/src/__tests__/recovery-services-json-fifth-audit.test.ts @@ -0,0 +1,70 @@ +/** + * Fifth-audit regressions: structural services_json validation. + */ +import { describe, expect, it } from 'vitest'; +import { + parseServicesJsonStrict, + scrapeRollbackTagsLenient, + type StackRecoveryServiceCapture, +} from '../services/recoveryServicesJson'; + +function validService(over: Partial = {}): StackRecoveryServiceCapture { + return { + serviceName: 'web', + scale: 1, + hasBuild: false, + declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', + replicas: [{ + containerId: 'c1', + imageId: 'sha256:aaa', + repoDigest: null, + state: 'running', + rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold', + }], + ...over, + }; +} + +describe('parseServicesJsonStrict', () => { + it('accepts a well-formed services array', () => { + const parsed = parseServicesJsonStrict(JSON.stringify([validService()])); + expect(parsed.ok).toBe(true); + if (parsed.ok) expect(parsed.services[0].serviceName).toBe('web'); + }); + + it('rejects empty object elements like [{}]', () => { + expect(parseServicesJsonStrict('[{}]').ok).toBe(false); + }); + + it('rejects malformed replica arrays', () => { + expect(parseServicesJsonStrict(JSON.stringify([{ + ...validService(), + replicas: [{ state: 'running' }], + }])).ok).toBe(false); + + expect(parseServicesJsonStrict(JSON.stringify([{ + ...validService(), + replicas: 'nope', + }])).ok).toBe(false); + }); + + it('scrapeRollbackTagsLenient recovers tags from near-valid JSON', () => { + expect(scrapeRollbackTagsLenient(JSON.stringify([{ + serviceName: 'web', + replicas: [{ rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold' }], + }]))).toEqual(['sencho-rb/aaaaaaaaaaaa/web:hold']); + }); + + it('rejects invalid referenceKind or scale', () => { + expect(parseServicesJsonStrict(JSON.stringify([{ + ...validService(), + referenceKind: 'mystery', + }])).ok).toBe(false); + + expect(parseServicesJsonStrict(JSON.stringify([{ + ...validService(), + scale: -1, + }])).ok).toBe(false); + }); +}); diff --git a/backend/src/__tests__/rollback-eligibility-assess.test.ts b/backend/src/__tests__/rollback-eligibility-assess.test.ts new file mode 100644 index 00000000..c3dac602 --- /dev/null +++ b/backend/src/__tests__/rollback-eligibility-assess.test.ts @@ -0,0 +1,103 @@ +/** + * Integration coverage for assessGenerationEligibility: structural services_json + * refusal and opaque hold-tag presence checks. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockInspectByRef = vi.fn(async (_ref: string) => ({ Id: 'ok' })); + +vi.mock('../services/DockerController', () => ({ + default: { + getInstance: () => ({ + getDocker: () => ({ + getImage: (ref: string) => ({ + inspect: () => mockInspectByRef(ref), + }), + }), + }), + }, +})); + +vi.mock('../services/RollbackGenerationStore', () => ({ + RollbackGenerationStore: { + verifyGenerationContent: vi.fn().mockResolvedValue(true), + }, +})); + +vi.mock('../services/PolicyEnforcement', () => ({ + enforcePolicyForImageRefs: vi.fn().mockResolvedValue({ ok: true, bypassed: false, violations: [] }), +})); + +import { assessGenerationEligibility } from '../services/rollbackEligibility'; +import type { StackUpdateRecoveryGenerationRow } from '../services/DatabaseService'; + +const HOLD_TAG = 'sencho-rb/aaaaaaaaaaaa/web:hold'; +const IMAGE_ID = 'sha256:aaa'; + +function baseRow(over: Partial = {}): StackUpdateRecoveryGenerationRow { + return { + id: '11111111-1111-4111-8111-111111111111', + node_id: 1, + stack_name: 'my-stack', + status: 'active', + phase: 'reconciling', + is_current: 1, + backup_slot_id: '11111111-1111-4111-8111-111111111111', + content_path: '11111111-1111-4111-8111-111111111111', + operation_kind: 'update', + override_path: null, + services_json: JSON.stringify([{ + serviceName: 'web', + scale: 1, + hasBuild: false, + declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', + replicas: [{ + containerId: 'c1', + imageId: IMAGE_ID, + repoDigest: null, + state: 'running', + rollbackTag: HOLD_TAG, + }], + }]), + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + ...over, + }; +} + +describe('assessGenerationEligibility', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockInspectByRef.mockResolvedValue({ Id: 'ok' }); + }); + + it('prohibits structurally malformed services_json such as [{}]', async () => { + await expect(assessGenerationEligibility(baseRow({ services_json: '[{}]' }))).resolves.toBe('prohibited'); + }); + + it('returns eligible_with_warning when an opaque hold tag is missing', async () => { + mockInspectByRef.mockImplementation(async (ref: string) => { + if (ref === HOLD_TAG) { + throw Object.assign(new Error('No such image'), { statusCode: 404 }); + } + return { Id: 'ok' }; + }); + + await expect(assessGenerationEligibility(baseRow())).resolves.toBe('eligible_with_warning'); + expect(mockInspectByRef).toHaveBeenCalledWith(IMAGE_ID); + expect(mockInspectByRef).toHaveBeenCalledWith(HOLD_TAG); + }); + + it('returns eligible when image ids and hold tags both resolve', async () => { + await expect(assessGenerationEligibility(baseRow())).resolves.toBe('eligible'); + }); +}); diff --git a/backend/src/__tests__/rollback-eligibility.test.ts b/backend/src/__tests__/rollback-eligibility.test.ts new file mode 100644 index 00000000..f5ccacf0 --- /dev/null +++ b/backend/src/__tests__/rollback-eligibility.test.ts @@ -0,0 +1,55 @@ +/** + * Unit tests for evaluateRollbackEligibility (pure verdict mapping). + */ +import { describe, expect, it } from 'vitest'; +import { + evaluateRollbackEligibility, + type RollbackEligibilityInput, +} from '../services/rollbackEligibility'; + +const base = (over: Partial = {}): RollbackEligibilityInput => ({ + generationIntegrityOk: true, + heldImagesPresent: true, + securityPostureBlocked: false, + ...over, +}); + +describe('evaluateRollbackEligibility', () => { + it('returns eligible when every signal is known-good', () => { + expect(evaluateRollbackEligibility(base())).toBe('eligible'); + }); + + it('returns prohibited when security posture is blocked', () => { + expect(evaluateRollbackEligibility(base({ securityPostureBlocked: true }))).toBe('prohibited'); + }); + + it('returns prohibited when generation integrity is known bad', () => { + expect(evaluateRollbackEligibility(base({ generationIntegrityOk: false }))).toBe('prohibited'); + }); + + it('prefers prohibited over other signals when security is blocked', () => { + expect(evaluateRollbackEligibility(base({ + securityPostureBlocked: true, + generationIntegrityOk: null, + heldImagesPresent: false, + }))).toBe('prohibited'); + }); + + it('returns eligible_with_warning when held images are missing', () => { + expect(evaluateRollbackEligibility(base({ heldImagesPresent: false }))).toBe('eligible_with_warning'); + }); + + it('returns unknown when any remaining signal is null', () => { + expect(evaluateRollbackEligibility(base({ generationIntegrityOk: null }))).toBe('unknown'); + expect(evaluateRollbackEligibility(base({ heldImagesPresent: null }))).toBe('unknown'); + expect(evaluateRollbackEligibility(base({ securityPostureBlocked: null }))).toBe('unknown'); + }); + + it('returns unknown when all signals are null', () => { + expect(evaluateRollbackEligibility({ + generationIntegrityOk: null, + heldImagesPresent: null, + securityPostureBlocked: null, + })).toBe('unknown'); + }); +}); diff --git a/backend/src/__tests__/rollback-generation-lifecycle.test.ts b/backend/src/__tests__/rollback-generation-lifecycle.test.ts index bfd4045d..b13e6915 100644 --- a/backend/src/__tests__/rollback-generation-lifecycle.test.ts +++ b/backend/src/__tests__/rollback-generation-lifecycle.test.ts @@ -82,6 +82,8 @@ function makeRow(overrides: Partial = {}): Sta phase: 'immediate_verified', is_current: 1, backup_slot_id: null, + content_path: null, + operation_kind: null, override_path: null, services_json: JSON.stringify([{ serviceName: 'web', @@ -349,6 +351,22 @@ describe('StackUpdateRecoveryService.releaseGeneration', () => { if (!second.ok) expect(second.reason).toBe('already_released'); }); + it('refuses release when services_json is malformed and leaves hold tags intact', async () => { + const row = insertRow({ status: 'active', is_current: 1, services_json: '{not-json' }); + const svc = StackUpdateRecoveryService.getInstance(); + + expect(svc.isReleaseEligible(row)).toBe(false); + const result = await svc.releaseGeneration(row.id, 'tester'); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe('malformed_services'); + + const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!; + expect(after.released_at).toBeNull(); + expect(after.is_current).toBe(1); + expect(after.artifacts_retired).toBe(0); + expect(mockRemove).not.toHaveBeenCalled(); + }); + it('leaves artifacts_retired at 0 (retryable) when Docker tag removal fails', async () => { mockRemove.mockRejectedValueOnce(Object.assign(new Error('docker busy'), { statusCode: 500 })); const row = insertRow({ status: 'active', is_current: 1 }); @@ -415,6 +433,18 @@ describe('GET/POST /api/system/rollback/generations', () => { expect(res.status).toBe(409); expect(res.body.code).toBe('NOT_ELIGIBLE'); }); + + it('409s releasing a generation with malformed services_json', async () => { + const row = insertRow({ status: 'superseded', is_current: 0, services_json: '{not-json' }); + const res = await request(app) + .post(`/api/system/rollback/generations/${row.id}/release`) + .set('Authorization', authHeader); + + expect(res.status).toBe(409); + expect(res.body.code).toBe('MALFORMED_SERVICES'); + const after = DatabaseService.getInstance().getStackUpdateRecoveryGeneration(row.id)!; + expect(after.released_at).toBeNull(); + }); }); describe('RBAC on the rollback-generation routes', () => { diff --git a/backend/src/__tests__/rollback-generation-store.test.ts b/backend/src/__tests__/rollback-generation-store.test.ts new file mode 100644 index 00000000..dee09b27 --- /dev/null +++ b/backend/src/__tests__/rollback-generation-store.test.ts @@ -0,0 +1,788 @@ +/** + * Unit tests for RollbackGenerationStore: capture staging, checksum verify, + * restore round-trip, mid-capture failure isolation, and managed-set tombstones. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import path from 'path'; +import os from 'os'; +import { promises as fsPromises } from 'fs'; +import { randomUUID } from 'crypto'; + +const mockState = { composeDir: '', composeDirs: new Map() }; + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getComposeDir: (nodeId?: number) => mockState.composeDirs.get(nodeId ?? 1) ?? mockState.composeDir, + getDefaultNodeId: () => 1, + }), + }, +})); + +vi.mock('../utils/debug', () => ({ + isDebugEnabled: () => false, +})); + +import { RollbackGenerationStore } from '../services/RollbackGenerationStore'; +import type { ResolvedRollbackInventory } from '../types/rollbackGeneration'; + +const NODE = 1; + +function inventoryFor( + stackName: string, + files: Array<{ + relativePath: string; + absolutePath: string | null; + kind?: ResolvedRollbackInventory['entries'][number]['dependencyKind']; + sensitivity?: ResolvedRollbackInventory['entries'][number]['sensitivity']; + }>, +): ResolvedRollbackInventory { + return { + entries: files.map((f) => ({ + relativePath: f.relativePath, + dependencyKind: f.kind ?? 'compose-root', + provenance: 'authored' as const, + sensitivity: f.sensitivity ?? 'low', + absolutePath: f.absolutePath, + })), + invocation: { + composeArgsPrefix: [], + projectDirectory: null, + projectName: stackName, + explicitComposeFiles: files.map((f) => f.relativePath).filter((p) => !p.includes('/')), + meshEnabled: false, + meshOverrideRelativePath: null, + }, + git: null, + appliedDeploySpec: null, + lastAppliedContentHash: null, + manifestState: null, + manifestGeneration: null, + exactCoverage: true, + coverageRefusal: null, + }; +} + +describe('RollbackGenerationStore', () => { + let composeDir: string; + let dataDir: string; + let originalDataDir: string | undefined; + + beforeEach(async () => { + composeDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-compose-')); + dataDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-data-')); + mockState.composeDir = composeDir; + mockState.composeDirs = new Map([[1, composeDir]]); + originalDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = dataDir; + }); + + afterEach(async () => { + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + await fsPromises.rm(composeDir, { recursive: true, force: true }); + await fsPromises.rm(dataDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it('captures and restores a nested path round-trip', async () => { + const stackName = 'web'; + const stackDir = path.join(composeDir, stackName); + const nestedRel = 'includes/base.yaml'; + await fsPromises.mkdir(path.join(stackDir, 'includes'), { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: a\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, nestedRel), 'services: {}\n', 'utf8'); + + const generationId = randomUUID(); + const inv = inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + { relativePath: nestedRel, absolutePath: path.join(stackDir, nestedRel), kind: 'include' }, + ]); + + const manifest = await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inv, + operationKind: 'manual_backup', + }); + + expect(manifest.managedRelativePaths).toEqual(['compose.yaml', nestedRel].sort((a, b) => a.localeCompare(b))); + const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + await expect(fsPromises.access(path.join(genDir, 'generation.json'))).resolves.toBeUndefined(); + await expect(fsPromises.access(path.join(genDir, 'files', nestedRel))).resolves.toBeUndefined(); + + // Mutate live files, then restore + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: { mutated: {} }\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, nestedRel), 'services: { mutated: true }\n', 'utf8'); + + await RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, [ + 'compose.yaml', + nestedRel, + ]); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe( + 'services:\n app:\n image: a\n', + ); + expect(await fsPromises.readFile(path.join(stackDir, nestedRel), 'utf8')).toBe('services: {}\n'); + }); + + it('leaves no final generation dir when capture fails mid-write', async () => { + const stackName = 'failcap'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + await fsPromises.mkdir(path.join(stackDir, 'configs'), { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'configs', 'app.conf'), 'ok\n', 'utf8'); + + const generationId = randomUUID(); + const inv = inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + { + relativePath: 'configs/app.conf', + absolutePath: path.join(stackDir, 'configs', 'app.conf'), + kind: 'config', + sensitivity: 'high', + }, + ]); + + const realWriteFile = fsPromises.writeFile.bind(fsPromises); + const writeSpy = vi.spyOn(fsPromises, 'writeFile').mockImplementation(async (file, data, options) => { + const normalized = path.normalize(String(file)); + if (normalized.includes(`${path.sep}configs${path.sep}app.conf`) && normalized.includes('staging-')) { + throw new Error('disk full during capture'); + } + return realWriteFile(file, data, options); + }); + + try { + await expect( + RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inv, + }), + ).rejects.toThrow(/disk full during capture/); + + const finalDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + await expect(fsPromises.access(finalDir)).rejects.toMatchObject({ code: 'ENOENT' }); + + const gensRoot = RollbackGenerationStore.getGenerationsRoot(NODE, stackName); + let leftoverStaging = false; + try { + const entries = await fsPromises.readdir(gensRoot); + leftoverStaging = entries.some((e) => e.startsWith('staging-')); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + } + expect(leftoverStaging).toBe(false); + } finally { + writeSpy.mockRestore(); + } + }); + + it('refuses restore when a checksum is corrupt before mutating live files', async () => { + const stackName = 'corrupt'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + const original = 'services:\n app:\n image: good\n'; + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original, 'utf8'); + + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + ]), + }); + + const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + await fsPromises.writeFile(path.join(genDir, 'files', 'compose.yaml'), 'services: { tampered: true }\n', 'utf8'); + + const liveMutated = 'services: { live: true }\n'; + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), liveMutated, 'utf8'); + + await expect( + RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, ['compose.yaml']), + ).rejects.toThrow(/Checksum mismatch/); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe(liveMutated); + }); + + it('tombstones post-capture managed additions and leaves unrelated files', async () => { + const stackName = 'tomb'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(path.join(stackDir, 'configs'), { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'configs', 'app.conf'), 'v1\n', 'utf8'); + + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + { + relativePath: 'configs/app.conf', + absolutePath: path.join(stackDir, 'configs', 'app.conf'), + kind: 'config', + sensitivity: 'high', + }, + ]), + }); + + // Post-capture addition inside the managed discovery set + await fsPromises.writeFile(path.join(stackDir, 'configs', 'extra.conf'), 'new\n', 'utf8'); + // Unrelated file outside managed discovery + await fsPromises.writeFile(path.join(stackDir, 'README.md'), 'keep me\n', 'utf8'); + // Mutate a present managed file + await fsPromises.writeFile(path.join(stackDir, 'configs', 'app.conf'), 'v2\n', 'utf8'); + + await RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, [ + 'compose.yaml', + 'configs/app.conf', + 'configs/extra.conf', + ]); + + expect(await fsPromises.readFile(path.join(stackDir, 'configs', 'app.conf'), 'utf8')).toBe('v1\n'); + await expect(fsPromises.access(path.join(stackDir, 'configs', 'extra.conf'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + expect(await fsPromises.readFile(path.join(stackDir, 'README.md'), 'utf8')).toBe('keep me\n'); + }); + + it('encrypts medium sensitivity entries and refuses symlink escapes', async () => { + const stackName = 'secure'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, '.env'), 'SECRET=1\n', 'utf8'); + + const outside = path.join(composeDir, 'outside-secret.txt'); + await fsPromises.writeFile(outside, 'escaped\n', 'utf8'); + const linkPath = path.join(stackDir, 'escape.link'); + let symlinkOk = true; + try { + await fsPromises.symlink(outside, linkPath); + } catch { + symlinkOk = false; + } + + if (symlinkOk) { + await expect( + RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId: randomUUID(), + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + { + relativePath: 'escape.link', + absolutePath: linkPath, + kind: 'other', + sensitivity: 'low', + }, + ]), + }), + ).rejects.toMatchObject({ code: 'SYMLINK_ESCAPE' }); + } + + const generationId = randomUUID(); + const manifest = await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + { + relativePath: '.env', + absolutePath: path.join(stackDir, '.env'), + kind: 'interpolation-env', + sensitivity: 'medium', + }, + ]), + }); + const envEntry = manifest.entries.find((e) => e.relativePath === '.env'); + expect(envEntry?.encrypted).toBe(true); + expect(envEntry?.state).toBe('present'); + }); + it('reverts an interrupted multi-file restore from pre-restore snapshot', async () => { + const stackName = 'tx'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(path.join(stackDir, 'includes'), { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'OLD_BASE\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'includes/base.yaml'), 'OLD_INC\n', 'utf8'); + + const generationId = randomUUID(); + const inv = inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + { relativePath: 'includes/base.yaml', absolutePath: path.join(stackDir, 'includes/base.yaml'), kind: 'include' }, + ]); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inv, + operationKind: 'manual_backup', + }); + + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'NEW_BASE\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'includes/base.yaml'), 'NEW_INC\n', 'utf8'); + + const { FileSystemService } = await import('../services/FileSystemService'); + const originalWrite = FileSystemService.prototype.writeStackFile; + let n = 0; + vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockImplementation(async function (this: InstanceType, stack, rel, content) { + n += 1; + if (n === 2) throw new Error('injected write failure'); + return originalWrite.call(this, stack, rel, content); + }); + + await expect( + RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, ['compose.yaml', 'includes/base.yaml']), + ).rejects.toThrow(/injected write failure/); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe('NEW_BASE\n'); + expect(await fsPromises.readFile(path.join(stackDir, 'includes/base.yaml'), 'utf8')).toBe('NEW_INC\n'); + + const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + await expect(fsPromises.access(path.join(genDir, 'restore-intent.json'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('restores a snapshotted Git manifesto on compensate path helper', async () => { + const stackName = 'gitman'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + + const managedDir = path.join(dataDir, 'git-managed', '1', stackName); + await fsPromises.mkdir(managedDir, { recursive: true }); + const priorManifest = { + manifestVersion: 3, + state: 'active', + repo: { url: 'https://example.com/r.git', branch: 'main' }, + resolvedRevision: { commitSha: 'abc1234' }, + project: { + root: '.', + composeFiles: ['compose.yaml'], + projectName: stackName, + effectiveProjectDir: null, + invocation: ['-f', 'compose.yaml'], + }, + inputs: [], + buildContexts: [], + counts: { total: 0, refused: 0 }, + refusals: [], + generation: { appliedDir: 'generations/prior', previousDir: null }, + }; + await fsPromises.writeFile( + path.join(managedDir, 'manifest.v1.json'), + JSON.stringify(priorManifest), + 'utf8', + ); + + const generationId = randomUUID(); + const inv = inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + ]); + inv.git = { + repoUrl: 'https://example.com/r.git', + branch: 'main', + commitSha: 'abc1234', + manifestVersion: 3, + }; + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const readSpy = vi.spyOn(GitProjectManifestService.getInstance(), 'readManifest') + .mockResolvedValue(priorManifest as never); + let genDir = ''; + try { + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inv, + }); + + genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + const snap = await fsPromises.readFile(path.join(genDir, 'git-manifest.v1.json'), 'utf8'); + expect(JSON.parse(snap).resolvedRevision.commitSha).toBe('abc1234'); + } finally { + readSpy.mockRestore(); + } + const newer = { ...priorManifest, resolvedRevision: { commitSha: 'ffffff' } }; + await fsPromises.writeFile(path.join(managedDir, 'manifest.v1.json'), JSON.stringify(newer), 'utf8'); + + const captured = JSON.parse(await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8')); + await RollbackGenerationStore.restoreCapturedGitManifest(stackName, genDir, captured); + const restored = JSON.parse(await fsPromises.readFile(path.join(managedDir, 'manifest.v1.json'), 'utf8')); + expect(restored.resolvedRevision.commitSha).toBe('abc1234'); + }); + + it('does not snapshot a corrupt Git manifesto into the generation', async () => { + const stackName = 'nocorrupt'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + + const generationId = randomUUID(); + const inv = inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + ]); + inv.git = { + repoUrl: 'https://example.com/r.git', + branch: 'main', + commitSha: 'abc', + manifestVersion: 3, + }; + const { GitProjectManifestService } = await import('../services/GitProjectManifestService'); + const readSpy = vi.spyOn(GitProjectManifestService.getInstance(), 'readManifest') + .mockResolvedValue({ corrupt: 'identity mismatch' }); + try { + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inv, + }); + const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + await expect(fsPromises.access(path.join(genDir, 'git-manifest.v1.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + const captured = JSON.parse(await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8')); + expect(captured.priorRecords.gitManifestCaptured).toBe(false); + } finally { + readSpy.mockRestore(); + } + }); + + it('clears post-capture manifesto for first-apply preimage', async () => { + const stackName = 'firstapply'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + + const generationId = randomUUID(); + const inv = inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + ]); + inv.git = { + repoUrl: 'https://example.com/r.git', + branch: 'main', + commitSha: '', + manifestVersion: null, + }; + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inv, + }); + + const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + const captured = JSON.parse(await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8')); + expect(captured.priorRecords.gitManifestCaptured).toBe(false); + + const managedDir = path.join(dataDir, 'git-managed', '1', stackName); + await fsPromises.mkdir(managedDir, { recursive: true }); + await fsPromises.writeFile(path.join(managedDir, 'manifest.v1.json'), '{"manifestVersion":3}\n', 'utf8'); + + await RollbackGenerationStore.restoreCapturedGitManifest(stackName, genDir, captured); + await expect(fsPromises.access(path.join(managedDir, 'manifest.v1.json'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('does not clear live manifesto for legacy generations without a snapshot', async () => { + const stackName = 'legacygen'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + ]), + }); + + const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + const captured = JSON.parse(await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8')); + delete captured.priorRecords.gitManifestCaptured; + captured.git = { + repoUrl: 'https://example.com/r.git', + branch: 'main', + commitSha: 'abc', + manifestVersion: 3, + }; + await fsPromises.writeFile(path.join(genDir, 'generation.json'), JSON.stringify(captured), 'utf8'); + + const managedDir = path.join(dataDir, 'git-managed', '1', stackName); + await fsPromises.mkdir(managedDir, { recursive: true }); + await fsPromises.writeFile(path.join(managedDir, 'manifest.v1.json'), '{"keep":true}\n', 'utf8'); + + await RollbackGenerationStore.restoreCapturedGitManifest(stackName, genDir, captured); + expect(await fsPromises.readFile(path.join(managedDir, 'manifest.v1.json'), 'utf8')).toBe('{"keep":true}\n'); + }); + + it('refuses restore when a managed file path is currently a directory', async () => { + const stackName = 'dircollide'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'OLD\n', 'utf8'); + + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + ]), + }); + + await fsPromises.rm(path.join(stackDir, 'compose.yaml')); + await fsPromises.mkdir(path.join(stackDir, 'compose.yaml'), { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml', 'sentinel.txt'), 'keep-me\n', 'utf8'); + + await expect( + RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, ['compose.yaml']), + ).rejects.toMatchObject({ code: 'DIRECTORY_COLLISION' }); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml', 'sentinel.txt'), 'utf8')).toBe('keep-me\n'); + }); + + it('restores a legitimate file whose name ends with .sencho-tombstone (index-based snapshot)', async () => { + const stackName = 'tombname'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + const special = 'config.sencho-tombstone'; + await fsPromises.writeFile(path.join(stackDir, special), 'KEEP_ME\n', 'utf8'); + + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + { relativePath: special, absolutePath: path.join(stackDir, special), kind: 'other' }, + ]), + }); + + await fsPromises.writeFile(path.join(stackDir, special), 'CHANGED\n', 'utf8'); + await RollbackGenerationStore.restoreGeneration( + NODE, + stackName, + generationId, + ['compose.yaml', special], + ); + + expect(await fsPromises.readFile(path.join(stackDir, special), 'utf8')).toBe('KEEP_ME\n'); + }); + + it('captures POSIX mode and restores it on generation restore', async () => { + if (process.platform === 'win32') return; + + const stackName = 'modes'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + const scriptPath = path.join(stackDir, 'entrypoint.sh'); + await fsPromises.writeFile(scriptPath, '#!/bin/sh\necho hi\n', 'utf8'); + await fsPromises.chmod(scriptPath, 0o755); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + + const generationId = randomUUID(); + const manifest = await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + { relativePath: 'entrypoint.sh', absolutePath: scriptPath, kind: 'other' }, + ]), + }); + const entry = manifest.entries.find((e) => e.relativePath === 'entrypoint.sh'); + expect(entry?.mode).toBe(0o755); + + await fsPromises.chmod(scriptPath, 0o644); + await RollbackGenerationStore.restoreGeneration( + NODE, + stackName, + generationId, + ['compose.yaml', 'entrypoint.sh'], + ); + const st = await fsPromises.stat(scriptPath); + expect(st.mode & 0o777).toBe(0o755); + }); + + it('reverts an interrupted restore that deleted a post-capture file via index absent entries', async () => { + const stackName = 'absenttx'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'OLD\n', 'utf8'); + + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + ]), + }); + + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'NEW\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'extra.yml'), 'EXTRA\n', 'utf8'); + + const { FileSystemService } = await import('../services/FileSystemService'); + const originalWrite = FileSystemService.prototype.writeStackFile; + vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockImplementation(async function ( + this: InstanceType, + stack, + rel, + content, + ) { + if (rel === 'compose.yaml') throw new Error('injected write failure'); + return originalWrite.call(this, stack, rel, content); + }); + + await expect( + RollbackGenerationStore.restoreGeneration(NODE, stackName, generationId, ['compose.yaml', 'extra.yml']), + ).rejects.toThrow(/injected write failure/); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf8')).toBe('NEW\n'); + expect(await fsPromises.readFile(path.join(stackDir, 'extra.yml'), 'utf8')).toBe('EXTRA\n'); + }); + + it('encrypts sensitive pre-restore snapshots and restores them from ciphertext', async () => { + const stackName = 'presensitive'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, '.env'), 'CAPTURED=1\n', 'utf8'); + + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + { + relativePath: '.env', + absolutePath: path.join(stackDir, '.env'), + kind: 'interpolation-env', + sensitivity: 'medium', + }, + ]), + }); + + await fsPromises.writeFile(path.join(stackDir, '.env'), 'SUPERSECRET=live\n', 'utf8'); + await RollbackGenerationStore.restoreGeneration( + NODE, + stackName, + generationId, + ['compose.yaml', '.env'], + ); + + const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + const blobsDir = path.join(genDir, 'pre-restore', 'blobs'); + const blobNames = await fsPromises.readdir(blobsDir); + expect(blobNames.length).toBeGreaterThan(0); + for (const name of blobNames) { + const blob = await fsPromises.readFile(path.join(blobsDir, name)); + expect(blob.toString('utf8')).not.toContain('SUPERSECRET'); + expect(blob.toString('utf8')).not.toContain('CAPTURED=1'); + } + const index = JSON.parse( + await fsPromises.readFile(path.join(genDir, 'pre-restore', 'index.json'), 'utf8'), + ) as { entries: Array<{ relativePath: string; encrypted?: boolean; blobId?: string }> }; + const envEntry = index.entries.find((e) => e.relativePath === '.env'); + expect(envEntry?.encrypted).toBe(true); + expect(envEntry?.blobId).toBeTruthy(); + const envBlob = await fsPromises.readFile(path.join(blobsDir, envEntry!.blobId!), 'utf8'); + expect(envBlob.startsWith('enc:')).toBe(true); + expect(envBlob).not.toContain('SUPERSECRET'); + + await RollbackGenerationStore.reconcileInterruptedRestore(NODE, stackName, generationId); + expect(await fsPromises.readFile(path.join(stackDir, '.env'), 'utf8')).toBe('SUPERSECRET=live\n'); + }); + + it('persists runtime image identity on an existing generation', async () => { + const stackName = 'imgid'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + ]), + }); + + await RollbackGenerationStore.attachImages(NODE, stackName, generationId, [{ + serviceName: 'web', + imageId: 'sha256:abc', + repoDigest: 'nginx@sha256:digest', + platform: 'linux/amd64', + declaredImageRef: 'nginx:latest', + }]); + + const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + const manifest = JSON.parse( + await fsPromises.readFile(path.join(genDir, 'generation.json'), 'utf8'), + ) as { images: Array<{ serviceName: string; imageId: string; repoDigest: string; platform: string }> }; + expect(manifest.images).toEqual([{ + serviceName: 'web', + imageId: 'sha256:abc', + repoDigest: 'nginx@sha256:digest', + platform: 'linux/amd64', + declaredImageRef: 'nginx:latest', + }]); + }); + + it('does not recursively delete a directory created after an absent-file snapshot', async () => { + const stackName = 'dirrace'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'OLD\n', 'utf8'); + + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: NODE, + stackName, + generationId, + inventory: inventoryFor(stackName, [ + { relativePath: 'compose.yaml', absolutePath: path.join(stackDir, 'compose.yaml') }, + ]), + }); + + await RollbackGenerationStore.restoreGeneration( + NODE, + stackName, + generationId, + ['compose.yaml', 'scratch'], + ); + + const scratchDir = path.join(stackDir, 'scratch'); + await fsPromises.mkdir(scratchDir, { recursive: true }); + await fsPromises.writeFile(path.join(scratchDir, 'keep.txt'), 'unrelated\n', 'utf8'); + + await expect( + RollbackGenerationStore.reconcileInterruptedRestore(NODE, stackName, generationId), + ).rejects.toMatchObject({ code: 'DIRECTORY_COLLISION' }); + + expect(await fsPromises.readFile(path.join(scratchDir, 'keep.txt'), 'utf8')).toBe('unrelated\n'); + const genDir = RollbackGenerationStore.getGenerationDir(NODE, stackName, generationId); + await expect(fsPromises.access(path.join(genDir, 'restore-intent.json'))).resolves.toBeUndefined(); + }); +}); diff --git a/backend/src/__tests__/rollback-inventory.test.ts b/backend/src/__tests__/rollback-inventory.test.ts new file mode 100644 index 00000000..5ec2c5a5 --- /dev/null +++ b/backend/src/__tests__/rollback-inventory.test.ts @@ -0,0 +1,311 @@ +/** + * Git manifesto inventory: first-apply merge vs established fail-closed. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import path from 'path'; +import os from 'os'; +import { promises as fsPromises } from 'fs'; + +const mockGetGitSource = vi.fn(); +const mockReadManifest = vi.fn(); +const mockGetOverrideFilename = vi.fn().mockResolvedValue(null); +const mockGetStackProjectEnvFiles = vi.fn().mockReturnValue([]); +const mockBuildAuthoredComposeArgs = vi.fn().mockResolvedValue(['compose', '-f', 'compose.yaml', 'config', '--quiet']); +const mockAuthoredComposeFileArgs = vi.fn().mockReturnValue(['-f', 'compose.yaml']); +const mockAuthoredComposeEnvFileArgs = vi.fn().mockResolvedValue([]); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getGitSource: mockGetGitSource, + getStackProjectEnvFiles: mockGetStackProjectEnvFiles, + isMeshStackEnabled: () => false, + }), + }, +})); + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getDefaultNodeId: () => 1, + }), + }, +})); + +vi.mock('../services/GitProjectManifestService', () => ({ + GitProjectManifestService: { + getInstance: () => ({ + readManifest: mockReadManifest, + }), + }, +})); + +const mockState = { composeDir: '' }; + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBaseDir: () => mockState.composeDir, + getOverrideFilename: mockGetOverrideFilename, + }), + }, +})); + +vi.mock('../services/ComposeService', () => ({ + ComposeService: { + getInstance: () => ({ + buildAuthoredComposeArgs: mockBuildAuthoredComposeArgs, + }), + }, +})); + +vi.mock('../utils/authoredComposeArgs', () => ({ + authoredComposeFileArgs: (...args: unknown[]) => mockAuthoredComposeFileArgs(...args), + authoredComposeEnvFileArgs: (...args: unknown[]) => mockAuthoredComposeEnvFileArgs(...args), +})); + +import { resolveRollbackInventory } from '../services/rollbackInventory'; + +describe('resolveRollbackInventory', () => { + let composeDir: string; + + beforeEach(async () => { + vi.clearAllMocks(); + composeDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-inv-')); + mockState.composeDir = composeDir; + mockGetOverrideFilename.mockResolvedValue(null); + mockGetStackProjectEnvFiles.mockReturnValue([]); + mockBuildAuthoredComposeArgs.mockResolvedValue(['compose', '-f', 'compose.yaml', 'config', '--quiet']); + mockAuthoredComposeFileArgs.mockReturnValue(['-f', 'compose.yaml']); + mockAuthoredComposeEnvFileArgs.mockResolvedValue([]); + }); + + it('merges authored files with Git identity on first apply when manifesto is missing', async () => { + const stackName = 'gitapp'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile( + path.join(stackDir, 'compose.yaml'), + 'services:\n web:\n image: nginx\n', + 'utf8', + ); + + mockGetGitSource.mockReturnValue({ + stack_name: stackName, + repo_url: 'https://example.com/repo.git', + branch: 'main', + last_applied_commit_sha: null, + last_applied_content_hash: null, + applied_deploy_spec: null, + sync_env: false, + manifest_version: null, + manifest_state: null, + manifest_generation: null, + }); + mockReadManifest.mockResolvedValue(null); + + const inventory = await resolveRollbackInventory(1, stackName); + + expect(inventory.exactCoverage).toBe(true); + expect(inventory.coverageRefusal).toBeNull(); + expect(inventory.entries.map((e) => e.relativePath)).toContain('compose.yaml'); + expect(inventory.git).toEqual({ + repoUrl: 'https://example.com/repo.git', + branch: 'main', + commitSha: '', + manifestVersion: null, + }); + }); + + it('fails closed when an established Git stack is missing its manifesto and cannot rebuild from applied spec', async () => { + const stackName = 'established'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + + mockGetGitSource.mockReturnValue({ + stack_name: stackName, + repo_url: 'https://example.com/repo.git', + branch: 'main', + last_applied_commit_sha: 'abc1234', + last_applied_content_hash: 'hash', + applied_deploy_spec: null, + sync_env: false, + manifest_version: 3, + manifest_state: 'active', + manifest_generation: 'generations/prior', + }); + mockReadManifest.mockResolvedValue(null); + + const inventory = await resolveRollbackInventory(1, stackName); + + expect(inventory.exactCoverage).toBe(false); + expect(inventory.coverageRefusal).toMatch(/missing/i); + expect(inventory.git?.commitSha).toBe('abc1234'); + }); + + it('fails closed for established missing manifesto instead of applied_deploy_spec exact coverage', async () => { + const stackName = 'rebuild'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n a: {}\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'compose.prod.yaml'), 'services:\n b: {}\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'app.env'), 'FOO=1\n', 'utf8'); + + mockGetGitSource.mockReturnValue({ + stack_name: stackName, + repo_url: 'https://example.com/repo.git', + branch: 'main', + last_applied_commit_sha: 'deadbeef', + last_applied_content_hash: 'h1', + applied_deploy_spec: { files: ['compose.yaml', 'compose.prod.yaml'], contextDir: null }, + sync_env: false, + manifest_version: 3, + manifest_state: 'active', + manifest_generation: 'generations/x', + }); + mockReadManifest.mockResolvedValue(null); + + const inventory = await resolveRollbackInventory(1, stackName); + + expect(inventory.exactCoverage).toBe(false); + expect(inventory.coverageRefusal).toMatch(/missing/i); + expect(inventory.git?.commitSha).toBe('deadbeef'); + }); + + it('fails closed when the manifesto is corrupt even if applied_deploy_spec files exist', async () => { + const stackName = 'corruptgit'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'app.env'), 'PASS=x\n', 'utf8'); + + mockGetGitSource.mockReturnValue({ + stack_name: stackName, + repo_url: 'https://example.com/repo.git', + branch: 'main', + last_applied_commit_sha: 'abc', + last_applied_content_hash: 'h', + applied_deploy_spec: { files: ['compose.yaml'], contextDir: null }, + sync_env: false, + manifest_version: 3, + manifest_state: 'active', + manifest_generation: 'generations/x', + }); + mockReadManifest.mockResolvedValue({ corrupt: 'identity mismatch' }); + + const inventory = await resolveRollbackInventory(1, stackName); + + expect(inventory.exactCoverage).toBe(false); + expect(inventory.coverageRefusal).toMatch(/unreadable/i); + expect(inventory.git?.commitSha).toBe('abc'); + }); + + it('fails closed when the manifesto is corrupt and applied_deploy_spec cannot be rebuilt', async () => { + const stackName = 'corrupt-norebuild'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + + mockGetGitSource.mockReturnValue({ + stack_name: stackName, + repo_url: 'https://example.com/repo.git', + branch: 'main', + last_applied_commit_sha: 'abc', + last_applied_content_hash: 'h', + applied_deploy_spec: { files: ['compose.yaml', 'missing.override.yaml'], contextDir: null }, + sync_env: false, + manifest_version: 3, + manifest_state: 'active', + manifest_generation: 'generations/x', + }); + mockReadManifest.mockResolvedValue({ corrupt: 'identity mismatch' }); + + const inventory = await resolveRollbackInventory(1, stackName); + + expect(inventory.exactCoverage).toBe(false); + expect(inventory.coverageRefusal).toMatch(/unreadable/i); + }); + + it('fails closed for missing manifesto with include/env/config dependency files on disk', async () => { + const stackName = 'deps-matrix'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(path.join(stackDir, 'configs'), { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n a:\n env_file: [app.env]\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'compose.prod.yaml'), 'include:\n - path: compose.yaml\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'app.env'), 'A=1\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'configs/app.conf'), 'x=1\n', 'utf8'); + + mockGetGitSource.mockReturnValue({ + stack_name: stackName, + repo_url: 'https://example.com/repo.git', + branch: 'main', + last_applied_commit_sha: 'cafebabe', + last_applied_content_hash: 'h2', + applied_deploy_spec: { files: ['compose.yaml', 'compose.prod.yaml'], contextDir: null }, + sync_env: false, + manifest_version: 3, + manifest_state: 'active', + manifest_generation: 'generations/y', + }); + mockReadManifest.mockResolvedValue(null); + + const inventory = await resolveRollbackInventory(1, stackName); + + expect(inventory.exactCoverage).toBe(false); + expect(inventory.coverageRefusal).toMatch(/missing/i); + }); + + it('fails closed when the Git manifesto is corrupt even before first apply', async () => { + const stackName = 'corrupt-first'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + + mockGetGitSource.mockReturnValue({ + stack_name: stackName, + repo_url: 'https://example.com/repo.git', + branch: 'main', + last_applied_commit_sha: null, + last_applied_content_hash: null, + applied_deploy_spec: null, + sync_env: false, + manifest_version: null, + manifest_state: null, + manifest_generation: null, + }); + mockReadManifest.mockResolvedValue({ corrupt: 'identity mismatch' }); + + const inventory = await resolveRollbackInventory(1, stackName); + + expect(inventory.exactCoverage).toBe(false); + expect(inventory.coverageRefusal).toMatch(/unreadable/i); + }); + + it('refuses exact coverage when case-colliding managed paths exist', async () => { + const stackName = 'casefold'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'App.conf'), 'A\n', 'utf8'); + await fsPromises.writeFile(path.join(stackDir, 'app.conf'), 'B\n', 'utf8'); + + mockGetGitSource.mockReturnValue(undefined); + + // Force discovery to see both via include parse would be heavy; instead seed + // by making them both compose roots is impossible. Use override + compose: + mockGetOverrideFilename.mockResolvedValue('App.conf'); + // Also plant app.conf as project env so both enter the map. + mockGetStackProjectEnvFiles.mockReturnValue(['app.conf']); + + const inventory = await resolveRollbackInventory(1, stackName); + // On case-sensitive FS both files exist; inventory should refuse collision. + // On Windows case-folding FS the second write may overwrite the first. + if (process.platform === 'win32') { + expect(inventory).toBeTruthy(); + return; + } + expect(inventory.exactCoverage).toBe(false); + expect(inventory.coverageRefusal).toMatch(/Case-colliding/i); + }); +}); diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index 090ab55e..c1c294e3 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -28,6 +28,7 @@ const { mockRunCommand, mockDeployStack, mockBackupStackFiles, + mockCaptureCurrentBackup, mockEnforcePolicyPreDeploy, } = vi.hoisted(() => ({ mockGetDueScheduledTasks: vi.fn().mockReturnValue([]), @@ -76,6 +77,7 @@ const { mockRunCommand: vi.fn().mockResolvedValue(undefined), mockDeployStack: vi.fn().mockResolvedValue(undefined), mockBackupStackFiles: vi.fn().mockResolvedValue(undefined), + mockCaptureCurrentBackup: vi.fn().mockResolvedValue({ id: 'gen-1' }), mockEnforcePolicyPreDeploy: vi.fn(), })); @@ -165,6 +167,14 @@ vi.mock('../services/FileSystemService', () => ({ }, })); +vi.mock('../services/StackUpdateRecoveryService', () => ({ + StackUpdateRecoveryService: { + getInstance: () => ({ + captureCurrentBackup: mockCaptureCurrentBackup, + }), + }, +})); + vi.mock('../services/ImageUpdateService', () => ({ ImageUpdateService: { // Default on so existing executeUpdate tests keep prior behavior. @@ -1907,10 +1917,15 @@ describe('SchedulerService - lifecycle actions', () => { expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); }); - it('auto_backup calls backupStackFiles', async () => { + it('auto_backup captures a current recovery generation', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup')); await SchedulerService.getInstance().triggerTask(300); - expect(mockBackupStackFiles).toHaveBeenCalledWith('my-stack'); + expect(mockCaptureCurrentBackup).toHaveBeenCalledWith({ + nodeId: 1, + stackName: 'my-stack', + createdBy: 'system:scheduler', + }); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'success' })); }); @@ -1924,6 +1939,7 @@ describe('SchedulerService - lifecycle actions', () => { it('auto_backup records failure when node_id is missing', async () => { mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { node_id: null })); await SchedulerService.getInstance().triggerTask(300); + expect(mockCaptureCurrentBackup).not.toHaveBeenCalled(); expect(mockBackupStackFiles).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(1, expect.objectContaining({ status: 'failure' })); }); @@ -2015,6 +2031,7 @@ describe('SchedulerService - lifecycle remote proxy', () => { 'http://remote:1852/api/stacks/my-stack/backup', expect.objectContaining({ method: 'POST' }), ); + expect(mockCaptureCurrentBackup).not.toHaveBeenCalled(); expect(mockBackupStackFiles).not.toHaveBeenCalled(); }); @@ -2246,7 +2263,7 @@ describe('SchedulerService - delete_after_run', () => { }); it('does not delete task when run fails even if delete_after_run is 1', async () => { - mockBackupStackFiles.mockRejectedValueOnce(new Error('disk full')); + mockCaptureCurrentBackup.mockRejectedValueOnce(new Error('disk full')); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_backup', { delete_after_run: 1 })); await SchedulerService.getInstance().triggerTask(300); expect(mockDeleteScheduledTask).not.toHaveBeenCalled(); diff --git a/backend/src/__tests__/stack-backup-route.test.ts b/backend/src/__tests__/stack-backup-route.test.ts index 13409c95..ca4ada84 100644 --- a/backend/src/__tests__/stack-backup-route.test.ts +++ b/backend/src/__tests__/stack-backup-route.test.ts @@ -1,9 +1,9 @@ /** * Integration tests for POST /api/stacks/:stackName/backup, the on-demand - * stack-files backup trigger. Covers auth, role, the success path, the + * recovery-generation capture. Covers auth, role, the success path, the * missing-stack 404, name validation, and error propagation. The route exists * so a scheduled auto_backup can run on a remote node through the proxy path, - * and so an operator can take a snapshot on demand. The backup is available on + * and so an operator can capture a current generation on demand. Available on * every tier (no paid gate). */ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; @@ -11,9 +11,11 @@ import request from 'supertest'; import bcrypt from 'bcrypt'; import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; -const { mockBackupStackFiles, mockHasComposeFile } = vi.hoisted(() => ({ +const { mockBackupStackFiles, mockHasComposeFile, mockCaptureCurrentBackup, mockGetCurrent } = vi.hoisted(() => ({ mockBackupStackFiles: vi.fn(), mockHasComposeFile: vi.fn(), + mockCaptureCurrentBackup: vi.fn(), + mockGetCurrent: vi.fn(), })); vi.mock('../services/FileSystemService', () => ({ @@ -27,6 +29,15 @@ vi.mock('../services/FileSystemService', () => ({ }, })); +vi.mock('../services/StackUpdateRecoveryService', () => ({ + StackUpdateRecoveryService: { + getInstance: () => ({ + getCurrent: mockGetCurrent, + captureCurrentBackup: mockCaptureCurrentBackup, + }), + }, +})); + let tmpDir: string; let app: import('express').Express; let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; @@ -59,6 +70,8 @@ afterAll(() => { beforeEach(() => { mockBackupStackFiles.mockReset().mockResolvedValue(undefined); mockHasComposeFile.mockReset().mockResolvedValue(true); + mockCaptureCurrentBackup.mockReset().mockResolvedValue({ id: 'gen-1' }); + mockGetCurrent.mockReset().mockReturnValue(null); tierSpy.mockReturnValue('paid'); }); @@ -67,18 +80,25 @@ describe('POST /api/stacks/:stackName/backup', () => { const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie); expect(res.status).toBe(200); expect(res.body.success).toBe(true); - expect(mockBackupStackFiles).toHaveBeenCalledWith('web'); + expect(mockCaptureCurrentBackup).toHaveBeenCalledWith(expect.objectContaining({ + nodeId: expect.any(Number), + stackName: 'web', + createdBy: expect.any(String), + })); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); }); it('returns 401 without an auth cookie', async () => { const res = await request(app).post('/api/stacks/web/backup'); expect(res.status).toBe(401); + expect(mockCaptureCurrentBackup).not.toHaveBeenCalled(); expect(mockBackupStackFiles).not.toHaveBeenCalled(); }); it('returns 403 for a viewer (no deploy permission)', async () => { const res = await request(app).post('/api/stacks/web/backup').set('Cookie', viewerCookie); expect(res.status).toBe(403); + expect(mockCaptureCurrentBackup).not.toHaveBeenCalled(); expect(mockBackupStackFiles).not.toHaveBeenCalled(); }); @@ -87,32 +107,39 @@ describe('POST /api/stacks/:stackName/backup', () => { const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie); expect(res.status).toBe(200); expect(res.body.success).toBe(true); - expect(mockBackupStackFiles).toHaveBeenCalledWith('web'); + expect(mockCaptureCurrentBackup).toHaveBeenCalledWith(expect.objectContaining({ + nodeId: expect.any(Number), + stackName: 'web', + createdBy: expect.any(String), + })); + expect(mockBackupStackFiles).not.toHaveBeenCalled(); }); it('returns 404 when the stack does not exist', async () => { mockHasComposeFile.mockResolvedValue(false); const res = await request(app).post('/api/stacks/ghost/backup').set('Cookie', adminCookie); expect(res.status).toBe(404); + expect(mockCaptureCurrentBackup).not.toHaveBeenCalled(); expect(mockBackupStackFiles).not.toHaveBeenCalled(); }); it('returns 400 for an invalid stack name', async () => { const res = await request(app).post('/api/stacks/..bad../backup').set('Cookie', adminCookie); expect(res.status).toBe(400); + expect(mockCaptureCurrentBackup).not.toHaveBeenCalled(); expect(mockBackupStackFiles).not.toHaveBeenCalled(); }); it('returns 500 when the backup operation fails', async () => { - mockBackupStackFiles.mockRejectedValue(new Error('disk full')); + mockCaptureCurrentBackup.mockRejectedValue(new Error('disk full')); const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie); expect(res.status).toBe(500); expect(res.body.error).toContain('disk full'); }); it('returns 409 when the stack is busy with another operation', async () => { - // The backup shares the rollback slot, so it must not run while a deploy - // holds the stack-op lock for the same stack. + // Handoff mutates the current generation, so backup must not run while a + // deploy holds the stack-op lock for the same stack. const { StackOpLockService } = await import('../services/StackOpLockService'); const localNodeId = DatabaseService.getInstance().getNodes().find(n => n.type === 'local')!.id; StackOpLockService.getInstance().tryAcquire(localNodeId, 'web', 'deploy', 'someone'); @@ -120,6 +147,7 @@ describe('POST /api/stacks/:stackName/backup', () => { const res = await request(app).post('/api/stacks/web/backup').set('Cookie', adminCookie); expect(res.status).toBe(409); expect(res.body.code).toBe('stack_op_in_progress'); + expect(mockCaptureCurrentBackup).not.toHaveBeenCalled(); expect(mockBackupStackFiles).not.toHaveBeenCalled(); } finally { StackOpLockService.getInstance().release(localNodeId, 'web'); diff --git a/backend/src/__tests__/stack-op-lock-routes.test.ts b/backend/src/__tests__/stack-op-lock-routes.test.ts index 4d76ea95..5a6aa98a 100644 --- a/backend/src/__tests__/stack-op-lock-routes.test.ts +++ b/backend/src/__tests__/stack-op-lock-routes.test.ts @@ -98,7 +98,7 @@ afterAll(() => { }); beforeEach(async () => { - mockDeployStack.mockReset(); + mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null }); mockRunCommand.mockReset(); mockRunDown.mockReset(); mockUpdateStack.mockReset(); @@ -128,7 +128,7 @@ function deferred(): Deferred { describe('Stack lifecycle mutex', () => { it('returns 409 with stack_op_in_progress when a deploy is already running', async () => { - const gate = deferred(); + const gate = deferred<{ recoveryId: string | null }>(); mockDeployStack.mockImplementationOnce(() => gate.promise); const first = request(app) @@ -152,20 +152,20 @@ describe('Stack lifecycle mutex', () => { expect(second.body.error).toMatch(/already deploying/i); expect(typeof second.body.inProgress.startedAt).toBe('number'); - gate.resolve(); + gate.resolve({ recoveryId: null }); const firstRes = await first; expect(firstRes.status).toBe(200); }); it('releases the lock after a successful deploy so the next request acquires', async () => { - mockDeployStack.mockResolvedValueOnce(undefined); + mockDeployStack.mockResolvedValueOnce({ recoveryId: null }); const first = await request(app) .post('/api/stacks/web/deploy') .set('Cookie', authCookie) .send({ skip_scan: true }); expect(first.status).toBe(200); - mockDeployStack.mockResolvedValueOnce(undefined); + mockDeployStack.mockResolvedValueOnce({ recoveryId: null }); const second = await request(app) .post('/api/stacks/web/deploy') .set('Cookie', authCookie) @@ -181,7 +181,7 @@ describe('Stack lifecycle mutex', () => { .send({ skip_scan: true }); expect(first.status).toBe(500); - mockDeployStack.mockResolvedValueOnce(undefined); + mockDeployStack.mockResolvedValueOnce({ recoveryId: null }); const second = await request(app) .post('/api/stacks/web/deploy') .set('Cookie', authCookie) @@ -190,7 +190,7 @@ describe('Stack lifecycle mutex', () => { }); it('blocks restart while a deploy is in flight on the same stack', async () => { - const gate = deferred(); + const gate = deferred<{ recoveryId: string | null }>(); mockDeployStack.mockImplementationOnce(() => gate.promise); const deploy = request(app) @@ -207,12 +207,12 @@ describe('Stack lifecycle mutex', () => { expect(restart.body.code).toBe('stack_op_in_progress'); expect(restart.body.inProgress.action).toBe('deploy'); - gate.resolve(); + gate.resolve({ recoveryId: null }); await deploy; }); it('allows concurrent ops on different stacks', async () => { - const gate = deferred(); + const gate = deferred<{ recoveryId: string | null }>(); mockDeployStack.mockImplementation(() => gate.promise); const webDeploy = request(app) @@ -229,7 +229,7 @@ describe('Stack lifecycle mutex', () => { .then(r => r); await vi.waitFor(() => expect(mockDeployStack).toHaveBeenCalledTimes(2)); - gate.resolve(); + gate.resolve({ recoveryId: null }); const [webRes, apiRes] = await Promise.all([webDeploy, apiDeploy]); expect(webRes.status).toBe(200); expect(apiRes.status).toBe(200); diff --git a/backend/src/__tests__/stack-update-recovery-service.test.ts b/backend/src/__tests__/stack-update-recovery-service.test.ts index 385aa08c..18856c72 100644 --- a/backend/src/__tests__/stack-update-recovery-service.test.ts +++ b/backend/src/__tests__/stack-update-recovery-service.test.ts @@ -7,11 +7,15 @@ const mockTag = vi.fn().mockResolvedValue(undefined); const mockRemove = vi.fn().mockResolvedValue(undefined); const mockListContainers = vi.fn().mockResolvedValue([]); const mockInspectContainer = vi.fn(); -const mockGetContainer = vi.fn(() => ({ inspect: mockInspectContainer })); +const mockGetContainer = vi.fn((..._args: unknown[]) => ({ inspect: mockInspectContainer })); const mockGetImage = vi.fn((ref: string) => ({ tag: mockTag, remove: mockRemove, - inspect: vi.fn().mockResolvedValue({ RepoDigests: [] }), + inspect: vi.fn().mockResolvedValue({ + RepoDigests: ['nginx@sha256:digest'], + Os: 'linux', + Architecture: 'amd64', + }), _ref: ref, })); @@ -41,9 +45,51 @@ vi.mock('../services/composeProjectContext', () => ({ classifyReferenceKind: () => 'moving_tag', resolveComposeProjectContext: vi.fn().mockResolvedValue({ validateForMutation: vi.fn().mockResolvedValue(undefined), - backupFromContext: vi.fn().mockResolvedValue('backup-1'), + backupFromContext: vi.fn().mockResolvedValue('11111111-1111-4111-8111-111111111111'), restoreFromContext: vi.fn().mockResolvedValue(undefined), }), + resolveComposeProjectContextForGeneration: vi.fn().mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue('11111111-1111-4111-8111-111111111111'), + restoreFromContext: vi.fn().mockResolvedValue(undefined), + backupSlotId: '11111111-1111-4111-8111-111111111111', + }), +})); + +vi.mock('../services/RollbackGenerationStore', () => ({ + RollbackGenerationStore: { + retireGenerationContent: vi.fn().mockResolvedValue(undefined), + getGenerationDir: vi.fn(() => '/tmp/gen'), + verifyGenerationContent: vi.fn().mockResolvedValue(false), + commitRestoreTransaction: vi.fn().mockResolvedValue(undefined), + reconcileInterruptedRestore: vi.fn().mockResolvedValue(false), + restoreCapturedGitManifest: vi.fn().mockResolvedValue(undefined), + hasPendingRestoreIntent: vi.fn().mockResolvedValue(false), + attachImages: vi.fn().mockResolvedValue(undefined), + }, + getBackupBaseDir: () => '/tmp/backups', +})); + +vi.mock('../services/GitProjectManifestService', () => ({ + GitProjectManifestService: { + getInstance: () => ({ + readRawManifestText: vi.fn().mockResolvedValue(null), + writeManifest: vi.fn().mockResolvedValue(undefined), + clearManifestFile: vi.fn().mockResolvedValue(undefined), + }), + }, + MANIFEST_FILENAME: 'manifest.v1.json', +})); + + +vi.mock('../services/PolicyEnforcement', () => ({ + enforcePolicyForImageRefs: vi.fn().mockResolvedValue({ ok: true, bypassed: false, violations: [] }), + enforcePolicyPreDeploy: vi.fn().mockResolvedValue({ ok: true, bypassed: false, violations: [] }), +})); + +vi.mock('../services/rollbackEligibility', () => ({ + assessGenerationEligibility: vi.fn().mockResolvedValue('eligible'), + evaluateRollbackEligibility: vi.fn(), })); vi.mock('../services/effectiveServiceModel', () => ({ @@ -61,7 +107,7 @@ vi.mock('../services/ComposeService', async () => { ComposeService: { getInstance: () => ({ validateExactComposeInvocation: mockValidateExact, - buildAuthoredComposeArgs: vi.fn(), + buildAuthoredComposeArgs: vi.fn().mockResolvedValue(['compose', '-f', 'compose.yaml', 'config', '--quiet']), }), }, getComposeCommandTimeoutMs: () => 60_000, @@ -71,15 +117,27 @@ vi.mock('../services/ComposeService', async () => { const mockUnlink = vi.fn().mockResolvedValue(undefined); const mockWriteFile = vi.fn().mockResolvedValue(undefined); const mockRealpath = vi.fn(async (p: string) => p); +const mockAccess = vi.fn().mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })); +const mockReaddir = vi.fn().mockResolvedValue([]); +const mockStat = vi.fn(); +const mockRm = vi.fn().mockResolvedValue(undefined); vi.mock('fs/promises', () => ({ default: { unlink: (p: string) => mockUnlink(p), writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc), realpath: (p: string) => mockRealpath(p), + access: (p: string) => mockAccess(p), + readdir: (p: string) => mockReaddir(p), + stat: (p: string) => mockStat(p), + rm: (p: string, opts?: unknown) => mockRm(p, opts), }, unlink: (p: string) => mockUnlink(p), writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc), realpath: (p: string) => mockRealpath(p), + access: (p: string) => mockAccess(p), + readdir: (p: string) => mockReaddir(p), + stat: (p: string) => mockStat(p), + rm: (p: string, opts?: unknown) => mockRm(p, opts), })); import { DatabaseService } from '../services/DatabaseService'; @@ -89,6 +147,9 @@ describe('StackUpdateRecoveryService', () => { beforeEach(() => { vi.clearAllMocks(); StackUpdateRecoveryService.resetForTests(); + vi.spyOn(DatabaseService.prototype, 'getGitSource').mockReturnValue(undefined as never); + vi.spyOn(DatabaseService.prototype, 'listStackUpdateRecoveryGenerationsForNode') + .mockReturnValue([]); mockListContainers.mockResolvedValue([ { Id: 'c1', State: 'running', Labels: {} }, ]); @@ -102,6 +163,7 @@ describe('StackUpdateRecoveryService', () => { mockRealpath.mockImplementation(async (p: string) => p); mockRemove.mockResolvedValue(undefined); mockTag.mockResolvedValue(undefined); + mockGetContainer.mockImplementation(() => ({ inspect: mockInspectContainer })); }); it('validates exact invocation before tagging images', async () => { @@ -135,6 +197,8 @@ describe('StackUpdateRecoveryService', () => { phase: 'captured' as const, is_current: 0, backup_slot_id: null, + content_path: null, + operation_kind: null, override_path: '/test/compose/my-stack/.sencho-recovery-aaaaaaaaaaaa.yml', services_json: JSON.stringify([{ serviceName: 'web', @@ -185,6 +249,8 @@ describe('StackUpdateRecoveryService', () => { phase: 'reconciling' as const, is_current: 1, backup_slot_id: 'b1', + content_path: null, + operation_kind: null, override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml', services_json: servicesJson, health_gate_id: null, @@ -206,15 +272,15 @@ describe('StackUpdateRecoveryService', () => { mockInspectContainer.mockResolvedValue({ State: { ExitCode: 0 } }); vi.useFakeTimers(); - const promise = StackUpdateRecoveryService.getInstance().compensateWithCandidate( + const result = StackUpdateRecoveryService.getInstance().compensateWithCandidate( 'gen-2', async () => undefined, - ); + ).then(() => null, (error: unknown) => error); await vi.advanceTimersByTimeAsync(3100); - const ok = await promise; + const error = await result; vi.useRealTimers(); - expect(ok).toBe(false); + expect(error).toMatchObject({ code: 'RECOVERY_PROBE_FAILED' }); expect(update).toHaveBeenCalledWith('gen-2', expect.objectContaining({ status: 'recovery_required' })); expect(update).not.toHaveBeenCalledWith( 'gen-2', @@ -222,6 +288,117 @@ describe('StackUpdateRecoveryService', () => { ); }); + it('throws HELD_IMAGE_MISSING when compensation compose-up cannot find the hold tag', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'web', + scale: 1, + hasBuild: false, + declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', + replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/x/web:hold' }], + }]); + const row = { + id: 'gen-missing-hold', + node_id: 1, + stack_name: 'my-stack', + status: 'active' as const, + phase: 'reconciling' as const, + is_current: 1, + backup_slot_id: 'b1', + content_path: null, + operation_kind: null, + override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml', + services_json: servicesJson, + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + }; + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row); + const update = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration') + .mockImplementation(() => undefined); + + await expect( + StackUpdateRecoveryService.getInstance().compensateWithCandidate( + 'gen-missing-hold', + async () => { + throw new Error('Error response from daemon: No such image: sencho-rb/x/web:hold'); + }, + ), + ).rejects.toMatchObject({ code: 'HELD_IMAGE_MISSING', message: 'Held recovery image is missing' }); + expect(update).toHaveBeenCalledWith('gen-missing-hold', expect.objectContaining({ status: 'recovery_required' })); + }); + + it('fails closed when content_path is set but generation dir is missing', async () => { + const genId = '11111111-1111-4111-8111-111111111111'; + const restoreSpy = vi.fn().mockResolvedValue(undefined); + const { resolveComposeProjectContext, resolveComposeProjectContextForGeneration } = + await import('../services/composeProjectContext'); + vi.mocked(resolveComposeProjectContext).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(genId), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: null, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + }); + vi.mocked(resolveComposeProjectContextForGeneration).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(genId), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: genId, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + }); + + const row = { + id: genId, + node_id: 1, + stack_name: 'my-stack', + status: 'active' as const, + phase: 'reconciling' as const, + is_current: 1, + backup_slot_id: genId, + content_path: genId, + operation_kind: 'update' as const, + override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml', + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + }; + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row); + const update = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration') + .mockImplementation(() => undefined); + mockAccess.mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })); + + await expect( + StackUpdateRecoveryService.getInstance().compensateWithCandidate(genId, async () => undefined), + ).rejects.toMatchObject({ code: 'GENERATION_CONTENT_MISSING' }); + + expect(restoreSpy).not.toHaveBeenCalled(); + expect(update).toHaveBeenCalledWith(genId, expect.objectContaining({ status: 'recovery_required' })); + }); + const capturedWebReplica = { containerId: 'c1', imageId: 'sha256:oldimg', @@ -335,6 +512,87 @@ describe('StackUpdateRecoveryService', () => { vi.useRealTimers(); }); + it('probeRecoveredStack rejects extra running replicas beyond captured scale', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', replicas: [capturedWebReplica], + }]); + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + { Id: 'c2', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + ]); + mockInspectContainer.mockResolvedValue({ + Image: 'sha256:oldimg', + State: { Status: 'running' }, + }); + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(false); + vi.useRealTimers(); + }); + + it('probeRecoveredStack rejects unexpected running services', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', replicas: [capturedWebReplica], + }]); + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + { Id: 'c2', State: 'running', Labels: { 'com.docker.compose.service': 'sidecar' } }, + ]); + mockInspectContainer.mockResolvedValue({ + Image: 'sha256:oldimg', + State: { Status: 'running' }, + }); + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(false); + vi.useRealTimers(); + }); + + it('probeRecoveredStack rejects structurally empty service objects', async () => { + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', '[{}]'); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(false); + vi.useRealTimers(); + }); + + it('refuses capture when a service has mixed replica image IDs', async () => { + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + { Id: 'c2', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + ]); + mockGetContainer.mockImplementation((...args: unknown[]) => ({ + inspect: vi.fn().mockResolvedValue({ + Id: String(args[0] ?? 'c1'), + State: { Status: 'running', ExitCode: 0 }, + Image: args[0] === 'c1' ? 'sha256:aaa' : 'sha256:bbb', + }), + })); + vi.spyOn(DatabaseService.prototype, 'getGlobalSettings').mockReturnValue({}); + + await expect( + StackUpdateRecoveryService.getInstance().captureCandidate({ + nodeId: 1, + stackName: 'my-stack', + createdBy: 'test', + }), + ).rejects.toMatchObject({ code: 'MIXED_REPLICA_IMAGES' }); + // Refuse must happen before hold tagging so mixed identity cannot leave Docker residue. + expect(mockTag).not.toHaveBeenCalled(); + }); + + it('getHeldImageIds fails closed when held-image listing throws', () => { + vi.spyOn(DatabaseService.prototype, 'listHeldStackUpdateRecoveryImageIds') + .mockImplementation(() => { + throw new Error('Malformed stack recovery services_json while listing held images'); + }); + expect(StackUpdateRecoveryService.getInstance().getHeldImageIds(1)).toBeNull(); + }); + it('does not mark artifacts retired when tag removal fails', async () => { const row = { id: 'gen-3', @@ -344,6 +602,8 @@ describe('StackUpdateRecoveryService', () => { phase: 'captured' as const, is_current: 0, backup_slot_id: null, + content_path: null, + operation_kind: null, override_path: null, services_json: JSON.stringify([{ serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest', @@ -368,4 +628,763 @@ describe('StackUpdateRecoveryService', () => { expect(ok).toBe(false); expect(markRetired).not.toHaveBeenCalled(); }); + it('uses legacy restore for UUID backup_slot_id when content_path is null (B1)', async () => { + const legacyUuid = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'; + const { resolveComposeProjectContext, resolveComposeProjectContextForGeneration } = + await import('../services/composeProjectContext'); + const restoreSpy = vi.fn().mockResolvedValue(undefined); + vi.mocked(resolveComposeProjectContext).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(legacyUuid), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: null, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + } as never); + vi.mocked(resolveComposeProjectContextForGeneration).mockClear(); + + const row = { + id: 'gen-legacy', + node_id: 1, + stack_name: 'my-stack', + status: 'active' as const, + phase: 'reconciling' as const, + is_current: 1, + backup_slot_id: legacyUuid, + content_path: null, + operation_kind: 'update' as const, + override_path: '/test/compose/my-stack/.sencho-recovery-aaaaaaaaaaaa.yml', + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + }; + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row); + vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration').mockImplementation(() => undefined); + mockListContainers.mockResolvedValue([]); + + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().compensateWithCandidate( + 'gen-legacy', + async () => undefined, + ); + await vi.advanceTimersByTimeAsync(3100); + const ok = await promise; + vi.useRealTimers(); + + expect(ok).toBe(true); + expect(resolveComposeProjectContextForGeneration).not.toHaveBeenCalled(); + expect(resolveComposeProjectContext).toHaveBeenCalled(); + expect(restoreSpy).toHaveBeenCalled(); + }); + + it('refuses compensation when held recovery images fail policy (B4)', async () => { + const { enforcePolicyForImageRefs } = await import('../services/PolicyEnforcement'); + vi.mocked(enforcePolicyForImageRefs).mockResolvedValueOnce({ + ok: false, + bypassed: false, + violations: [{ imageRef: 'sencho-rb/aaaaaaaaaaaa/web:hold', reasons: ['critical'] }] as never, + policy: { name: 'block-crit' } as never, + }); + + const genId = '11111111-1111-4111-8111-111111111111'; + const restoreSpy = vi.fn().mockResolvedValue({ + priorRecords: { appliedDeploySpec: null, lkgHint: null }, + invocation: { composeArgsPrefix: [], projectDirectory: null, projectName: 'my-stack', explicitComposeFiles: [] }, + }); + const { resolveComposeProjectContextForGeneration, resolveComposeProjectContext } = + await import('../services/composeProjectContext'); + vi.mocked(resolveComposeProjectContextForGeneration).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(genId), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: genId, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + } as never); + vi.mocked(resolveComposeProjectContext).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(genId), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: null, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + } as never); + + const row = { + id: genId, + node_id: 1, + stack_name: 'my-stack', + status: 'active' as const, + phase: 'reconciling' as const, + is_current: 1, + backup_slot_id: genId, + content_path: genId, + operation_kind: 'update' as const, + override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml', + services_json: JSON.stringify([{ + serviceName: 'web', + scale: 1, + hasBuild: false, + declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', + replicas: [{ + containerId: 'c1', + imageId: 'sha256:held', + repoDigest: null, + state: 'running', + rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold', + }], + }]), + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + }; + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row); + const update = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration') + .mockImplementation(() => undefined); + mockAccess.mockResolvedValue(undefined); + + await expect( + StackUpdateRecoveryService.getInstance().compensateWithCandidate(genId, async () => undefined), + ).rejects.toMatchObject({ code: 'ROLLBACK_PROHIBITED' }); + expect(restoreSpy).toHaveBeenCalled(); + expect(update).not.toHaveBeenCalledWith(genId, expect.objectContaining({ status: 'restored_current' })); + expect(enforcePolicyForImageRefs).toHaveBeenCalledWith( + 'my-stack', + 1, + ['sencho-rb/aaaaaaaaaaaa/web:hold'], + expect.objectContaining({ actor: 'recovery-compensate' }), + ); + }); + + it('refuses compensation when services_json is malformed', async () => { + const genId = '55555555-5555-4555-8555-555555555555'; + const restoreSpy = vi.fn().mockResolvedValue({ + priorRecords: { appliedDeploySpec: null, lkgHint: null }, + invocation: { composeArgsPrefix: [], projectDirectory: null, projectName: 'my-stack', explicitComposeFiles: [] }, + }); + const { resolveComposeProjectContextForGeneration } = await import('../services/composeProjectContext'); + vi.mocked(resolveComposeProjectContextForGeneration).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(genId), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: genId, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + } as never); + + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue({ + id: genId, + node_id: 1, + stack_name: 'my-stack', + status: 'active', + phase: 'reconciling', + is_current: 1, + backup_slot_id: genId, + content_path: genId, + operation_kind: 'update', + override_path: '/test/compose/my-stack/.sencho-recovery-cccccccccccc.yml', + services_json: '{not-json', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + } as never); + mockAccess.mockResolvedValue(undefined); + + await expect( + StackUpdateRecoveryService.getInstance().compensateWithCandidate(genId, async () => undefined), + ).rejects.toMatchObject({ code: 'ROLLBACK_PROHIBITED' }); + expect(restoreSpy).not.toHaveBeenCalled(); + }); + + it('refuses compensation when services_json is structurally empty objects', async () => { + const genId = '66666666-6666-4666-8666-666666666666'; + const restoreSpy = vi.fn().mockResolvedValue({ + priorRecords: { appliedDeploySpec: null, lkgHint: null }, + invocation: { composeArgsPrefix: [], projectDirectory: null, projectName: 'my-stack', explicitComposeFiles: [] }, + }); + const { resolveComposeProjectContextForGeneration } = await import('../services/composeProjectContext'); + vi.mocked(resolveComposeProjectContextForGeneration).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(genId), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: genId, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + } as never); + + const { assessGenerationEligibility } = await import('../services/rollbackEligibility'); + vi.mocked(assessGenerationEligibility).mockResolvedValueOnce('eligible'); + + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue({ + id: genId, + node_id: 1, + stack_name: 'my-stack', + status: 'active', + phase: 'reconciling', + is_current: 1, + backup_slot_id: genId, + content_path: genId, + operation_kind: 'update', + override_path: '/test/compose/my-stack/.sencho-recovery-dddddddddddd.yml', + services_json: '[{}]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + } as never); + mockAccess.mockResolvedValue(undefined); + + await expect( + StackUpdateRecoveryService.getInstance().compensateWithCandidate(genId, async () => undefined), + ).rejects.toMatchObject({ code: 'ROLLBACK_PROHIBITED' }); + expect(restoreSpy).not.toHaveBeenCalled(); + }); + + + it('restores captured Git appliedDeploySpec on compensate (B2)', async () => { + const genId = '22222222-2222-4222-8222-222222222222'; + const priorSpec = { files: ['compose.yaml', 'docker-compose.override.yaml'], contextDir: 'app' }; + const restoreSpy = vi.fn().mockResolvedValue({ + priorRecords: { + appliedDeploySpec: JSON.stringify(priorSpec), + lkgHint: null, + lastAppliedContentHash: 'hash-prior', + manifestState: 'active', + manifestGeneration: 'generations/prior', + }, + git: { + repoUrl: 'https://example.com/r.git', + branch: 'main', + commitSha: 'abc1234deadbeef', + manifestVersion: 3, + }, + }); + const { resolveComposeProjectContextForGeneration } = await import('../services/composeProjectContext'); + vi.mocked(resolveComposeProjectContextForGeneration).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(genId), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: genId, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + } as never); + + vi.spyOn(DatabaseService.prototype, 'getGitSource').mockReturnValue({ + stack_name: 'my-stack', + applied_deploy_spec: { files: ['compose.yaml'], contextDir: null }, + last_applied_commit_sha: 'newsha', + last_applied_content_hash: 'hash-new', + manifest_version: 3, + manifest_state: 'active', + manifest_generation: 'generations/new', + } as never); + const setSpec = vi.spyOn(DatabaseService.prototype, 'setGitSourceAppliedSpec').mockImplementation(() => undefined); + const markApplied = vi.spyOn(DatabaseService.prototype, 'markGitSourceApplied').mockImplementation(() => undefined); + const setManifest = vi.spyOn(DatabaseService.prototype, 'setGitSourceManifestState').mockImplementation(() => undefined); + + const row = { + id: genId, + node_id: 1, + stack_name: 'my-stack', + status: 'active' as const, + phase: 'reconciling' as const, + is_current: 1, + backup_slot_id: genId, + content_path: genId, + operation_kind: 'git_apply' as const, + override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml', + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + }; + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row); + vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration').mockImplementation(() => undefined); + mockAccess.mockResolvedValue(undefined); + mockListContainers.mockResolvedValue([]); + + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().compensateWithCandidate(genId, async () => undefined); + await vi.advanceTimersByTimeAsync(3100); + const ok = await promise; + vi.useRealTimers(); + + expect(ok).toBe(true); + expect(setSpec).toHaveBeenCalledWith('my-stack', priorSpec); + expect(markApplied).toHaveBeenCalledWith('my-stack', 'abc1234deadbeef', 'hash-prior'); + expect(setManifest).toHaveBeenCalledWith('my-stack', 3, 'active', 'generations/prior'); + }); + + + it('clears null appliedDeploySpec on compensate', async () => { + const genId = '33333333-3333-4333-8333-333333333333'; + const restoreSpy = vi.fn().mockResolvedValue({ + priorRecords: { + appliedDeploySpec: null, + lkgHint: null, + lastAppliedContentHash: null, + manifestState: null, + manifestGeneration: null, + }, + git: null, + }); + const { resolveComposeProjectContextForGeneration } = await import('../services/composeProjectContext'); + vi.mocked(resolveComposeProjectContextForGeneration).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(genId), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: genId, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + } as never); + + vi.spyOn(DatabaseService.prototype, 'getGitSource').mockReturnValue({ + stack_name: 'my-stack', + applied_deploy_spec: { files: ['compose.yaml'], contextDir: null }, + last_applied_commit_sha: 'newsha', + last_applied_content_hash: 'hash-new', + manifest_version: null, + manifest_state: null, + manifest_generation: null, + } as never); + const setSpec = vi.spyOn(DatabaseService.prototype, 'setGitSourceAppliedSpec').mockImplementation(() => undefined); + + const row = { + id: genId, + node_id: 1, + stack_name: 'my-stack', + status: 'active' as const, + phase: 'reconciling' as const, + is_current: 1, + backup_slot_id: genId, + content_path: genId, + operation_kind: 'git_apply' as const, + override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml', + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + }; + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row); + vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration').mockImplementation(() => undefined); + mockAccess.mockResolvedValue(undefined); + mockListContainers.mockResolvedValue([]); + + const { RollbackGenerationStore } = await import('../services/RollbackGenerationStore'); + vi.mocked(RollbackGenerationStore.commitRestoreTransaction).mockClear(); + + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().compensateWithCandidate(genId, async () => undefined); + await vi.advanceTimersByTimeAsync(3100); + const ok = await promise; + vi.useRealTimers(); + + expect(ok).toBe(true); + expect(setSpec).toHaveBeenCalledWith('my-stack', null); + expect(RollbackGenerationStore.commitRestoreTransaction).toHaveBeenCalled(); + }); + + it('does not commit Git state or the restore transaction when probe fails after generation restore', async () => { + const genId = '44444444-4444-4444-8444-444444444444'; + const restoreSpy = vi.fn().mockResolvedValue({ + priorRecords: { appliedDeploySpec: null, lkgHint: null, lastAppliedContentHash: null, manifestState: null, manifestGeneration: null }, + git: null, + }); + const { resolveComposeProjectContextForGeneration } = await import('../services/composeProjectContext'); + vi.mocked(resolveComposeProjectContextForGeneration).mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue(genId), + restoreFromContext: restoreSpy, + nodeId: 1, + stackName: 'my-stack', + stackDir: '/test/compose/my-stack', + backupSlotId: genId, + toComposeArgs: vi.fn(), + resolveServiceImageMap: vi.fn(), + } as never); + + const servicesJson = JSON.stringify([{ + serviceName: 'web', + scale: 1, + hasBuild: false, + declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', + replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/x/web:hold' }], + }]); + const row = { + id: genId, + node_id: 1, + stack_name: 'my-stack', + status: 'active' as const, + phase: 'reconciling' as const, + is_current: 1, + backup_slot_id: genId, + content_path: genId, + operation_kind: 'update' as const, + override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml', + services_json: servicesJson, + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + }; + vi.spyOn(DatabaseService.prototype, 'getGitSource').mockReturnValue({ + stack_name: 'my-stack', + applied_deploy_spec: { files: ['compose.yaml'], contextDir: null }, + last_applied_commit_sha: 'newsha', + last_applied_content_hash: 'hash-new', + manifest_version: 3, + manifest_state: 'active', + manifest_generation: 'generations/new', + } as never); + const setSpec = vi.spyOn(DatabaseService.prototype, 'setGitSourceAppliedSpec').mockImplementation(() => undefined); + const updateGen = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration') + .mockImplementation(() => undefined); + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row); + mockAccess.mockResolvedValue(undefined); + mockListContainers.mockResolvedValue([{ Id: 'c1', State: 'exited', Labels: { 'com.docker.compose.service': 'web' } }]); + mockInspectContainer.mockResolvedValue({ State: { ExitCode: 1 } }); + + const { RollbackGenerationStore } = await import('../services/RollbackGenerationStore'); + vi.mocked(RollbackGenerationStore.commitRestoreTransaction).mockClear(); + vi.mocked(RollbackGenerationStore.restoreCapturedGitManifest).mockClear(); + vi.mocked(RollbackGenerationStore.reconcileInterruptedRestore).mockClear(); + + vi.useFakeTimers(); + const result = StackUpdateRecoveryService.getInstance().compensateWithCandidate(genId, async () => undefined) + .then(() => null, (error: unknown) => error); + await vi.advanceTimersByTimeAsync(3100); + const error = await result; + vi.useRealTimers(); + + expect(error).toMatchObject({ code: 'RECOVERY_PROBE_FAILED' }); + expect(restoreSpy).toHaveBeenCalled(); + expect(setSpec).not.toHaveBeenCalled(); + expect(RollbackGenerationStore.commitRestoreTransaction).not.toHaveBeenCalled(); + expect(RollbackGenerationStore.restoreCapturedGitManifest).not.toHaveBeenCalled(); + expect(RollbackGenerationStore.reconcileInterruptedRestore).toHaveBeenCalledWith(1, 'my-stack', genId); + expect(updateGen).toHaveBeenCalledWith(genId, expect.objectContaining({ status: 'recovery_required' })); + }); + + it('fails closed when interrupted restore reconcile leaves a pending intent', async () => { + const { RollbackGenerationStore } = await import('../services/RollbackGenerationStore'); + vi.mocked(RollbackGenerationStore.reconcileInterruptedRestore).mockResolvedValueOnce(false); + vi.mocked(RollbackGenerationStore.hasPendingRestoreIntent).mockResolvedValueOnce(true); + + vi.spyOn(DatabaseService.prototype, 'getNodes').mockReturnValue([ + { id: 1, name: 'local', type: 'local' } as never, + ]); + vi.spyOn(DatabaseService.prototype, 'listStackUpdateRecoveryGenerationsForNode').mockReturnValue([ + { + id: 'gen-intent', + node_id: 1, + stack_name: 'my-stack', + status: 'recovery_required', + phase: 'reconciling', + is_current: 1, + backup_slot_id: 'gen-intent', + content_path: 'gen-intent', + operation_kind: 'update', + override_path: null, + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + }, + ] as never); + vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration') + .mockImplementation(() => undefined); + + await expect( + StackUpdateRecoveryService.getInstance().reconcileInterruptedRestoresAtStartup(), + ).rejects.toThrow(/Unresolved interrupted restore intent/); + }); + + it('blocks capture while a restore intent remains pending', async () => { + const { RollbackGenerationStore } = await import('../services/RollbackGenerationStore'); + vi.mocked(RollbackGenerationStore.hasPendingRestoreIntent).mockResolvedValueOnce(true); + vi.spyOn(DatabaseService.prototype, 'listStackUpdateRecoveryGenerationsForNode').mockReturnValue([ + { + id: 'gen-block', + node_id: 1, + stack_name: 'my-stack', + content_path: 'gen-block', + }, + ] as never); + + await expect( + StackUpdateRecoveryService.getInstance().captureCandidate({ + nodeId: 1, + stackName: 'my-stack', + createdBy: null, + operationKind: 'update', + }), + ).rejects.toThrow(/interrupted restore/); + }); + + it('excludes Compose one-off containers from replica capture and mixed-image checks', async () => { + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + { + Id: 'c-run', + State: 'running', + Labels: { 'com.docker.compose.service': 'web', 'com.docker.compose.oneoff': 'True' }, + }, + { + Id: 'c-stop', + State: 'exited', + Labels: { 'com.docker.compose.service': 'web', 'com.docker.compose.oneoff': 'True' }, + }, + ]); + mockGetContainer.mockImplementation((...args: unknown[]) => ({ + inspect: vi.fn().mockResolvedValue({ + Id: String(args[0] ?? 'c1'), + State: { Status: args[0] === 'c-stop' ? 'exited' : 'running', ExitCode: 0 }, + Image: args[0] === 'c1' ? 'sha256:abc' : 'sha256:oneoff', + }), + })); + const spyInsert = vi.spyOn(DatabaseService.prototype, 'insertStackUpdateRecoveryGeneration') + .mockImplementation(() => undefined); + vi.spyOn(DatabaseService.prototype, 'getGlobalSettings').mockReturnValue({}); + + await StackUpdateRecoveryService.getInstance().captureCandidate({ + nodeId: 1, + stackName: 'my-stack', + createdBy: 'test', + }); + + const { RollbackGenerationStore } = await import('../services/RollbackGenerationStore'); + expect(RollbackGenerationStore.attachImages).toHaveBeenCalledWith( + 1, + 'my-stack', + '11111111-1111-4111-8111-111111111111', + [ + expect.objectContaining({ + serviceName: 'web', + imageId: 'sha256:abc', + repoDigest: 'nginx@sha256:digest', + platform: 'linux/amd64', + declaredImageRef: 'nginx:latest', + }), + ], + ); + const inserted = spyInsert.mock.calls[0]?.[0] as { services_json: string }; + const services = JSON.parse(inserted.services_json) as Array<{ + scale: number; + replicas: Array<{ containerId: string }>; + }>; + expect(services[0].scale).toBe(1); + expect(services[0].replicas.map((r) => r.containerId)).toEqual(['c1']); + }); + + it('probeRecoveredStack ignores running and stopped one-off containers', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'web', + scale: 1, + hasBuild: false, + declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', + replicas: [{ + containerId: 'c1', + imageId: 'sha256:abc', + repoDigest: null, + state: 'running', + rollbackTag: 'sencho-rb/x/web:hold', + }], + }]); + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + { + Id: 'c-run', + State: 'running', + Labels: { 'com.docker.compose.service': 'web', 'com.docker.compose.oneoff': 'True' }, + }, + { + Id: 'c-stop', + State: 'exited', + Labels: { 'com.docker.compose.service': 'web', 'com.docker.compose.oneoff': 'True' }, + }, + ]); + mockGetContainer.mockImplementation((...args: unknown[]) => ({ + inspect: vi.fn().mockResolvedValue({ + Image: args[0] === 'c1' ? 'sha256:abc' : 'sha256:oneoff', + State: { Status: args[0] === 'c-stop' ? 'exited' : 'running' }, + }), + })); + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(true); + vi.useRealTimers(); + }); + + it('captureCurrentBackup hands off a candidate to immediate_verified', async () => { + const svc = StackUpdateRecoveryService.getInstance(); + const genId = '55555555-5555-4555-8555-555555555555'; + const row = { id: genId, node_id: 1, stack_name: 'my-stack' }; + vi.spyOn(svc, 'getCurrent').mockReturnValue(undefined); + vi.spyOn(svc, 'captureCandidate').mockResolvedValue(row as never); + vi.spyOn(svc, 'markAcquired').mockReturnValue(true); + vi.spyOn(svc, 'handoff').mockReturnValue(true); + vi.spyOn(svc, 'markReconciling').mockReturnValue(true); + vi.spyOn(svc, 'markImmediateVerified').mockReturnValue(true); + vi.spyOn(svc, 'get').mockReturnValue({ + ...row, + status: 'active', + phase: 'immediate_verified', + is_current: 1, + } as never); + + const result = await svc.captureCurrentBackup({ + nodeId: 1, + stackName: 'my-stack', + createdBy: 'admin', + }); + + expect(svc.captureCandidate).toHaveBeenCalledWith({ + nodeId: 1, + stackName: 'my-stack', + createdBy: 'admin', + operationKind: 'manual_backup', + }); + expect(svc.handoff).toHaveBeenCalledWith(genId, 1, 'my-stack'); + expect(result.phase).toBe('immediate_verified'); + }); + + it('captureCurrentBackup abandons the candidate when handoff fails', async () => { + const svc = StackUpdateRecoveryService.getInstance(); + const genId = '66666666-6666-4666-8666-666666666666'; + const row = { id: genId, node_id: 1, stack_name: 'my-stack' }; + vi.spyOn(svc, 'getCurrent').mockReturnValue(undefined); + vi.spyOn(svc, 'captureCandidate').mockResolvedValue(row as never); + vi.spyOn(svc, 'markAcquired').mockReturnValue(true); + vi.spyOn(svc, 'handoff').mockReturnValue(false); + const abandon = vi.spyOn(svc, 'abandon').mockResolvedValue(true); + + await expect( + svc.captureCurrentBackup({ nodeId: 1, stackName: 'my-stack', createdBy: 'admin' }), + ).rejects.toThrow(/hand off/); + expect(abandon).toHaveBeenCalledWith(genId); + }); + + it('captureCurrentBackup does not abandon after a successful handoff', async () => { + const svc = StackUpdateRecoveryService.getInstance(); + const genId = '77777777-7777-4777-8777-777777777777'; + const row = { id: genId, node_id: 1, stack_name: 'my-stack' }; + vi.spyOn(svc, 'getCurrent').mockReturnValue(undefined); + vi.spyOn(svc, 'captureCandidate').mockResolvedValue(row as never); + vi.spyOn(svc, 'markAcquired').mockReturnValue(true); + vi.spyOn(svc, 'handoff').mockReturnValue(true); + vi.spyOn(svc, 'markReconciling').mockReturnValue(true); + vi.spyOn(svc, 'markImmediateVerified').mockReturnValue(false); + vi.spyOn(svc, 'get').mockReturnValue({ + ...row, + status: 'active', + phase: 'reconciling', + is_current: 1, + } as never); + const abandon = vi.spyOn(svc, 'abandon').mockResolvedValue(true); + + const result = await svc.captureCurrentBackup({ + nodeId: 1, + stackName: 'my-stack', + createdBy: 'admin', + }); + expect(abandon).not.toHaveBeenCalled(); + expect(result.is_current).toBe(1); + }); + + it('captureCurrentBackup refuses to replace a generation while a health gate is observing', async () => { + const svc = StackUpdateRecoveryService.getInstance(); + vi.spyOn(svc, 'getCurrent').mockReturnValue({ + id: 'cur', + node_id: 1, + stack_name: 'my-stack', + health_gate_id: 'gate-1', + } as never); + vi.spyOn(DatabaseService.prototype, 'getHealthGateRun').mockReturnValue({ status: 'observing' } as never); + const capture = vi.spyOn(svc, 'captureCandidate'); + + await expect( + svc.captureCurrentBackup({ nodeId: 1, stackName: 'my-stack', createdBy: 'admin' }), + ).rejects.toMatchObject({ code: 'HEALTH_GATE_OBSERVING' }); + expect(capture).not.toHaveBeenCalled(); + }); }); diff --git a/backend/src/__tests__/stackRouteAuth.test.ts b/backend/src/__tests__/stackRouteAuth.test.ts index 4c115647..2fc5418e 100644 --- a/backend/src/__tests__/stackRouteAuth.test.ts +++ b/backend/src/__tests__/stackRouteAuth.test.ts @@ -39,6 +39,9 @@ describe('classifyStackApiPath', () => { expect(classifyStackApiPath('DELETE', '/stacks/web/git-source')).toEqual({ kind: 'named-stack', stackName: 'web', action: 'stack:edit', }); + expect(classifyStackApiPath('POST', '/stacks/web/fleet-snapshot-apply')).toEqual({ + kind: 'named-stack', stackName: 'web', action: 'stack:edit', + }); }); it('maps deploy routes and service lifecycle ops to stack:deploy', () => { diff --git a/backend/src/__tests__/stacks-failure-notifications.test.ts b/backend/src/__tests__/stacks-failure-notifications.test.ts index 588e311b..52fb5755 100644 --- a/backend/src/__tests__/stacks-failure-notifications.test.ts +++ b/backend/src/__tests__/stacks-failure-notifications.test.ts @@ -13,6 +13,7 @@ import jwt from 'jsonwebtoken'; import { setupTestDb, cleanupTestDb, loginAsTestAdmin, TEST_JWT_SECRET } from './helpers/setupTestDb'; import { ComposeRollbackError } from '../services/ComposeService'; import * as policyGate from '../helpers/policyGate'; +import type { StackUpdateRecoveryGenerationRow } from '../services/DatabaseService'; // ── Hoisted mocks (must come before importing the app) ────────────────────── @@ -141,10 +142,10 @@ afterAll(() => { }); beforeEach(() => { - mockDeployStack.mockReset(); + mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null }); mockRunCommand.mockReset(); mockRunDown.mockReset(); - mockUpdateStack.mockReset(); + mockUpdateStack.mockReset().mockResolvedValue({ recoveryId: null }); mockGetContainersByStack.mockReset(); mockRestartContainer.mockReset(); mockStopContainer.mockReset(); @@ -233,7 +234,7 @@ describe('deploy_failure notification on /deploy error', () => { }); it('uses trusted proxy tier headers for remote atomic deploys', async () => { - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); const res = await request(app) @@ -259,7 +260,7 @@ describe('health gate begin call sites', () => { }); it('begins a gate after a manual deploy and returns its id', async () => { - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); const res = await request(app) .post('/api/stacks/myapp/deploy') .set('Cookie', authCookie) @@ -269,6 +270,19 @@ describe('health gate begin call sites', () => { expect(res.body.healthGateId).toBe('gate-123'); }); + it('links the deploy recovery generation to the observing gate', async () => { + mockDeployStack.mockResolvedValue({ recoveryId: 'rec-deploy' }); + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const linkSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'linkGateOrRetain'); + const res = await request(app) + .post('/api/stacks/myapp/deploy') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + expect(res.status).toBe(200); + expect(linkSpy).toHaveBeenCalledWith('rec-deploy', 'gate-123'); + linkSpy.mockRestore(); + }); + it('begins a gate after a manual update and returns its id', async () => { mockUpdateStack.mockResolvedValue({ recoveryId: null }); const res = await request(app) @@ -304,7 +318,7 @@ describe('health gate begin call sites', () => { }); it('never begins a gate for the rollback recovery path', async () => { - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); const res = await request(app) .post('/api/stacks/myapp/rollback') .set('Cookie', authCookie); @@ -370,13 +384,39 @@ describe('failure classification on deploy/update error responses', () => { .set('Cookie', authCookie); expect(res.status).toBe(500); - expect(res.body.failure.reason).toBe('unknown'); + expect(res.body.failure).toMatchObject({ reason: 'unknown' }); + }); + + it('classifies mixed replica image capture refusals', async () => { + mockDeployStack.mockRejectedValue( + new Error('Service "web" has mixed replica images; refusing recovery capture that cannot restore exact prior identity'), + ); + + const res = await request(app) + .post('/api/stacks/myapp/deploy') + .set('Cookie', authCookie); + + expect(res.status).toBe(500); + expect(res.body.failure).toMatchObject({ reason: 'mixed_replica_images' }); + }); + + it('classifies rollback coverage refusals', async () => { + mockDeployStack.mockRejectedValue( + new Error('Host-absolute include path cannot be captured for exact rollback'), + ); + + const res = await request(app) + .post('/api/stacks/myapp/deploy') + .set('Cookie', authCookie); + + expect(res.status).toBe(500); + expect(res.body.failure).toMatchObject({ reason: 'rollback_coverage_unavailable' }); }); }); describe('post-deploy scan opt-out', () => { it('does not trigger a post-deploy scan when skip_scan is true', async () => { - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); const res = await request(app) .post('/api/stacks/myapp/deploy') @@ -503,6 +543,76 @@ describe('deploy_failure notification on /update error', () => { }); }); +describe('generation rollback error mapping', () => { + function stubCurrentGeneration(id: string): StackUpdateRecoveryGenerationRow { + return { + id, + node_id: 1, + stack_name: 'myapp', + status: 'active', + phase: 'immediate_verified', + is_current: 1, + backup_slot_id: id, + content_path: null, + operation_kind: null, + override_path: '/tmp/override.yml', + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: 0, + updated_at: 0, + created_by: null, + artifacts_retired: 0, + released_at: null, + released_by: null, + }; + } + + async function withGenerationRollback(compensate: () => Promise) { + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const svc = StackUpdateRecoveryService.getInstance(); + const getSpy = vi.spyOn(svc, 'getCurrent').mockReturnValue(stubCurrentGeneration('gen-1')); + const compensateSpy = vi.spyOn(svc, 'compensateWithCandidate').mockImplementation(compensate); + try { + return await request(app).post('/api/stacks/myapp/rollback').set('Cookie', authCookie); + } finally { + getSpy.mockRestore(); + compensateSpy.mockRestore(); + } + } + + it('returns HELD_IMAGE_MISSING when the hold tag is gone', async () => { + const res = await withGenerationRollback(async () => { + throw Object.assign(new Error('Held recovery image is missing'), { code: 'HELD_IMAGE_MISSING' }); + }); + expect(res.status).toBe(500); + expect(res.body).toMatchObject({ + code: 'HELD_IMAGE_MISSING', + error: 'Held recovery image is missing', + }); + }); + + it('returns RECOVERY_PROBE_FAILED when restore completed but the probe failed', async () => { + const res = await withGenerationRollback(async () => { + throw Object.assign(new Error('Recovery health probe failed'), { code: 'RECOVERY_PROBE_FAILED' }); + }); + expect(res.status).toBe(500); + expect(res.body).toMatchObject({ + code: 'RECOVERY_PROBE_FAILED', + error: 'Rollback restore completed but recovery probe failed.', + }); + }); + + it('returns a generic restore failure when compensation returns false', async () => { + const res = await withGenerationRollback(async () => false); + expect(res.status).toBe(500); + expect(res.body).toMatchObject({ error: 'Rollback restore did not complete.' }); + expect(res.body.code).toBeUndefined(); + }); +}); + describe('rollback file-revert safety on a policy-blocked rollback', () => { it('does not deploy and alerts the operator when the post-block file revert fails', async () => { // The restored backup is blocked by policy after files were already restored. diff --git a/backend/src/__tests__/stacks-from-git-skip-scan.test.ts b/backend/src/__tests__/stacks-from-git-skip-scan.test.ts index e1845cfa..fa4f622e 100644 --- a/backend/src/__tests__/stacks-from-git-skip-scan.test.ts +++ b/backend/src/__tests__/stacks-from-git-skip-scan.test.ts @@ -123,7 +123,7 @@ beforeEach(() => { envWritten: false, warnings: [], }); - mockDeployStack.mockResolvedValue(undefined); + mockDeployStack.mockResolvedValue({ recoveryId: null }); mockIsTrivyAvailable.mockReturnValue(true); mockListContainers.mockResolvedValue([{ Image: 'nginx:latest' }]); mockGetImageDigest.mockResolvedValue(null); diff --git a/backend/src/__tests__/update-guard-rollback.test.ts b/backend/src/__tests__/update-guard-rollback.test.ts index cb6a9db7..63b9bf0a 100644 --- a/backend/src/__tests__/update-guard-rollback.test.ts +++ b/backend/src/__tests__/update-guard-rollback.test.ts @@ -36,6 +36,9 @@ const baseInputs = (over: Partial = {}): RollbackInputs => ({ name: 'app-web-1', state: 'running', health: 'healthy', exitCode: null, hasHealthcheck: true, restartPolicy: 'unless-stopped', mounts: ['volume app_data'], }], + recoveryGeneration: { exists: false }, + policyEligibility: null, + managedInputs: { covered: true, detail: 'Exact authored-project coverage includes 1 managed path(s).' }, ...over, }); @@ -43,19 +46,20 @@ const itemById = (inputs: RollbackInputs, id: string) => buildRollbackItems(inputs, NOW).find(i => i.id === id)!; describe('buildRollbackItems', () => { - it('reports a full set of six items', () => { + it('reports the full set of rollback readiness items', () => { const items = buildRollbackItems(baseInputs(), NOW); expect(items.map(i => i.id)).toEqual([ - 'compose_source', 'env_keys', 'previous_images', 'last_deploy', 'healthchecks', 'volume_data', + 'recovery_generation', 'compose_source', 'env_keys', 'previous_images', 'last_deploy', 'healthchecks', + 'policy_eligibility', 'managed_inputs', 'volume_data', ]); }); it('marks the volume row not_covered unconditionally and names the mounts', () => { const item = itemById(baseInputs(), 'volume_data'); expect(item.state).toBe('not_covered'); - expect(item.detail).toContain('not included in file backups'); + expect(item.detail).toContain('not included in recovery generations'); expect(item.detail).toContain('volume app_data'); - + expect(item.detail).toMatch(/managed authored project/i); const noMounts = itemById(baseInputs({ containers: [] }), 'volume_data'); expect(noMounts.state).toBe('not_covered'); @@ -122,6 +126,28 @@ describe('buildRollbackItems', () => { }); }); + + it('marks policy eligibility blocked and warning states', () => { + expect(itemById(baseInputs({ + recoveryGeneration: { exists: true, shortId: 'abc' }, + policyEligibility: 'prohibited', + }), 'policy_eligibility').state).toBe('blocked'); + expect(itemById(baseInputs({ + recoveryGeneration: { exists: true, shortId: 'abc' }, + policyEligibility: 'eligible_with_warning', + }), 'policy_eligibility').state).toBe('warning'); + }); + + it('lets recovery_generation supersede the legacy backup slot for compose_source', () => { + const items = buildRollbackItems(baseInputs({ + backup: { exists: false, timestamp: null }, + recoveryGeneration: { exists: true, shortId: 'deadbeefcaf0' }, + policyEligibility: 'eligible', + }), NOW); + expect(items.find(i => i.id === 'compose_source')!.state).toBe('ready'); + expect(items.find(i => i.id === 'recovery_generation')!.state).toBe('ready'); + }); + describe('aggregateRollbackOverall', () => { it('is ready when compose, env, and previous image are all covered', () => { expect(aggregateRollbackOverall(buildRollbackItems(baseInputs(), NOW))).toBe('ready'); @@ -157,6 +183,27 @@ describe('aggregateRollbackOverall', () => { const items = buildRollbackItems(baseInputs({ containers: 'error' }), NOW); expect(aggregateRollbackOverall(items)).toBe('ready'); }); + it('is not_ready when policy eligibility is blocked', () => { + const items = buildRollbackItems(baseInputs({ + recoveryGeneration: { exists: true, shortId: 'abc' }, + policyEligibility: 'prohibited', + }), NOW); + expect(aggregateRollbackOverall(items)).toBe('not_ready'); + }); + + it('is at best partial when policy eligibility is warning or unknown', () => { + const warn = buildRollbackItems(baseInputs({ + recoveryGeneration: { exists: true, shortId: 'abc' }, + policyEligibility: 'eligible_with_warning', + }), NOW); + expect(aggregateRollbackOverall(warn)).toBe('partial'); + const unk = buildRollbackItems(baseInputs({ + recoveryGeneration: { exists: true, shortId: 'abc' }, + policyEligibility: 'unknown', + }), NOW); + expect(aggregateRollbackOverall(unk)).toBe('partial'); + }); + }); describe('FileSystemService.getBackupEnvSummary', () => { diff --git a/backend/src/__tests__/update-guard-service.test.ts b/backend/src/__tests__/update-guard-service.test.ts index cdeaf946..008e2312 100644 --- a/backend/src/__tests__/update-guard-service.test.ts +++ b/backend/src/__tests__/update-guard-service.test.ts @@ -74,6 +74,8 @@ vi.mock('../services/DatabaseService', () => ({ // Rollback-readiness partial-revert disclosure reads the git source row; // no git-managed stacks in these fixtures by default. getGitSource: mockGetGitSource, + // Generation supersede path; no current recovery generation in these fixtures. + getCurrentStackUpdateRecovery: () => undefined, }), }, })); diff --git a/backend/src/__tests__/webhooks-trigger.test.ts b/backend/src/__tests__/webhooks-trigger.test.ts index 982f96fe..267decb0 100644 --- a/backend/src/__tests__/webhooks-trigger.test.ts +++ b/backend/src/__tests__/webhooks-trigger.test.ts @@ -461,12 +461,15 @@ describe('WebhookService.execute: health gate begin call sites', () => { const { HealthGateService } = await import('../services/HealthGateService'); vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined); vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]); - vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue(undefined); + vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: 'rec-hook' }); const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook'); + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const linkSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'linkGateOrRetain'); const result = await WebhookService.getInstance().execute(webhook, 'deploy', 'test', true); expect(result.success).toBe(true); expect(beginSpy).toHaveBeenCalledWith(nodeId, stack, 'deploy', 'system:webhook'); + expect(linkSpy).toHaveBeenCalledWith('rec-hook', 'gate-hook'); }); it('begins an update gate after a webhook pull succeeds', async () => { diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index 37fd2c97..2361eab2 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -145,6 +145,15 @@ export async function startServer(server: Server): Promise { } catch (err) { console.error('[Startup] Deployed stack deletion reconcile failed:', (err as Error).message); } + // Interrupted rollback restores must finish before mutation-capable services + // or HTTP accept traffic. Fail closed: rethrow so unresolved intents never + // leave mutators or HTTP accepting writes. + try { + await StackUpdateRecoveryService.getInstance().reconcileInterruptedRestoresAtStartup(); + } catch (err) { + console.error('[Startup] Stack update recovery restore reconcile failed:', (err as Error).message); + throw err; + } StackUpdateRecoveryService.getInstance().start(); // Synchronous starts: schedule background timers and continue. None of diff --git a/backend/src/helpers/applyFleetSnapshotFiles.ts b/backend/src/helpers/applyFleetSnapshotFiles.ts new file mode 100644 index 00000000..a5a41af9 --- /dev/null +++ b/backend/src/helpers/applyFleetSnapshotFiles.ts @@ -0,0 +1,123 @@ +import path from 'path'; +import { FileSystemService } from '../services/FileSystemService'; +import { StackOpLockService } from '../services/StackOpLockService'; +import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService'; +import { isValidStackName } from '../utils/validation'; +import { invalidateNodeCaches } from './cacheInvalidation'; +import { + FLEET_SNAPSHOT_APPLY_FILENAMES, + type FleetSnapshotApplyFilename, +} from '../utils/snapshot-capture'; + +export interface FleetSnapshotApplyFile { + filename: FleetSnapshotApplyFilename; + content: string; +} + +/** Keep compose.yaml and .env; ignore any other snapshot filenames. */ +export function selectFleetSnapshotApplyFiles( + files: Array<{ filename: string; content: string }>, +): FleetSnapshotApplyFile[] { + return files.filter((file): file is FleetSnapshotApplyFile => + FLEET_SNAPSHOT_APPLY_FILENAMES.some((name) => name === file.filename), + ); +} + +export interface ApplyFleetSnapshotFilesInput { + nodeId: number; + stackName: string; + files: FleetSnapshotApplyFile[]; + actor: string; +} + +export interface ApplyFleetSnapshotFilesResult { + capturedGenerationId: string | null; +} + +type CodedError = Error & { code: string }; + +export function getCodedError(error: unknown): CodedError | undefined { + if (!(error instanceof Error) || !('code' in error) || typeof error.code !== 'string') { + return undefined; + } + return error as CodedError; +} + +export type FleetSnapshotApplyConflictCode = 'stack_op_in_progress' | 'HEALTH_GATE_OBSERVING'; + +export function fleetSnapshotApplyConflictCode(error: unknown): FleetSnapshotApplyConflictCode | undefined { + const code = getCodedError(error)?.code; + if (code === 'stack_op_in_progress' || code === 'HEALTH_GATE_OBSERVING') return code; + return undefined; +} + +function codedError(message: string, code: string): CodedError { + return Object.assign(new Error(message), { code }); +} + +async function writeApplyFile( + fsSvc: FileSystemService, + stackName: string, + file: FleetSnapshotApplyFile, +): Promise { + switch (file.filename) { + case 'compose.yaml': + await fsSvc.saveStackContent(stackName, file.content); + return; + case '.env': + await fsSvc.saveEnvContent(stackName, file.content); + return; + } +} + +/** + * Capture-then-write used by Fleet snapshot restore on the node that owns the + * stack. Takes the stack-op lock, then captures and writes. An existing Compose + * project creates the current recovery generation before the first snapshot + * file is written. Capture failure aborts with the live files unchanged. A + * directory with no compose file is treated as a new stack and has no recovery + * generation to roll back to. + */ +export async function applyFleetSnapshotFiles( + input: ApplyFleetSnapshotFilesInput, +): Promise { + const { nodeId, stackName, files, actor } = input; + if (!isValidStackName(stackName)) { + throw codedError('Invalid stack name', 'INVALID_STACK_NAME'); + } + if (files.length === 0) { + throw codedError('No restoreable snapshot files', 'INVALID_SNAPSHOT_FILES'); + } + + const lock = await StackOpLockService.getInstance().runExclusive( + nodeId, + stackName, + 'backup', + actor, + async () => { + const fsSvc = FileSystemService.getInstance(nodeId); + const exists = await fsSvc.hasComposeFile(path.join(fsSvc.getBaseDir(), stackName)); + let capturedGenerationId: string | null = null; + if (exists) { + capturedGenerationId = (await StackUpdateRecoveryService.getInstance().captureCurrentBackup({ + nodeId, + stackName, + createdBy: actor, + })).id; + } + for (const file of files) { + await writeApplyFile(fsSvc, stackName, file); + } + invalidateNodeCaches(nodeId); + return { capturedGenerationId }; + }, + ); + + if (!lock.ran) { + throw codedError( + `Cannot restore "${stackName}": another operation (${lock.existing.action}) is already in progress.`, + 'stack_op_in_progress', + ); + } + return lock.result; +} diff --git a/backend/src/helpers/manifestFilePaths.ts b/backend/src/helpers/manifestFilePaths.ts new file mode 100644 index 00000000..17576ed5 --- /dev/null +++ b/backend/src/helpers/manifestFilePaths.ts @@ -0,0 +1,25 @@ +/** + * Shared enumeration of stack-relative files owned by a managed-project + * manifest. Used by Git promotion and by authored-project rollback capture so + * the two cannot drift on which paths belong to a generation. + */ +import type { GitProjectManifest } from '../types/gitProjectManifest'; + +/** Exact file paths owned by one manifest, excluding directory-only inventory entries. */ +export function collectManifestFilePaths( + manifest: Pick, +): string[] { + const paths = new Map(); + for (const entry of manifest.inputs) { + if (entry.ownership !== 'managed' || entry.state !== 'present' || entry.materializedPath === null) continue; + if (entry.dependencyKind === 'build-context' || entry.dependencyKind === 'build-additional-context') continue; + paths.set(entry.materializedPath.toLowerCase(), entry.materializedPath); + } + for (const context of manifest.buildContexts) { + for (const file of context.files) { + const rel = context.repoPath ? `${context.repoPath}/${file.path}` : file.path; + paths.set(rel.toLowerCase(), rel); + } + } + return [...paths.values()].sort((a, b) => a.localeCompare(b)); +} diff --git a/backend/src/helpers/stackRouteAuth.ts b/backend/src/helpers/stackRouteAuth.ts index 8d567a59..75440791 100644 --- a/backend/src/helpers/stackRouteAuth.ts +++ b/backend/src/helpers/stackRouteAuth.ts @@ -80,6 +80,7 @@ const EXACT_SUFFIX_RULES: readonly SuffixRule[] = [ { method: 'PUT', suffix: '/files/permissions', action: 'stack:edit' }, { method: 'PUT', suffix: '/labels', action: 'stack:edit' }, { method: 'PUT', suffix: '/git-source', action: 'stack:edit' }, + { method: 'POST', suffix: '/fleet-snapshot-apply', action: 'stack:edit' }, { method: 'DELETE', suffix: '/git-source', action: 'stack:edit' }, { method: 'POST', suffix: '/git-source/pull', action: 'stack:edit' }, { method: 'POST', suffix: '/git-source/apply', action: 'stack:edit' }, diff --git a/backend/src/index.ts b/backend/src/index.ts index 55f3b418..871e0318 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -187,7 +187,10 @@ app.use(errorHandler); installShutdownHandlers(server); if (require.main === module) { - void startServer(server); + void startServer(server).catch((err) => { + console.error('[Startup] Fatal startup failure:', (err as Error).message); + process.exit(1); + }); } // Exports used by tests (supertest requires the http.Server instance). diff --git a/backend/src/middleware/jsonParser.ts b/backend/src/middleware/jsonParser.ts index 7154cffb..a2262373 100644 --- a/backend/src/middleware/jsonParser.ts +++ b/backend/src/middleware/jsonParser.ts @@ -2,26 +2,31 @@ import express, { type Request, type Response, type NextFunction, type RequestHa import { NodeRegistry } from '../services/NodeRegistry'; import { isProxyExemptPath } from '../helpers/proxyExemptPaths'; import { SYNC_BODY_LIMIT, SYNC_ERROR_CODES, SYNC_PATH_PREFIX } from '../services/fleetSyncConstants'; +import { FLEET_SNAPSHOT_APPLY_BODY_LIMIT } from '../utils/snapshot-capture'; -// JSON body parser that also captures the raw bytes for HMAC verification. // `rawBody` is part of the Express.Request augmentation (see types/express.ts); // the cast is required because body-parser's `verify` signature types `req` as // Node's IncomingMessage, not Express's Request. -const jsonParser = express.json({ - verify: (req, _res, buf) => { - (req as unknown as Request).rawBody = buf; - }, -}); +function jsonWithRawBody(limit?: string | number): RequestHandler { + return express.json({ + ...(limit === undefined ? {} : { limit }), + verify: (req, _res, buf) => { + (req as unknown as Request).rawBody = buf; + }, + }); +} + +const jsonParser = jsonWithRawBody(); // Larger-limit parser for the fleet sync receive endpoint. A control instance // can push up to MAX_SYNC_ROWS rows in a single payload; the default 100 KB // limit is too tight for that. -const fleetSyncJsonParser = express.json({ - limit: SYNC_BODY_LIMIT, - verify: (req, _res, buf) => { - (req as unknown as Request).rawBody = buf; - }, -}); +const fleetSyncJsonParser = jsonWithRawBody(SYNC_BODY_LIMIT); + +// Combined compose.yaml + .env restore payload; sized for two snapshot files. +const fleetSnapshotApplyJsonParser = jsonWithRawBody(FLEET_SNAPSHOT_APPLY_BODY_LIMIT); + +const FLEET_SNAPSHOT_APPLY_PATH = /^\/api\/stacks\/[^/]+\/fleet-snapshot-apply$/; /** * Parse JSON on local requests but preserve the raw stream for remote proxy @@ -62,5 +67,9 @@ export const conditionalJsonParser: RequestHandler = (req: Request, res: Respons }); return; } + if (FLEET_SNAPSHOT_APPLY_PATH.test(req.path)) { + fleetSnapshotApplyJsonParser(req, res, next); + return; + } jsonParser(req, res, next); }; diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts index f36e0f98..2bd91b95 100644 --- a/backend/src/routes/fleet.ts +++ b/backend/src/routes/fleet.ts @@ -24,7 +24,7 @@ import { ImageOperationService } from '../services/ImageOperationService'; import { classifyImageChannel } from '../helpers/imageChannel'; import { runPolicyGate, assertPolicyGateAllows, buildPolicyGateOptions } from '../helpers/policyGate'; import { remoteSupportsCrossNodeRbac } from '../helpers/remoteCapabilities'; -import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, pickDossierFields, dossierHasContent, type SnapshotNodeData, type SnapshotDocumentation } from '../utils/snapshot-capture'; +import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, pickDossierFields, dossierHasContent, FLEET_SNAPSHOT_APPLY_TIMEOUT_MS, type SnapshotNodeData, type SnapshotDocumentation } from '../utils/snapshot-capture'; import { getLatestVersion, getLatestRelease } from '../utils/version-check'; import { isValidStackName } from '../utils/validation'; import { isDebugEnabled } from '../utils/debug'; @@ -45,6 +45,11 @@ import { POLICY_SEVERITIES } from '../utils/severity'; import { isNoOpBlockingPolicy } from '../utils/policy-risk'; import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog'; import { formatNoTargetError } from '../utils/remoteTarget'; +import { + applyFleetSnapshotFiles, + fleetSnapshotApplyConflictCode, + selectFleetSnapshotApplyFiles, +} from '../helpers/applyFleetSnapshotFiles'; import { CloudBackupService } from '../services/CloudBackupService'; import { NotificationService } from '../services/NotificationService'; import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation'; @@ -2721,12 +2726,21 @@ interface RemoteProxyContext { // of a generic string. The body is truncated to keep the recorded message bounded. async function remoteStackError(action: string, res: Awaited>): Promise { let detail = ''; + let code: string | undefined; try { - detail = (await res.text()).slice(0, 300).trim(); + const raw = (await res.text()).trim(); + detail = raw.slice(0, 300); + const parsed: unknown = JSON.parse(raw); + if (typeof parsed === 'object' && parsed !== null && 'code' in parsed && typeof parsed.code === 'string') { + code = parsed.code; + } } catch { - // Remote body unavailable; the status code alone still names the failure. + // Body missing or not JSON; status (and any truncated text) still name the failure. } - return new Error(`${action} on remote node (${res.status})${detail ? `: ${detail}` : ''}`); + return Object.assign( + new Error(`${action} on remote node (${res.status})${detail ? `: ${detail}` : ''}`), + { code, httpStatus: res.status }, + ); } // Builds the base URL + proxy headers for a remote node, or null when the node @@ -2745,54 +2759,37 @@ function buildRemoteProxyContext(node: Node): RemoteProxyContext | null { return { baseUrl: proxyTarget.apiUrl.replace(/\/$/, ''), headers }; } -// Writes a snapshot stack's files back to its node: local nodes write to disk -// (backing up any current files first), remote nodes receive them over the -// proxy. Throws SnapshotProxyTargetError when a remote node is unreachable, and -// an Error carrying the remote node's status and reason on a failed remote write. +// Writes a snapshot stack's files back to its node. Existing stacks capture a +// recovery generation before the first write; capture failure aborts with no +// mutation. A later write failure can leave live files changed; the captured +// generation remains. Throws SnapshotProxyTargetError when the remote has no +// proxy target. A non-OK apply response becomes an Error with the remote +// status and body (capture, lock, or write). async function applySnapshotStackFiles( node: Node, stackName: string, files: Array<{ filename: string; content: string }>, ): Promise { + const applyFiles = selectFleetSnapshotApplyFiles(files); if (node.type === 'local') { - const fsService = FileSystemService.getInstance(node.id); - try { - await fsService.backupStackFiles(stackName); - } catch (e) { - // Stack may not exist yet before first restore; that is ok. - console.warn(`[Fleet Snapshot] Pre-restore backup failed for stack "${stackName}" (may not exist yet):`, getErrorMessage(e, 'unknown')); - } - for (const file of files) { - if (file.filename === 'compose.yaml') { - await fsService.saveStackContent(stackName, file.content); - } else if (file.filename === '.env') { - await fsService.saveEnvContent(stackName, file.content); - } - } + await applyFleetSnapshotFiles({ + nodeId: node.id, + stackName, + files: applyFiles, + actor: 'system:fleet-snapshot', + }); return; } const ctx = buildRemoteProxyContext(node); if (!ctx) throw new SnapshotProxyTargetError(formatNoTargetError(node)); - for (const file of files) { - if (file.filename === 'compose.yaml') { - const putRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, { - method: 'PUT', - headers: ctx.headers, - body: JSON.stringify({ content: file.content }), - signal: AbortSignal.timeout(15000), - }); - if (!putRes.ok) throw await remoteStackError('Failed to restore compose file', putRes); - } else if (file.filename === '.env') { - const putRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, { - method: 'PUT', - headers: ctx.headers, - body: JSON.stringify({ content: file.content }), - signal: AbortSignal.timeout(15000), - }); - if (!putRes.ok) throw await remoteStackError('Failed to restore env file', putRes); - } - } + const applyRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/fleet-snapshot-apply`, { + method: 'POST', + headers: ctx.headers, + body: JSON.stringify({ files: applyFiles }), + signal: AbortSignal.timeout(FLEET_SNAPSHOT_APPLY_TIMEOUT_MS), + }); + if (!applyRes.ok) throw await remoteStackError('Failed to restore stack files', applyRes); } // Redeploys a stack after its files are restored. The deploy policy gate stays @@ -2958,6 +2955,11 @@ fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request, res.status(503).json({ error: error.message }); return; } + const conflict = fleetSnapshotApplyConflictCode(error); + if (conflict) { + res.status(409).json({ error: getErrorMessage(error, 'Restore conflict'), code: conflict }); + return; + } console.error('[Fleet Snapshot] Restore error:', error); res.status(500).json({ error: 'Failed to restore stack from snapshot' }); } diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 5ba889c8..0c0d0df9 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -66,6 +66,12 @@ import { UPDATE_VERIFICATION_INCOMPLETE_WARNING, } from '../services/ImageUpdateService'; import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection'; +import { + applyFleetSnapshotFiles, + fleetSnapshotApplyConflictCode, + getCodedError, + selectFleetSnapshotApplyFiles, +} from '../helpers/applyFleetSnapshotFiles'; import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/envFileResolution'; import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants'; import { getTerminalWs, DEPLOY_SESSION_HEADER } from '../websocket/generic'; @@ -118,6 +124,7 @@ const STACK_OP_PRESENT_PARTICIPLE: Record = { rollback: 'rolling back', backup: 'backing up', delete: 'deleting', + git_apply: 'applying Git changes', }; function linkStackUpdateRecoveryGate(recoveryId: string | null | undefined, healthGateId: string | null): void { @@ -1814,7 +1821,7 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { const debug = isDebugEnabled(); const atomic = true; if (debug) console.debug('[Stacks:debug] Deploy starting', { stackName, atomic, nodeId: req.nodeId }); - await ComposeService.getInstance(req.nodeId).deployStack( + const deployResult = await ComposeService.getInstance(req.nodeId).deployStack( stackName, getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), atomic, @@ -1825,6 +1832,7 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { if (debug) console.debug(`[Stacks:debug] Deploy finished in ${Date.now() - t0}ms`); ok = true; const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'deploy', req.user?.username ?? null); + linkStackUpdateRecoveryGate(deployResult.recoveryId, healthGateId); res.json({ message: 'Deployed successfully', healthGateId }); notifyActionSuccess('deploy_success', `${stackName} deployed`, stackName, req.user?.username ?? 'system'); if (!skipScan) { @@ -2475,6 +2483,65 @@ stacksRouter.post('/:stackName/rollback', async (req: Request, res: Response) => if (!tryAcquireStackOpLock(req, res, stackName, 'rollback')) return; let revertRestore: (() => Promise) | null = null; try { + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const recoverySvc = StackUpdateRecoveryService.getInstance(); + const currentGen = recoverySvc.getCurrent(req.nodeId, stackName); + // Any current recovery row uses compensateWithCandidate. Policy is evaluated + // against the restored target inside compensate, not the live pre-restore project. + if (currentGen) { + dlog(`[Stacks] Rollback initiated via recovery generation: ${sanitizeForLog(stackName)}`); + try { + const rolledBack = await recoverySvc.compensateWithCandidate( + currentGen.id, + async (overridePath, invocation) => { + await ComposeService.getInstance(req.nodeId).composeUpWithRecoveryOverride( + stackName, + overridePath, + getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), + invocation, + ); + }, + buildPolicyGateOptions(req, { actor: req.user?.username ?? 'system' }), + ); + if (!rolledBack) { + res.status(500).json({ error: 'Rollback restore did not complete.' }); + notifyActionFailure('rollback', stackName, new Error('rollback restore did not complete'), req.user?.username ?? 'system'); + return; + } + } catch (compError: unknown) { + const compCode = (compError as { code?: string }).code; + if (compCode === 'ROLLBACK_PROHIBITED') { + res.status(409).json({ + error: (compError as Error).message || 'Rollback is prohibited for this generation', + code: 'ROLLBACK_PROHIBITED', + }); + return; + } + if (compCode === 'HELD_IMAGE_MISSING') { + res.status(500).json({ + error: (compError as Error).message || 'Held recovery image is missing.', + code: 'HELD_IMAGE_MISSING', + }); + notifyActionFailure('rollback', stackName, compError, req.user?.username ?? 'system'); + return; + } + if (compCode === 'RECOVERY_PROBE_FAILED') { + res.status(500).json({ + error: 'Rollback restore completed but recovery probe failed.', + code: 'RECOVERY_PROBE_FAILED', + }); + notifyActionFailure('rollback', stackName, new Error('recovery probe failed'), req.user?.username ?? 'system'); + return; + } + throw compError; + } + invalidateNodeCaches(req.nodeId); + dlog(`[Stacks] Rollback completed: ${sanitizeForLog(stackName)}`); + res.json({ message: 'Stack rolled back from recovery generation.', recoveryId: currentGen.id }); + notifyActionSuccess('deploy_success', `${stackName} rolled back`, stackName, req.user?.username ?? 'system'); + return; + } + const fsSvc = FileSystemService.getInstance(req.nodeId); const backupInfo = await fsSvc.getBackupInfo(stackName); if (!backupInfo.exists) { @@ -2548,7 +2615,18 @@ stacksRouter.get('/:stackName/backup', async (req: Request, res: Response) => { const stackName = req.params.stackName as string; const fsSvc = FileSystemService.getInstance(req.nodeId); const info = await fsSvc.getBackupInfo(stackName); - res.json(info); + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const currentGen = StackUpdateRecoveryService.getInstance().getCurrent(req.nodeId, stackName); + const recoveryAvailable = !!currentGen; + const exists = info.exists || recoveryAvailable; + const timestamp = currentGen?.created_at ?? info.timestamp; + res.json({ + exists, + timestamp, + recoveryAvailable, + recoveryId: currentGen?.id ?? null, + hasGenerationContent: !!(currentGen && currentGen.content_path), + }); } catch (error: unknown) { console.error('Failed to get backup info:', error); const message = getErrorMessage(error, 'Failed to get backup info.'); @@ -2557,20 +2635,23 @@ stacksRouter.get('/:stackName/backup', async (req: Request, res: Response) => { }); stacksRouter.post('/:stackName/backup', async (req: Request, res: Response) => { - // Triggers a server-side backup of the stack's managed files: the same - // rollback snapshot a deploy takes. Exposed so a scheduled backup can run on - // a remote node through the proxy path, and so an operator can capture an - // on-demand snapshot. + // Captures files, holds, and a recovery override, then hands off that + // generation as current without compose or a runtime probe. Exposed so a + // scheduled backup can run on a remote node through the proxy path, and so + // an operator can capture an on-demand snapshot. const stackName = req.params.stackName as string; if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; if (!(await requireStackExists(req.nodeId, stackName, res))) return; - // The backup slot is shared with the pre-deploy rollback snapshot, so hold the - // stack-op lock to keep a backup from interleaving with a concurrent - // deploy/update/rollback on the same stack. All early-returns stay inside the - // try so finally always releases. + // Handoff mutates the current generation, so hold the stack-op lock against + // a concurrent deploy/update/rollback. After a successful acquire, release + // in finally. if (!tryAcquireStackOpLock(req, res, stackName, 'backup')) return; try { - await FileSystemService.getInstance(req.nodeId).backupStackFiles(stackName); + await StackUpdateRecoveryService.getInstance().captureCurrentBackup({ + nodeId: req.nodeId, + stackName, + createdBy: req.user?.username ?? null, + }); dlog(`[Stacks] Backup completed: ${sanitizeForLog(stackName)}`); res.json({ success: true }); } catch (error: unknown) { @@ -2581,6 +2662,51 @@ stacksRouter.post('/:stackName/backup', async (req: Request, res: Response) => { } }); +function isFleetSnapshotFile(value: unknown): value is { filename: string; content: string } { + return typeof value === 'object' && value !== null + && 'filename' in value && 'content' in value + && typeof value.filename === 'string' + && typeof value.content === 'string'; +} + +stacksRouter.post('/:stackName/fleet-snapshot-apply', async (req: Request, res: Response) => { + // Node-local capture-then-write used by hub Fleet snapshot restore. + const stackName = req.params.stackName as string; + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + const rawFiles = req.body?.files; + if (!Array.isArray(rawFiles) || !rawFiles.every(isFleetSnapshotFile)) { + res.status(400).json({ error: 'files must be an array of { filename, content }' }); + return; + } + const files = selectFleetSnapshotApplyFiles(rawFiles); + if (files.length === 0) { + res.status(400).json({ error: 'files must include compose.yaml or .env' }); + return; + } + try { + const result = await applyFleetSnapshotFiles({ + nodeId: req.nodeId, + stackName, + files, + actor: req.user?.username ?? 'system', + }); + res.json({ success: true, capturedGenerationId: result.capturedGenerationId }); + } catch (error: unknown) { + const code = getCodedError(error)?.code; + if (code === 'INVALID_STACK_NAME' || code === 'INVALID_SNAPSHOT_FILES') { + res.status(400).json({ error: getErrorMessage(error, 'Invalid restore request'), code }); + return; + } + const conflict = fleetSnapshotApplyConflictCode(error); + if (conflict) { + res.status(409).json({ error: getErrorMessage(error, 'Restore conflict'), code: conflict }); + return; + } + console.error('[Stacks] Fleet snapshot apply failed: %s', sanitizeForLog(stackName), error); + res.status(500).json({ error: getErrorMessage(error, 'Failed to restore snapshot files') }); + } +}); + /** * Returns the latest post-deploy scan attempt for this stack, or null if * no scan has been attempted yet. Used by the editor UI to flag stacks diff --git a/backend/src/routes/systemMaintenance.ts b/backend/src/routes/systemMaintenance.ts index 80db7f84..aafe9319 100644 --- a/backend/src/routes/systemMaintenance.ts +++ b/backend/src/routes/systemMaintenance.ts @@ -513,6 +513,8 @@ systemMaintenanceRouter.get('/rollback/generations', async (req: Request, res: R phase: row.phase, createdAt: row.created_at, artifactExpiresAt: row.artifact_expires_at, + createdBy: row.created_by, + operationKind: row.operation_kind, releasable: service.isReleaseEligible(row), }))); } catch (error) { @@ -545,6 +547,11 @@ systemMaintenanceRouter.post('/rollback/generations/:id/release', async (req: Re error: 'This rollback generation cannot be released right now (it may be observing a health gate, mid-recovery, or already in progress).', code: 'NOT_ELIGIBLE', }); + case 'malformed_services': + return res.status(409).json({ + error: 'This rollback generation has malformed recovery image state and cannot be released until that record is repaired.', + code: 'MALFORMED_SERVICES', + }); default: { const _exhaustive: never = result.reason; throw new Error(`Unhandled release reason: ${_exhaustive}`); diff --git a/backend/src/services/ComposeService.ts b/backend/src/services/ComposeService.ts index 68d48b81..30853e37 100644 --- a/backend/src/services/ComposeService.ts +++ b/backend/src/services/ComposeService.ts @@ -1,1412 +1,1631 @@ -import { spawn } from 'child_process'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import WebSocket from 'ws'; -import DockerController from './DockerController'; -import { DatabaseService } from './DatabaseService'; -import { FileSystemService } from './FileSystemService'; -import { MeshService } from './MeshService'; -import { LogFormatter } from './LogFormatter'; -import { NodeRegistry } from './NodeRegistry'; -import { RegistryService } from './RegistryService'; -import { DriftLedgerService } from './DriftLedgerService'; -import SelfIdentityService from './SelfIdentityService'; -import { parseEffectiveModel } from './preflight/effectiveModel'; -import { deriveStackExposure } from './preflight/exposure'; - -import { isDebugEnabled } from '../utils/debug'; -import { getErrorMessage } from '../utils/errors'; -import { normalizeContainerName } from '../utils/log-parsing'; -import { describeSpawnError } from '../utils/spawnErrors'; -import { isPathWithinBase, isValidStackName } from '../utils/validation'; -import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs'; -import { parseMissingRequiredVars } from '../helpers/envVarParse'; -import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; -import { pathsMatch, resolveHostBindPath } from '../utils/composePathMapping'; -import { loadStackBuildServices } from './ImageUpdateService'; -import { resolveMissingExternalNetworks } from './network/resolveMissingExternalNetworks'; -import { - MissingExternalNetworksError, - type DeployInvocationContext, -} from './network/missingExternalNetworksError'; -import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages'; -import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; -import type { NotificationCategory } from './NotificationService'; - -function recordNetworkAutoCreatedActivity( - nodeId: number, - stackName: string, - createdNames: string[], - level: 'info' | 'warning', - ctx?: DeployInvocationContext, -): void { - if (createdNames.length === 0) return; - const names = [...createdNames].sort((a, b) => a.localeCompare(b)).join(', '); - const source = ctx?.source ?? 'manual'; - try { - DatabaseService.getInstance().addNotificationHistory(nodeId, { - level, - category: 'network_auto_created' as NotificationCategory, - message: `Auto-created external network(s) for ${stackName}: ${names} (source: ${source})`, - timestamp: Date.now(), - stack_name: stackName, - actor_username: ctx?.actor ?? null, - }); - } catch (error) { - console.error( - '[ComposeService] Failed to record network_auto_created activity for %s:', - sanitizeForLog(stackName), - sanitizeForLog(getErrorMessage(error, 'unknown')), - ); - } -} - -export class ComposeRollbackError extends Error { - public readonly rollbackAttempted: boolean; - public readonly rolledBack: boolean; - public readonly originalError: unknown; - - constructor(originalError: unknown, rollbackAttempted: boolean, rolledBack: boolean) { - super(getErrorMessage(originalError, 'Compose operation failed')); - this.name = 'ComposeRollbackError'; - this.rollbackAttempted = rollbackAttempted; - this.rolledBack = rolledBack; - this.originalError = originalError; - Object.setPrototypeOf(this, ComposeRollbackError.prototype); - } -} - -export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null { - if (!(error instanceof ComposeRollbackError)) { - return null; - } - return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack }; -} - -const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000; - -/** Public so other services (e.g. recovery claim leases) can size their own timers off the same ceiling without depending on a private module-local. */ -export function getComposeCommandTimeoutMs(): number { - const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS); - if (Number.isFinite(configured) && configured > 0) { - return configured; - } - return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS; -} - -// Idle backstop for long-running pull/recreate steps: if the child emits no -// output for this window while still running, the step is treated as stalled -// and terminated, so a hung `docker compose pull` surfaces a fast failure -// instead of spinning until the much longer command timeout above. Conservative -// by default because a working pull can be briefly silent while a large layer -// extracts; operators on slow links or heavy local builds can raise it. -const DEFAULT_COMPOSE_STALL_TIMEOUT_MS = 10 * 60 * 1000; - -function getComposeStallTimeoutMs(): number { - const configured = Number(process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS); - if (Number.isFinite(configured) && configured > 0) { - return configured; - } - return DEFAULT_COMPOSE_STALL_TIMEOUT_MS; -} - -/** - * ComposeService - local docker compose CLI execution. - * - * In the Distributed API model, remote node compose operations are handled - * by the remote Sencho instance. This service only executes commands locally. - */ -export class ComposeService { - private baseDir: string; - private nodeId: number; - - constructor(nodeId?: number) { - this.nodeId = nodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); - this.baseDir = NodeRegistry.getInstance().getComposeDir(this.nodeId); - } - - public static getInstance(nodeId?: number): ComposeService { - return new ComposeService(nodeId); - } - - /** - * Build the authored `docker compose` argument list for a stack: the validated - * multi-file deploy prefix (ordered `-f` files + `-p ` + - * `--project-directory`) for a Git source with an applied multi-file spec, then - * the Sencho Mesh override file last (highest `-f` precedence) when the stack is - * opted into the mesh, then the action. Single-file / non-git stacks get no file - * prefix, so docker compose's built-in discovery resolves the root compose.yaml, - * byte-identical to the pre-multi-file behavior. The user's source files are - * never mutated. Lifecycle commands (deploy, update, stop/start/restart/down) - * route through this method, so they share one file prefix plus the mesh override. - * Image scans (listStackImages) and the Compose Doctor (renderConfig) reuse the - * same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh - * override, rendering the user's authored model without mesh injection. - */ - /** Public wrapper for dual-arg assembly and recovery Compose invocations. */ - public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise { - return this.authoredComposeArgs(stackName, action); - } - - public async validateStackForMutation(stackName: string): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - } - - /** - * Render/validate the exact Compose invocation used by mutating operations - * (authored files, env pins, and generated Mesh override) before capture. - */ - public async validateExactComposeInvocation(stackName: string): Promise { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - const baseResolved = path.resolve(this.baseDir); - const stackDir = path.resolve(baseResolved, stackName); - if (!stackDir.startsWith(baseResolved + path.sep)) { - throw new Error('Invalid stack path'); - } - const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']); - await this.execute('docker', args, stackDir, undefined, true); - } - - private async authoredComposeArgs(stackName: string, action: string[]): Promise { - const args: string[] = ['compose']; - const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - args.push(...filePrefix); - // Pin env resolution to the root .env when a context dir shifts the project - // directory, so deploy/update resolve the same effective config the validator did. - args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId)); - - const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName); - let overridePath: string | null = null; - try { - overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); - } catch (err) { - if (meshEnabled) { - throw err instanceof Error - ? err - : new Error(`Mesh override generation failed: ${String(err)}`); - } - console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message)); - } - if (meshEnabled && !overridePath) { - throw new Error( - `Mesh override is required for stack "${stackName}" but could not be generated`, - ); - } - if (overridePath) { - if (filePrefix.length === 0) { - // Single-file stack: passing any -f disables compose's auto-discovery, so name - // the base file explicitly, then re-add the user's implicit override (if any) so - // it is not silently dropped, before layering the mesh override on top. - const fsSvc = FileSystemService.getInstance(this.nodeId); - const baseFilename = await fsSvc.getComposeFilename(stackName); - args.push('-f', baseFilename); - let userOverride: string | null = null; - try { - userOverride = await fsSvc.getOverrideFilename(stackName); - } catch (err) { - // Containment-guard rejections (bad stack name / symlink escape) are hard errors: - // abort the deploy rather than degrade. The "no override" case returns null rather - // than throwing, so any other throw is transient I/O: drop the override and proceed - // (logging the consequence) instead of failing the deploy. - const code = (err as { code?: string }).code; - if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { - throw err; - } - console.warn('[ComposeService] could not resolve user compose override; deploying without it:', sanitizeForLog((err as Error).message)); - } - if (userOverride) { - args.push('-f', userOverride); - } - } - args.push('-f', overridePath); - } - args.push(...action); - return args; - } - - private execute( - command: string, - args: string[], - cwd: string, - ws?: WebSocket, - throwOnError = true, - env?: Record, - // When set, terminate the child if it emits no output for this long while - // still running (idle stall backstop). Appended last so the existing - // registry-auth call sites that pass `env` are unaffected. - idleTimeoutMs?: number - ): Promise { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - cwd, - env: env ?? { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - } - }); - - let errorLog = ''; - let settled = false; - let exited = false; - let pendingTerminationError: Error | null = null; - const timeoutMs = getComposeCommandTimeoutMs(); - let timeout: ReturnType | null = null; - let forceKillTimeout: ReturnType | null = null; - let idleTimeout: ReturnType | null = null; - - const sendOutput = (text: string) => { - if (ws && ws.readyState === WebSocket.OPEN) { - ws.send(text); - } - }; - - const cleanup = () => { - if (timeout) { - clearTimeout(timeout); - timeout = null; - } - if (forceKillTimeout) { - clearTimeout(forceKillTimeout); - forceKillTimeout = null; - } - if (idleTimeout) { - clearTimeout(idleTimeout); - idleTimeout = null; - } - }; - - const finish = (complete: () => void) => { - if (settled) return; - settled = true; - cleanup(); - complete(); - }; - - const terminateChild = (error: Error) => { - pendingTerminationError = pendingTerminationError ?? error; - if (exited) return; - try { - child.kill('SIGTERM'); - } catch (error) { - console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - } - forceKillTimeout = setTimeout(() => { - if (exited) return; - try { - child.kill('SIGKILL'); - } catch (error) { - console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - } - }, 5000); - }; - - // Idle stall backstop. Armed once below and reset on every output chunk; - // if it ever fires, the step has been silent for idleTimeoutMs while still - // running, so terminate it. Never rearmed after a termination is pending or - // the child has exited, so it cannot re-fire during the SIGTERM grace. - const armIdleTimeout = () => { - if (idleTimeoutMs === undefined) return; - if (exited || settled || pendingTerminationError) return; - if (idleTimeout) clearTimeout(idleTimeout); - idleTimeout = setTimeout(() => { - const seconds = Math.round(idleTimeoutMs / 1000); - sendOutput(`=== No output for ${seconds}s; the operation appears stalled and was stopped ===\n`); - terminateChild(new Error(`STACK_STALLED_OUTPUT: no output for ${seconds}s`)); - }, idleTimeoutMs); - }; - - // The progress socket is output-only: a deploy/update/down is owned by the - // HTTP request that started it, so closing or losing the socket (the user - // minimizes the panel, navigates away, or the connection blips) must not - // terminate the compose process. Termination is driven solely by the - // command timeout here and the optional idle stall backstop above. - timeout = setTimeout(() => { - const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`; - sendOutput(`${message}\n`); - terminateChild(new Error(message)); - }, timeoutMs); - - armIdleTimeout(); - - const onData = (data: Buffer) => { - const text = data.toString(); - errorLog += text; - sendOutput(text); - armIdleTimeout(); - }; - - child.stdout.on('data', onData); - child.stderr.on('data', onData); - - child.on('close', (code: number | null) => { - exited = true; - finish(() => { - sendOutput(`Command exited with code ${code}\n`); - if (pendingTerminationError) { - if (throwOnError) reject(pendingTerminationError); - else resolve(); - return; - } - if (code === 0) resolve(); - else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`)); - else resolve(); - }); - }); - - child.on('error', (error: Error & { code?: string }) => { - exited = true; - finish(() => { - const mapped = describeSpawnError(error as NodeJS.ErrnoException, { command }); - const message = redactSensitiveText(mapped.message); - sendOutput(`Error: ${message}\n`); - if (mapped.isLowMemory) { - console.warn('[ComposeService] spawn failed under memory pressure:', message); - } - if (pendingTerminationError) { - if (throwOnError) reject(pendingTerminationError); - else resolve(); - return; - } - if (throwOnError) reject(new Error(message)); - else resolve(); - }); - }); - }); - } - - private async withRegistryAuth( - fn: (env: Record) => Promise, - sendOutput?: (data: string) => void, - ): Promise { - const registries = DatabaseService.getInstance().getRegistries(); - if (registries.length === 0) { - return fn({ - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }); - } - - const { config, warnings } = await RegistryService.getInstance().resolveDockerConfig(); - if (warnings.length > 0 && sendOutput) { - for (const warning of warnings) { - sendOutput(`[Sencho] Warning: ${warning}\n`); - } - } - - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-docker-')); - const configPath = path.join(tmpDir, 'config.json'); - - try { - fs.writeFileSync(configPath, JSON.stringify(config), { mode: 0o600 }); - return await fn({ - ...process.env, - DOCKER_CONFIG: tmpDir, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }); - } finally { - // Best-effort cleanup; each step runs independently so a file that was never - // written (e.g., writeFileSync threw) does not prevent the directory removal. - try { fs.unlinkSync(configPath); } catch { /* file may not exist */ } - try { fs.rmdirSync(tmpDir); } catch (e) { - console.warn('[ComposeService] Could not remove temp Docker config dir:', (e as Error).message); - } - } - } - - private async createAtomicBackup( - stackName: string, - operation: 'deployment' | 'update', - sendOutput: (data: string) => void, - ): Promise { - try { - const fsSvc = FileSystemService.getInstance(this.nodeId); - await fsSvc.backupStackFiles(stackName); - sendOutput(`=== Backup created for atomic ${operation} ===\n`); - } catch (error) { - console.error('Atomic backup failed for %s:', sanitizeForLog(stackName), getErrorMessage(error, 'unknown error')); - sendOutput(`=== Atomic ${operation} backup failed. Operation aborted ===\n`); - throw new Error(`Atomic ${operation} backup failed: ${getErrorMessage(error, 'unknown error')}`); - } - } - - private async restoreAtomicBackup( - stackName: string, - stackDir: string, - ws: WebSocket | undefined, - sendOutput: (data: string) => void, - ): Promise { - try { - const fsSvc = FileSystemService.getInstance(this.nodeId); - await fsSvc.restoreStackFiles(stackName); - await this.withRegistryAuth(async (env) => { - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env); - }, sendOutput); - sendOutput('=== Restored previous compose and env files ===\n'); - return true; - } catch (rollbackError) { - console.error('Rollback failed for %s:', sanitizeForLog(stackName), getErrorMessage(rollbackError, 'unknown error')); - sendOutput('=== Rollback failed. Manual intervention may be required ===\n'); - return false; - } - } - - private createContainerCrashError(exitCode: number): Error { - return new Error( - `CONTAINER_CRASHED\nExit Code: ${exitCode}\nContainer exited after deployment. Check container logs for details.` - ); - } - - async runCommand(stackName: string, action: 'down' | 'start' | 'stop' | 'restart', ws?: WebSocket): Promise { - const stackDir = path.join(this.baseDir, stackName); - await this.execute('docker', await this.authoredComposeArgs(stackName, [action]), stackDir, ws); - } - - /** Interactive compose down (Take down UI / POST /down). Plain `down` by default. */ - async runDown(stackName: string, options?: { removeVolumes?: boolean }, ws?: WebSocket): Promise { - const stackDir = path.join(this.baseDir, stackName); - const args = options?.removeVolumes ? ['down', '--volumes'] : ['down']; - await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackDir, ws); - } - - /** - * Opt-in guard: when `env_block_deploy_on_missing_required` is enabled, refuse a - * deploy whose required `${VAR:?err}` variables are unset OR empty, before any - * backup, cleanup, pull, or `up` runs. Compose's own resolution is authoritative - * (it passes process.env), and on the failing path it emits no rendered model, so - * no env value is materialized. Default off and any settings-read failure both - * fall through without blocking. - */ - private async assertRequiredEnvPresent(stackName: string): Promise { - let enabled = false; - try { - enabled = DatabaseService.getInstance().getGlobalSettings()['env_block_deploy_on_missing_required'] === '1'; - } catch { - return; // safe default: a settings-read failure never blocks a deploy - } - if (!enabled) return; - const result = await this.renderConfig(stackName); - const missing = parseMissingRequiredVars(result.stderr); - if (missing.length === 0) return; - const plural = missing.length > 1; - throw new Error( - `Deploy blocked: required environment variable${plural ? 's' : ''} ${missing.join(', ')} ` + - `${plural ? 'are' : 'is'} missing. Define ${plural ? 'them' : 'it'} in a .env or env_file, then deploy again.`, - ); - } - - private async assertSafePilotBindMapping(stackName: string): Promise { - if (process.env.SENCHO_MODE !== 'pilot') return; - - let mounts: Array<{ source: string; destination: string }> | null; - try { - mounts = await SelfIdentityService.getInstance().getBindMounts(); - } catch (error) { - console.warn('[ComposeService] Could not verify pilot compose path mapping:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - return; - } - if (mounts === null) return; - - const composeDir = path.resolve(this.baseDir); - const hostComposeDir = resolveHostBindPath(composeDir, mounts); - if (!hostComposeDir || pathsMatch(hostComposeDir, composeDir)) return; - - const rendered = await this.renderConfig(stackName); - if (rendered.rendered === null) return; - - let parsed: unknown; - try { - parsed = JSON.parse(rendered.rendered); - } catch (error) { - console.warn('[ComposeService] Could not inspect rendered binds for pilot path safety:', sanitizeForLog(getErrorMessage(error, 'unknown'))); - return; - } - const model = parseEffectiveModel(parsed, stackName); - const unsafeBind = model.services - .flatMap((service) => service.binds) - .find((bind) => isPathWithinBase(path.resolve(bind.source), composeDir)); - if (!unsafeBind) return; - - throw new Error( - `Deploy blocked: relative bind mounts resolve under ${composeDir}, but the host path is ${hostComposeDir}. ` + - `Use a 1:1 mount with the same absolute path on the host and in the Pilot Agent, then retry.`, - ); - } - - /** - * Missing-external gate: after env/Pilot asserts, before atomic backup. - * Creates safe bridge networks only when the opt-in setting is on. - */ - private async ensureExternalNetworksForDeploy( - stackName: string, - ctx?: DeployInvocationContext, - ): Promise { - const resolved = await resolveMissingExternalNetworks(this.nodeId, stackName); - if (resolved.status === 'render_unavailable') { - throw new MissingExternalNetworksError({ - kind: 'unavailable', - message: 'Sencho could not render this stack\'s Compose model to check external networks.', - }); - } - if (resolved.status === 'runtime_unavailable') { - // No declared externals: nothing to verify; proceed. - if (resolved.declaredExternalCount === 0) return; - throw new MissingExternalNetworksError({ - kind: 'unavailable', - message: 'Sencho could not read Docker networking state to check external networks.', - }); - } - - if (resolved.networks.length === 0) return; - - const unsafe = resolved.networks.filter((n) => !n.safe); - if (unsafe.length > 0) { - throw new MissingExternalNetworksError({ - kind: 'unsupported', - message: 'One or more missing external networks cannot be created safely by Sencho.', - networks: resolved.networks, - }); - } - - if (!resolved.autoCreateEnabled) { - throw new MissingExternalNetworksError({ - kind: 'prompt', - message: 'One or more external networks required by this stack are missing on this node.', - networks: resolved.networks, - }); - } - - const docker = DockerController.getInstance(this.nodeId); - const createdNames: string[] = []; - const recordCreatedNetworks = (level: 'info' | 'warning') => { - if (createdNames.length === 0) return; - invalidateNodeCaches(this.nodeId); - recordNetworkAutoCreatedActivity(this.nodeId, stackName, createdNames, level, ctx); - }; - - for (const network of resolved.networks) { - try { - await docker.createNetwork({ Name: network.name, Driver: 'bridge' }); - createdNames.push(network.name); - } catch (createErr) { - // Authoritative re-check: continue only if the network now exists. - let exists = false; - try { - const knownStacks = await FileSystemService.getInstance(this.nodeId).getStacks(); - const snapshot = await docker.getDependencySnapshot(knownStacks); - exists = snapshot.networks.some((n) => n.name === network.name); - } catch (snapErr) { - console.warn( - '[ComposeService] Post-create snapshot failed for %s:', - sanitizeForLog(network.name), - sanitizeForLog(getErrorMessage(snapErr, 'unknown')), - ); - } - if (!exists) { - recordCreatedNetworks('warning'); - throw new MissingExternalNetworksError({ - kind: 'create_failed', - message: `Failed to create external network "${network.name}".`, - networks: resolved.networks, - createdNames, - remainingNames: resolved.networks - .map((missingNetwork) => missingNetwork.name) - .filter((name) => !createdNames.includes(name)), - }); - } - // Race-existing: do not record in createdNames. - } - } - - // Re-resolve before Compose. - const recheck = await resolveMissingExternalNetworks(this.nodeId, stackName); - if (recheck.status !== 'ok' || recheck.networks.length > 0) { - recordCreatedNetworks('warning'); - throw new MissingExternalNetworksError({ - kind: recheck.status === 'ok' ? 'create_failed' : 'unavailable', - message: 'External networks were still missing after automatic creation.', - networks: recheck.networks, - createdNames, - remainingNames: recheck.networks.map((n) => n.name), - }); - } - - recordCreatedNetworks('info'); - } - - async deployStack( - stackName: string, - ws?: WebSocket, - atomic?: boolean, - ctx?: DeployInvocationContext, - ): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - await this.ensureExternalNetworksForDeploy(stackName, ctx); - - const stackDir = path.join(this.baseDir, stackName); - const debug = isDebugEnabled(); - const t0 = Date.now(); - if (debug) console.debug('[ComposeService:debug] deployStack', { stackName, stackDir, atomic }); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - - if (atomic) { - await this.createAtomicBackup(stackName, 'deployment', sendOutput); - } - - try { - try { - const dockerController = DockerController.getInstance(this.nodeId); - const legacyOrphans = await dockerController.getLegacyOrphanContainersByStack(stackName); - if (legacyOrphans.length > 0) { - sendOutput(`=== Cleaning up legacy orphan containers before deployment ===\n`); - await dockerController.removeContainers(legacyOrphans.map((c) => c.Id)); - } - } catch (e) { - console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e); - } - - await this.withRegistryAuth(async (env) => { - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - - // Post-Deploy Health Probe - await new Promise(resolve => setTimeout(resolve, 3000)); - - const dockerController = DockerController.getInstance(this.nodeId); - const containers = await dockerController.getDocker().listContainers({ - all: true, - filters: { label: [`com.docker.compose.project=${stackName}`] } - }); - - for (const containerInfo of containers) { - if (containerInfo.State === 'exited') { - const container = dockerController.getDocker().getContainer(containerInfo.Id); - const inspectData = await container.inspect(); - const exitCode = inspectData.State.ExitCode; - - if (exitCode !== 0) { - throw this.createContainerCrashError(exitCode); - } - } - } - if (debug) console.debug(`[ComposeService:debug] deployStack completed in ${Date.now() - t0}ms`, { stackName }); - } catch (deployError) { - if (atomic) { - sendOutput('\n=== Deployment failed - restoring previous compose and env files ===\n'); - const rolledBack = await this.restoreAtomicBackup(stackName, stackDir, ws, sendOutput); - throw new ComposeRollbackError(deployError, true, rolledBack); - } - throw deployError; - } - // Reached only on a successful deploy (the catch above always rethrows). Record - // the drift baseline here so every deploy path gets one, not just the manual - // route: bulk, Git-source, App Store, scheduler, and webhook deploys all funnel - // through this method. Internally guarded; awaited so it cannot race later work. - await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); - // Reconcile the ledger against the just-deployed runtime: findings this deploy - // fixed are resolved and any it left are recorded (and surfaced in the activity - // feed) now, instead of waiting for someone to open the Drift tab. The rollback - // route re-deploys through this method, so it is covered; a failed atomic deploy - // instead restores the previous files and throws above, so that recovery path - // reconciles on its next deploy or scan, not here. Best-effort internally. - await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); - // Refresh the exposure cache so posture reflects the just-deployed model. - // Best-effort: a refresh failure logs a warning but never fails the deploy. - try { - await this.refreshExposureCache(stackName); - } catch (err) { - console.warn('[ComposeService] Exposure refresh failed after deploy for %s:', - sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown'))); - } - } - - streamLogs(stackName: string, ws: WebSocket) { - let isClosed = false; - let isFirstRun = true; - let isWaitingForActivity = false; - - ws.on('close', () => { isClosed = true; }); - - const startStream = async () => { - if (isClosed || ws.readyState !== WebSocket.OPEN) return; - - try { - const dockerController = DockerController.getInstance(this.nodeId); - const containers = await dockerController.getContainersByStack(stackName); - - if (!containers || containers.length === 0) { - if (!isWaitingForActivity) { - ws.send(`\r\n\x1b[33m[Sencho] No containers found. Waiting for activity...\x1b[0m\r\n`); - isWaitingForActivity = true; - } - setTimeout(startStream, 2000); - return; - } - - const runningContainers = containers.filter((c: any) => c.State === 'running'); - - if (!isFirstRun && runningContainers.length === 0) { - if (!isWaitingForActivity) { - ws.send(`\r\n\x1b[33m[Sencho] Log stream ended. Waiting for container activity...\x1b[0m\r\n`); - isWaitingForActivity = true; - } - setTimeout(startStream, 2000); - return; - } - - const containersToLog = isFirstRun ? containers : runningContainers; - isFirstRun = false; - isWaitingForActivity = false; - - let activeProcesses = 0; - let streamEndedHandled = false; - const localProcesses: ReturnType[] = []; - - const onWsClose = () => { - localProcesses.forEach(cp => { try { cp.kill(); } catch { } }); - }; - - ws.on('close', onWsClose); - - const handleProcessEnd = () => { - activeProcesses--; - if (activeProcesses <= 0 && !streamEndedHandled) { - streamEndedHandled = true; - ws.removeListener('close', onWsClose); - if (!isClosed && ws.readyState === WebSocket.OPEN) { - setTimeout(startStream, 1000); - } - } - }; - - for (const container of containersToLog) { - const rawName = container.Names?.[0]?.replace(/^\//, '') || container.Id; - const displayName = normalizeContainerName(rawName, stackName); - activeProcesses++; - let lineBuffer = ''; - - const sendOutput = (data: Buffer) => { - if (ws.readyState === WebSocket.OPEN) { - lineBuffer += data.toString(); - const lines = lineBuffer.split(/\r?\n/); - lineBuffer = lines.pop() || ''; - for (const line of lines) { - ws.send(LogFormatter.process(`${displayName} | ${line}`) + '\r\n'); - } - } - }; - - const flushBuffer = () => { - if (lineBuffer && ws.readyState === WebSocket.OPEN) { - ws.send(LogFormatter.process(`${displayName} | ${lineBuffer}`) + '\r\n'); - lineBuffer = ''; - } - }; - - const child = spawn('docker', ['logs', '-f', '-t', '--tail', '100', rawName], { - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - } - }); - localProcesses.push(child); - child.stdout.on('data', sendOutput); - child.stderr.on('data', sendOutput); - child.on('error', handleProcessEnd); - child.on('close', () => { - flushBuffer(); - handleProcessEnd(); - }); - } - } catch (err) { - if (!isClosed && ws.readyState === WebSocket.OPEN) { - if (!isWaitingForActivity) { - ws.send(`\r\n\x1b[31m[Sencho] Error tracking containers. Retrying...\x1b[0m\r\n`); - isWaitingForActivity = true; - } - setTimeout(startStream, 2000); - } - } - }; - - startStream(); - } - - - /** - * Authored (+ mesh) compose args with an optional recovery override layered LAST. - * Used for pinned lifecycle / compensation ups (`--pull never --no-build`). - */ - public async buildComposeArgsWithRecoveryOverride( - stackName: string, - action: string[], - recoveryOverridePath: string | null, - ): Promise { - const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']); - const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__'); - const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel; - const out = [...prefix]; - if (recoveryOverridePath) { - if (!out.includes('-f')) { - const fsSvc = FileSystemService.getInstance(this.nodeId); - const baseFilename = await fsSvc.getComposeFilename(stackName); - out.push('-f', baseFilename); - try { - const userOverride = await fsSvc.getOverrideFilename(stackName); - if (userOverride) out.push('-f', userOverride); - } catch (err) { - const code = (err as { code?: string }).code; - if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { - throw err; - } - console.warn( - '[ComposeService] could not resolve user compose override for recovery args:', - sanitizeForLog((err as Error).message), - ); - } - } - out.push('-f', recoveryOverridePath); - } - out.push(...action); - return out; - } - - async updateStack( - stackName: string, - ws?: WebSocket, - atomic?: boolean, - ): Promise<{ recoveryId: string | null }> { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - const stackDir = path.join(this.baseDir, stackName); - const debug = isDebugEnabled(); - const t0 = Date.now(); - if (debug) console.debug('[ComposeService:debug] updateStack', { stackName, stackDir, atomic }); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - - // Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs). - const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); - const recoverySvc = StackUpdateRecoveryService.getInstance(); - let recoveryId: string | null = null; - let handedOff = false; - - try { - sendOutput('=== Validating stack for update ===\n'); - sendOutput('=== Capturing rollback generation ===\n'); - const candidate = await recoverySvc.captureCandidate({ - nodeId: this.nodeId, - stackName, - createdBy: null, - }); - recoveryId = candidate.id; - - const buildServices = await loadStackBuildServices(this.nodeId, stackName); - const buildAware = buildServices.length > 0; - - try { - await this.withRegistryAuth(async (env) => { - if (buildAware) { - sendOutput('=== Building images ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['build', '--pull']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - sendOutput('=== Pulling registry images ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - } else { - sendOutput('=== Pulling latest images ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['pull']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - } - }, sendOutput); - } catch (acquireError) { - // Acquisition failure: abandon candidate; leave runtime untouched. - await recoverySvc.abandon(candidate.id); - recoveryId = null; - throw acquireError; - } - - if (!recoverySvc.markAcquired(candidate.id)) { - await recoverySvc.abandon(candidate.id); - throw new Error('Failed to mark recovery generation as acquired'); - } - - const dockerController = DockerController.getInstance(this.nodeId); - sendOutput('=== Classifying legacy orphans ===\n'); - const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName); - if (classified.status === 'classification_failed') { - await recoverySvc.abandon(candidate.id); - recoveryId = null; - throw new Error(`Legacy orphan classification failed: ${classified.error}`); - } - - if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { - await recoverySvc.abandon(candidate.id); - recoveryId = null; - throw new Error('Failed to hand off recovery generation'); - } - handedOff = true; - if (!recoverySvc.markReconciling(candidate.id)) { - throw new Error('Failed to mark recovery generation as reconciling after handoff'); - } - - if (classified.status === 'orphans') { - sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`); - const results = await dockerController.removeContainers(classified.ids); - const failed = results.filter((r) => !r.success); - if (failed.length > 0) { - throw new Error( - `Failed to remove ${failed.length} legacy orphan container(s) after handoff`, - ); - } - } - - await this.withRegistryAuth(async (env) => { - sendOutput('=== Recreating containers ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); - }, sendOutput); - - // Immediate verification probe - await new Promise((resolve) => setTimeout(resolve, 3000)); - - const containers = await dockerController.getDocker().listContainers({ - all: true, - filters: { label: [`com.docker.compose.project=${stackName}`] }, - }); - - for (const containerInfo of containers) { - if (containerInfo.State === 'exited') { - const container = dockerController.getDocker().getContainer(containerInfo.Id); - const inspectData = await container.inspect(); - const exitCode = inspectData.State.ExitCode; - if (exitCode !== 0) { - throw this.createContainerCrashError(exitCode); - } - } - } - - if (!recoverySvc.markImmediateVerified(candidate.id)) { - console.warn( - '[ComposeService] Could not CAS immediate_verified for recovery %s', - sanitizeForLog(candidate.id), - ); - } - - sendOutput('=== Stack updated successfully ===\n'); - - // Defer prune until gate retention / gate link: only prune when no active holds - // would be violated. Still honor prune_on_update, but use unified holds so - // candidate/current rollback images are retained. - try { - const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'; - if (pruneOnUpdate) { - const isImageHeld = buildUnifiedHeldImagePredicate(this.nodeId); - const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld); - const reclaimed = result.reclaimedBytes > 0 - ? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB` - : ''; - sendOutput(`=== Pruned dangling images${reclaimed} ===\n`); - } - } catch (pruneError) { - console.warn( - 'Failed to prune dangling images after update for %s:', - sanitizeForLog(stackName), - pruneError, - ); - } - - if (debug) { - console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName }); - } - } catch (updateError) { - if (!handedOff && recoveryId) { - await recoverySvc.abandon(recoveryId); - recoveryId = null; - } - if (handedOff && recoveryId) { - sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n'); - const rolledBack = await recoverySvc.compensateWithCandidate( - recoveryId, - async (overridePath) => { - await this.withRegistryAuth(async (env) => { - await this.execute( - 'docker', - await this.buildComposeArgsWithRecoveryOverride( - stackName, - ['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'], - overridePath, - ), - stackDir, - ws, - true, - env, - getComposeStallTimeoutMs(), - ); - }, sendOutput); - }, - ); - throw new ComposeRollbackError(updateError, true, rolledBack); - } - // Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched. - throw updateError; - } - - await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); - await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); - try { - await this.refreshExposureCache(stackName); - } catch (err) { - console.warn( - '[ComposeService] Exposure refresh failed after update for %s:', - sanitizeForLog(stackName), - sanitizeForLog(getErrorMessage(err, 'unknown')), - ); - } - return { recoveryId }; - } - - /** - * Service-scoped update: pull (or `build --pull` for a build-backed service) - * and recreate a single service's replicas in place. Always - * `--no-deps --force-recreate`, never `--remove-orphans`, so sibling services - * keep their container ids and StartedAt. No drift re-baseline and no dangling - * prune here; the orchestrator owns per-service post-update reconciliation. - */ - async updateService(stackName: string, serviceName: string, hasBuild: boolean, ws?: WebSocket): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - const stackDir = path.join(this.baseDir, stackName); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - await this.withRegistryAuth(async (env) => { - if (hasBuild) { - sendOutput(`=== Building ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - } else { - sendOutput(`=== Pulling ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - } - sendOutput(`=== Recreating ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - } - - /** - * Recreate a single service from the image already present locally, without - * pulling or building. Used by service restore after the recovery image id has - * been retagged onto the declared ref (`--pull never --no-build` so Compose - * uses the just-retagged local image). Always `--no-deps --force-recreate`, - * never `--remove-orphans`. - */ - async recreateServiceFromLocal(stackName: string, serviceName: string, ws?: WebSocket): Promise { - await this.assertRequiredEnvPresent(stackName); - await this.assertSafePilotBindMapping(stackName); - const stackDir = path.join(this.baseDir, stackName); - const sendOutput = (data: string) => { - if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); - }; - await this.withRegistryAuth(async (env) => { - sendOutput(`=== Restoring ${serviceName} ===\n`); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', '--pull', 'never', '--no-build', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - } - - public async downStack(stackName: string, options?: { removeVolumes?: boolean }): Promise { - const stackPath = path.join(this.baseDir, stackName); - try { - const args = options?.removeVolumes - ? ['down', '--volumes', '--remove-orphans'] - : ['down', '--remove-orphans']; - await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackPath, undefined, false); - } catch (error) { - console.warn(`[Teardown] Docker down failed or nothing to clean up for ${sanitizeForLog(stackName)}`); - } - } - - /** - * Enumerate image references declared in a stack's compose file. - * - * Used by the pre-deploy policy gate to decide which images to scan before - * `docker compose up` runs. Path traversal is guarded against the node's - * compose base directory; missing / unreadable compose files or `.env` - * interpolation failures surface as a rejected Promise so the gate can - * block the deploy rather than silently allow it. - */ - public async listStackImages(stackName: string): Promise { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - const stackDir = path.resolve(this.baseDir, stackName); - if (!isPathWithinBase(stackDir, this.baseDir) || path.resolve(this.baseDir) === stackDir) { - throw new Error('Invalid stack path'); - } - // Use the authored multi-file model (no mesh override) so override-only image - // refs are scanned by the policy gate; single-file stacks get an empty prefix. - const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); - const stdout = await this.captureCompose([...filePrefix, ...envFileArgs, 'config', '--images'], stackDir); - const seen = new Set(); - const images: string[] = []; - for (const raw of stdout.split(/\r?\n/)) { - const line = raw.trim(); - if (!line) continue; - if (line.startsWith('sha256:')) continue; - if (seen.has(line)) continue; - seen.add(line); - images.push(line); - } - return images; - } - - /** Render the effective Compose model and cache the per-stack exposure - * descriptor so the Security posture can join exposed images against - * vulnerability findings without re-rendering config on every poll. - * Best-effort: render or parse failure logs a warning and keeps the - * prior cached descriptor, never failing the deploy. */ - private async refreshExposureCache(stackName: string): Promise { - const result = await this.renderConfig(stackName); - if (result.rendered === null) { - console.warn('[ComposeService] Exposure cache skipped for %s: model not renderable', - sanitizeForLog(stackName)); - return; - } - let parsed: unknown; - try { - parsed = JSON.parse(result.rendered); - } catch { - console.warn('[ComposeService] Exposure cache skipped for %s: unparseable model JSON', - sanitizeForLog(stackName)); - return; - } - const model = parseEffectiveModel(parsed, stackName); - const descriptor = deriveStackExposure(model, stackName, Date.now()); - DatabaseService.getInstance().upsertStackExposure( - this.nodeId, - stackName, - JSON.stringify(descriptor), - descriptor.computedAt, - ); - } - - private captureCompose(args: string[], cwd: string): Promise { - return new Promise((resolve, reject) => { - const child = spawn('docker', ['compose', ...args], { - cwd, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }, - }); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (data: Buffer) => { stdout += data.toString(); }); - child.stderr.on('data', (data: Buffer) => { stderr += data.toString(); }); - child.on('error', (err: NodeJS.ErrnoException) => { - const mapped = describeSpawnError(err, { command: 'docker compose' }); - if (mapped.isLowMemory) { - console.warn('[ComposeService] captureCompose spawn failed under memory pressure:', mapped.message); - } - reject(new Error(mapped.message)); - }); - child.on('close', (code) => { - if (code === 0) resolve(stdout); - else reject(new Error(stderr.trim() || `docker compose ${args.join(' ')} failed with code ${code}`)); - }); - }); - } - - /** - * Render the effective compose model as YAML (the default `docker compose - * config` output) with the exact authored invocation and NO mesh override. - * Used by the Git source detach/export contract: the rendered model becomes - * the stack's single compose.yaml. Throws when the render fails or times - * out, so the detach transaction aborts before anything changes. - */ - public async renderComposeYaml(stackName: string): Promise { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - const baseResolved = path.resolve(this.baseDir); - const stackDir = path.resolve(baseResolved, stackName); - if (!stackDir.startsWith(baseResolved + path.sep)) { - throw new Error('Invalid stack path'); - } - let filePrefix: string[]; - try { - filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - } catch (err) { - throw err instanceof Error ? err : new Error(String(err)); - } - const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); - const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config'], { - cwd: stackDir, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }, - }); - return new Promise((resolve, reject) => { - const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream - const TIMEOUT_MS = 30_000; - // Accumulate Buffer chunks and decode ONCE at the end: chunk-wise - // toString() can split a multi-byte UTF-8 sequence across a chunk - // boundary and mangle non-ASCII values. - const outChunks: Buffer[] = []; - let outBytes = 0; - let stderr = ''; - let capped = false; - let settled = false; - const timer = setTimeout(() => { - settled = true; - clearTimeout(timer); - try { - child.kill('SIGKILL'); - } catch { - // best effort - } - reject(new Error(`docker compose config timed out after ${TIMEOUT_MS / 1000}s`)); - }, TIMEOUT_MS); - const finish = (error: Error | null) => { - if (settled) return; - settled = true; - clearTimeout(timer); - if (error) reject(error); - else resolve(Buffer.concat(outChunks).toString('utf8')); - }; - child.stdout.on('data', (data: Buffer) => { - if (capped) return; - outBytes += data.length; - if (outBytes > MAX_OUTPUT) { - // A truncated model frequently still parses as YAML; overwriting a - // working compose.yaml with it would be silent corruption. The cap is - // an error, not a truncation. - capped = true; - settled = true; - clearTimeout(timer); - try { - child.kill('SIGKILL'); - } catch { - // best effort - } - reject(new Error(`docker compose config output exceeded ${MAX_OUTPUT} bytes`)); - return; - } - outChunks.push(data); - }); - child.stderr.on('data', (data: Buffer) => { - if (stderr.length < MAX_OUTPUT) stderr += data.toString(); - }); - child.on('close', (code) => { - if (capped) return; - if (code === 0) finish(null); - else finish(new Error(stderr.trim() || `docker compose config exited with code ${code}`)); - }); - child.on('error', (err) => finish(err)); - }); - } - - /** - * Render the fully-resolved effective Compose model via `docker compose - * config --format json`. This is the AUTHORED model: it does NOT splice in - * the Sencho Mesh override, so it stays read-only (the override is - * write-generated) and reflects what the user actually edits. The override - * would also add the managed `sencho_mesh` external network and per-service - * mesh attachments, which would make preflight emit a false "external network - * not found" finding, so rendering the authored model is both safer and more - * accurate here. - * Captures stderr (where Compose reports unset variables) and never rejects - * on a non-zero exit, so the Compose Doctor can turn a failed render into a - * finding rather than an exception. Bounded by a timeout and an output cap. - * Rejects only when the docker binary cannot be spawned. - */ - public async renderConfig( - stackName: string, - ): Promise<{ rendered: string | null; stderr: string; code: number | null; timedOut: boolean }> { - if (!isValidStackName(stackName)) { - throw new Error('Invalid stack path'); - } - // Canonical inline js/path-injection barrier, kept in the same scope as the - // spawn cwd sink below. CodeQL credits neither the wrapped isPathWithinBase - // helper nor a barrier separated from the sink by the Promise-executor - // closure, so the spawn is hoisted out of the executor. startsWith already - // rejects the base dir itself, since base does not start with base + sep. - const baseResolved = path.resolve(this.baseDir); - const stackDir = path.resolve(baseResolved, stackName); - if (!stackDir.startsWith(baseResolved + path.sep)) { - throw new Error('Invalid stack path'); - } - // Render the authored multi-file model (no mesh override) so the Compose Doctor - // sees every override file; single-file stacks get an empty prefix. The env-file - // flag keeps render resolving the same root .env the validator and deploy use. - let filePrefix: string[]; - try { - filePrefix = authoredComposeFileArgs(stackName, this.nodeId); - } catch (err) { - throw err instanceof Error ? err : new Error(String(err)); - } - const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); - const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config', '--format', 'json'], { - cwd: stackDir, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', - }, - }); - return new Promise((resolve, reject) => { - const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream - const TIMEOUT_MS = 20_000; - let stdout = ''; - let stderr = ''; - let timedOut = false; - let capped = false; - let settled = false; - const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, TIMEOUT_MS); - const finish = (result: { rendered: string | null; stderr: string; code: number | null; timedOut: boolean }) => { - if (settled) return; - settled = true; - clearTimeout(timer); - resolve(result); - }; - child.stdout.on('data', (data: Buffer) => { - stdout += data.toString(); - if (stdout.length > MAX_OUTPUT && !capped) { capped = true; child.kill('SIGKILL'); } - }); - child.stderr.on('data', (data: Buffer) => { - if (stderr.length < MAX_OUTPUT) stderr += data.toString(); - }); - child.on('error', (err: NodeJS.ErrnoException) => { - if (settled) return; - settled = true; - clearTimeout(timer); - reject(new Error(describeSpawnError(err, { command: 'docker compose' }).message)); - }); - child.on('close', (code) => { - if (timedOut) finish({ rendered: null, stderr: stderr.trim() || 'docker compose config timed out', code, timedOut: true }); - else if (capped) finish({ rendered: null, stderr: 'Rendered model exceeded the size limit', code, timedOut: false }); - else if (code === 0) finish({ rendered: stdout, stderr, code, timedOut: false }); - else finish({ rendered: null, stderr: stderr.trim() || `docker compose config failed with code ${code}`, code, timedOut: false }); - }); - }); - } -} +import { spawn } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import WebSocket from 'ws'; +import DockerController from './DockerController'; +import { DatabaseService } from './DatabaseService'; +import { FileSystemService } from './FileSystemService'; +import { MeshService } from './MeshService'; +import { LogFormatter } from './LogFormatter'; +import { NodeRegistry } from './NodeRegistry'; +import { RegistryService } from './RegistryService'; +import { DriftLedgerService } from './DriftLedgerService'; +import SelfIdentityService from './SelfIdentityService'; +import { parseEffectiveModel } from './preflight/effectiveModel'; +import { deriveStackExposure } from './preflight/exposure'; + +import { isDebugEnabled } from '../utils/debug'; +import { getErrorMessage } from '../utils/errors'; +import { normalizeContainerName } from '../utils/log-parsing'; +import { describeSpawnError } from '../utils/spawnErrors'; +import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation'; +import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs'; +import type { RollbackInvocationRecord } from '../types/rollbackGeneration'; +import { parseMissingRequiredVars } from '../helpers/envVarParse'; +import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; +import { pathsMatch, resolveHostBindPath } from '../utils/composePathMapping'; +import { loadStackBuildServices } from './ImageUpdateService'; +import { resolveMissingExternalNetworks } from './network/resolveMissingExternalNetworks'; +import { + MissingExternalNetworksError, + type DeployInvocationContext, +} from './network/missingExternalNetworksError'; +import { buildUnifiedHeldImagePredicate } from './recoveryHeldImages'; +import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; +import type { NotificationCategory } from './NotificationService'; + +/** True when a generation capture recorded an invocation object (prefix may be empty). */ +function hasUsableCapturedInvocation( + invocation: RollbackInvocationRecord | null | undefined, +): invocation is RollbackInvocationRecord { + return invocation != null; +} + +function recordNetworkAutoCreatedActivity( + nodeId: number, + stackName: string, + createdNames: string[], + level: 'info' | 'warning', + ctx?: DeployInvocationContext, +): void { + if (createdNames.length === 0) return; + const names = [...createdNames].sort((a, b) => a.localeCompare(b)).join(', '); + const source = ctx?.source ?? 'manual'; + try { + DatabaseService.getInstance().addNotificationHistory(nodeId, { + level, + category: 'network_auto_created' as NotificationCategory, + message: `Auto-created external network(s) for ${stackName}: ${names} (source: ${source})`, + timestamp: Date.now(), + stack_name: stackName, + actor_username: ctx?.actor ?? null, + }); + } catch (error) { + console.error( + '[ComposeService] Failed to record network_auto_created activity for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } +} + +export class ComposeRollbackError extends Error { + public readonly rollbackAttempted: boolean; + public readonly rolledBack: boolean; + public readonly originalError: unknown; + + constructor(originalError: unknown, rollbackAttempted: boolean, rolledBack: boolean) { + super(getErrorMessage(originalError, 'Compose operation failed')); + this.name = 'ComposeRollbackError'; + this.rollbackAttempted = rollbackAttempted; + this.rolledBack = rolledBack; + this.originalError = originalError; + Object.setPrototypeOf(this, ComposeRollbackError.prototype); + } +} + +export function getComposeRollbackInfo(error: unknown): { attempted: boolean; rolledBack: boolean } | null { + if (!(error instanceof ComposeRollbackError)) { + return null; + } + return { attempted: error.rollbackAttempted, rolledBack: error.rolledBack }; +} + +function isNonFatalCompensationError(error: unknown): boolean { + const code = (error as { code?: string }).code; + return code === 'HELD_IMAGE_MISSING' || code === 'RECOVERY_PROBE_FAILED'; +} + +async function compensateOrSwallow(compensate: () => Promise): Promise { + try { + return await compensate(); + } catch (compError) { + if (!isNonFatalCompensationError(compError)) throw compError; + return false; + } +} + +const DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS = 30 * 60 * 1000; + +/** Public so other services (e.g. recovery claim leases) can size their own timers off the same ceiling without depending on a private module-local. */ +export function getComposeCommandTimeoutMs(): number { + const configured = Number(process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS); + if (Number.isFinite(configured) && configured > 0) { + return configured; + } + return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS; +} + +// Idle backstop for long-running pull/recreate steps: if the child emits no +// output for this window while still running, the step is treated as stalled +// and terminated, so a hung `docker compose pull` surfaces a fast failure +// instead of spinning until the much longer command timeout above. Conservative +// by default because a working pull can be briefly silent while a large layer +// extracts; operators on slow links or heavy local builds can raise it. +const DEFAULT_COMPOSE_STALL_TIMEOUT_MS = 10 * 60 * 1000; + +function getComposeStallTimeoutMs(): number { + const configured = Number(process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS); + if (Number.isFinite(configured) && configured > 0) { + return configured; + } + return DEFAULT_COMPOSE_STALL_TIMEOUT_MS; +} + +/** + * ComposeService - local docker compose CLI execution. + * + * In the Distributed API model, remote node compose operations are handled + * by the remote Sencho instance. This service only executes commands locally. + */ +export class ComposeService { + private baseDir: string; + private nodeId: number; + + constructor(nodeId?: number) { + this.nodeId = nodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); + this.baseDir = NodeRegistry.getInstance().getComposeDir(this.nodeId); + } + + public static getInstance(nodeId?: number): ComposeService { + return new ComposeService(nodeId); + } + + /** + * Build the authored `docker compose` argument list for a stack: the validated + * multi-file deploy prefix (ordered `-f` files + `-p ` + + * `--project-directory`) for a Git source with an applied multi-file spec, then + * the Sencho Mesh override file last (highest `-f` precedence) when the stack is + * opted into the mesh, then the action. Single-file / non-git stacks get no file + * prefix, so docker compose's built-in discovery resolves the root compose.yaml, + * byte-identical to the pre-multi-file behavior. The user's source files are + * never mutated. Lifecycle commands (deploy, update, stop/start/restart/down) + * route through this method, so they share one file prefix plus the mesh override. + * Image scans (listStackImages) and the Compose Doctor (renderConfig) reuse the + * same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh + * override, rendering the user's authored model without mesh injection. + */ + /** Public wrapper for dual-arg assembly and recovery Compose invocations. */ + public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise { + return this.authoredComposeArgs(stackName, action); + } + + public async validateStackForMutation(stackName: string): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + } + + /** + * Render/validate the exact Compose invocation used by mutating operations + * (authored files, env pins, and generated Mesh override) before capture. + */ + public async validateExactComposeInvocation(stackName: string): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']); + await this.execute('docker', args, stackDir, undefined, true); + } + + private async authoredComposeArgs(stackName: string, action: string[]): Promise { + const args: string[] = ['compose']; + const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + args.push(...filePrefix); + // Pin env resolution to the root .env when a context dir shifts the project + // directory, so deploy/update resolve the same effective config the validator did. + args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId)); + + const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName); + let overridePath: string | null = null; + try { + overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); + } catch (err) { + if (meshEnabled) { + throw err instanceof Error + ? err + : new Error(`Mesh override generation failed: ${String(err)}`); + } + console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message)); + } + if (meshEnabled && !overridePath) { + throw new Error( + `Mesh override is required for stack "${stackName}" but could not be generated`, + ); + } + if (overridePath) { + if (filePrefix.length === 0) { + // Single-file stack: passing any -f disables compose's auto-discovery, so name + // the base file explicitly, then re-add the user's implicit override (if any) so + // it is not silently dropped, before layering the mesh override on top. + const fsSvc = FileSystemService.getInstance(this.nodeId); + const baseFilename = await fsSvc.getComposeFilename(stackName); + args.push('-f', baseFilename); + let userOverride: string | null = null; + try { + userOverride = await fsSvc.getOverrideFilename(stackName); + } catch (err) { + // Containment-guard rejections (bad stack name / symlink escape) are hard errors: + // abort the deploy rather than degrade. The "no override" case returns null rather + // than throwing, so any other throw is transient I/O: drop the override and proceed + // (logging the consequence) instead of failing the deploy. + const code = (err as { code?: string }).code; + if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { + throw err; + } + console.warn('[ComposeService] could not resolve user compose override; deploying without it:', sanitizeForLog((err as Error).message)); + } + if (userOverride) { + args.push('-f', userOverride); + } + } + args.push('-f', overridePath); + } + args.push(...action); + return args; + } + + private execute( + command: string, + args: string[], + cwd: string, + ws?: WebSocket, + throwOnError = true, + env?: Record, + // When set, terminate the child if it emits no output for this long while + // still running (idle stall backstop). Appended last so the existing + // registry-auth call sites that pass `env` are unaffected. + idleTimeoutMs?: number + ): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + env: env ?? { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + } + }); + + let errorLog = ''; + let settled = false; + let exited = false; + let pendingTerminationError: Error | null = null; + const timeoutMs = getComposeCommandTimeoutMs(); + let timeout: ReturnType | null = null; + let forceKillTimeout: ReturnType | null = null; + let idleTimeout: ReturnType | null = null; + + const sendOutput = (text: string) => { + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(text); + } + }; + + const cleanup = () => { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + if (forceKillTimeout) { + clearTimeout(forceKillTimeout); + forceKillTimeout = null; + } + if (idleTimeout) { + clearTimeout(idleTimeout); + idleTimeout = null; + } + }; + + const finish = (complete: () => void) => { + if (settled) return; + settled = true; + cleanup(); + complete(); + }; + + const terminateChild = (error: Error) => { + pendingTerminationError = pendingTerminationError ?? error; + if (exited) return; + try { + child.kill('SIGTERM'); + } catch (error) { + console.warn('[ComposeService] Failed to terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } + forceKillTimeout = setTimeout(() => { + if (exited) return; + try { + child.kill('SIGKILL'); + } catch (error) { + console.warn('[ComposeService] Failed to force terminate compose command:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + } + }, 5000); + }; + + // Idle stall backstop. Armed once below and reset on every output chunk; + // if it ever fires, the step has been silent for idleTimeoutMs while still + // running, so terminate it. Never rearmed after a termination is pending or + // the child has exited, so it cannot re-fire during the SIGTERM grace. + const armIdleTimeout = () => { + if (idleTimeoutMs === undefined) return; + if (exited || settled || pendingTerminationError) return; + if (idleTimeout) clearTimeout(idleTimeout); + idleTimeout = setTimeout(() => { + const seconds = Math.round(idleTimeoutMs / 1000); + sendOutput(`=== No output for ${seconds}s; the operation appears stalled and was stopped ===\n`); + terminateChild(new Error(`STACK_STALLED_OUTPUT: no output for ${seconds}s`)); + }, idleTimeoutMs); + }; + + // The progress socket is output-only: a deploy/update/down is owned by the + // HTTP request that started it, so closing or losing the socket (the user + // minimizes the panel, navigates away, or the connection blips) must not + // terminate the compose process. Termination is driven solely by the + // command timeout here and the optional idle stall backstop above. + timeout = setTimeout(() => { + const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`; + sendOutput(`${message}\n`); + terminateChild(new Error(message)); + }, timeoutMs); + + armIdleTimeout(); + + const onData = (data: Buffer) => { + const text = data.toString(); + errorLog += text; + sendOutput(text); + armIdleTimeout(); + }; + + child.stdout.on('data', onData); + child.stderr.on('data', onData); + + child.on('close', (code: number | null) => { + exited = true; + finish(() => { + sendOutput(`Command exited with code ${code}\n`); + if (pendingTerminationError) { + if (throwOnError) reject(pendingTerminationError); + else resolve(); + return; + } + if (code === 0) resolve(); + else if (throwOnError) reject(new Error(redactSensitiveText(errorLog.trim()) || `Command failed with code ${code}`)); + else resolve(); + }); + }); + + child.on('error', (error: Error & { code?: string }) => { + exited = true; + finish(() => { + const mapped = describeSpawnError(error as NodeJS.ErrnoException, { command }); + const message = redactSensitiveText(mapped.message); + sendOutput(`Error: ${message}\n`); + if (mapped.isLowMemory) { + console.warn('[ComposeService] spawn failed under memory pressure:', message); + } + if (pendingTerminationError) { + if (throwOnError) reject(pendingTerminationError); + else resolve(); + return; + } + if (throwOnError) reject(new Error(message)); + else resolve(); + }); + }); + }); + } + + private async withRegistryAuth( + fn: (env: Record) => Promise, + sendOutput?: (data: string) => void, + ): Promise { + const registries = DatabaseService.getInstance().getRegistries(); + if (registries.length === 0) { + return fn({ + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }); + } + + const { config, warnings } = await RegistryService.getInstance().resolveDockerConfig(); + if (warnings.length > 0 && sendOutput) { + for (const warning of warnings) { + sendOutput(`[Sencho] Warning: ${warning}\n`); + } + } + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-docker-')); + const configPath = path.join(tmpDir, 'config.json'); + + try { + fs.writeFileSync(configPath, JSON.stringify(config), { mode: 0o600 }); + return await fn({ + ...process.env, + DOCKER_CONFIG: tmpDir, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }); + } finally { + // Best-effort cleanup; each step runs independently so a file that was never + // written (e.g., writeFileSync threw) does not prevent the directory removal. + try { fs.unlinkSync(configPath); } catch { /* file may not exist */ } + try { fs.rmdirSync(tmpDir); } catch (e) { + console.warn('[ComposeService] Could not remove temp Docker config dir:', (e as Error).message); + } + } + } + + private async createAtomicBackup( + stackName: string, + operation: 'deployment' | 'update', + sendOutput: (data: string) => void, + ): Promise { + try { + const fsSvc = FileSystemService.getInstance(this.nodeId); + await fsSvc.backupStackFiles(stackName); + sendOutput(`=== Backup created for atomic ${operation} ===\n`); + } catch (error) { + console.error('Atomic backup failed for %s:', sanitizeForLog(stackName), getErrorMessage(error, 'unknown error')); + sendOutput(`=== Atomic ${operation} backup failed. Operation aborted ===\n`); + throw new Error(`Atomic ${operation} backup failed: ${getErrorMessage(error, 'unknown error')}`); + } + } + + private async restoreAtomicBackup( + stackName: string, + stackDir: string, + ws: WebSocket | undefined, + sendOutput: (data: string) => void, + ): Promise { + try { + const fsSvc = FileSystemService.getInstance(this.nodeId); + await fsSvc.restoreStackFiles(stackName); + await this.withRegistryAuth(async (env) => { + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env); + }, sendOutput); + sendOutput('=== Restored previous compose and env files ===\n'); + return true; + } catch (rollbackError) { + console.error('Rollback failed for %s:', sanitizeForLog(stackName), getErrorMessage(rollbackError, 'unknown error')); + sendOutput('=== Rollback failed. Manual intervention may be required ===\n'); + return false; + } + } + + private createContainerCrashError(exitCode: number): Error { + return new Error( + `CONTAINER_CRASHED\nExit Code: ${exitCode}\nContainer exited after deployment. Check container logs for details.` + ); + } + + async runCommand(stackName: string, action: 'down' | 'start' | 'stop' | 'restart', ws?: WebSocket): Promise { + const stackDir = path.join(this.baseDir, stackName); + await this.execute('docker', await this.authoredComposeArgs(stackName, [action]), stackDir, ws); + } + + /** Interactive compose down (Take down UI / POST /down). Plain `down` by default. */ + async runDown(stackName: string, options?: { removeVolumes?: boolean }, ws?: WebSocket): Promise { + const stackDir = path.join(this.baseDir, stackName); + const args = options?.removeVolumes ? ['down', '--volumes'] : ['down']; + await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackDir, ws); + } + + /** + * Opt-in guard: when `env_block_deploy_on_missing_required` is enabled, refuse a + * deploy whose required `${VAR:?err}` variables are unset OR empty, before any + * backup, cleanup, pull, or `up` runs. Compose's own resolution is authoritative + * (it passes process.env), and on the failing path it emits no rendered model, so + * no env value is materialized. Default off and any settings-read failure both + * fall through without blocking. + */ + private async assertRequiredEnvPresent(stackName: string): Promise { + let enabled = false; + try { + enabled = DatabaseService.getInstance().getGlobalSettings()['env_block_deploy_on_missing_required'] === '1'; + } catch { + return; // safe default: a settings-read failure never blocks a deploy + } + if (!enabled) return; + const result = await this.renderConfig(stackName); + const missing = parseMissingRequiredVars(result.stderr); + if (missing.length === 0) return; + const plural = missing.length > 1; + throw new Error( + `Deploy blocked: required environment variable${plural ? 's' : ''} ${missing.join(', ')} ` + + `${plural ? 'are' : 'is'} missing. Define ${plural ? 'them' : 'it'} in a .env or env_file, then deploy again.`, + ); + } + + private async assertSafePilotBindMapping(stackName: string): Promise { + if (process.env.SENCHO_MODE !== 'pilot') return; + + let mounts: Array<{ source: string; destination: string }> | null; + try { + mounts = await SelfIdentityService.getInstance().getBindMounts(); + } catch (error) { + console.warn('[ComposeService] Could not verify pilot compose path mapping:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + return; + } + if (mounts === null) return; + + const composeDir = path.resolve(this.baseDir); + const hostComposeDir = resolveHostBindPath(composeDir, mounts); + if (!hostComposeDir || pathsMatch(hostComposeDir, composeDir)) return; + + const rendered = await this.renderConfig(stackName); + if (rendered.rendered === null) return; + + let parsed: unknown; + try { + parsed = JSON.parse(rendered.rendered); + } catch (error) { + console.warn('[ComposeService] Could not inspect rendered binds for pilot path safety:', sanitizeForLog(getErrorMessage(error, 'unknown'))); + return; + } + const model = parseEffectiveModel(parsed, stackName); + const unsafeBind = model.services + .flatMap((service) => service.binds) + .find((bind) => isPathWithinBase(path.resolve(bind.source), composeDir)); + if (!unsafeBind) return; + + throw new Error( + `Deploy blocked: relative bind mounts resolve under ${composeDir}, but the host path is ${hostComposeDir}. ` + + `Use a 1:1 mount with the same absolute path on the host and in the Pilot Agent, then retry.`, + ); + } + + /** + * Missing-external gate: after env/Pilot asserts, before atomic backup. + * Creates safe bridge networks only when the opt-in setting is on. + */ + private async ensureExternalNetworksForDeploy( + stackName: string, + ctx?: DeployInvocationContext, + ): Promise { + const resolved = await resolveMissingExternalNetworks(this.nodeId, stackName); + if (resolved.status === 'render_unavailable') { + throw new MissingExternalNetworksError({ + kind: 'unavailable', + message: 'Sencho could not render this stack\'s Compose model to check external networks.', + }); + } + if (resolved.status === 'runtime_unavailable') { + // No declared externals: nothing to verify; proceed. + if (resolved.declaredExternalCount === 0) return; + throw new MissingExternalNetworksError({ + kind: 'unavailable', + message: 'Sencho could not read Docker networking state to check external networks.', + }); + } + + if (resolved.networks.length === 0) return; + + const unsafe = resolved.networks.filter((n) => !n.safe); + if (unsafe.length > 0) { + throw new MissingExternalNetworksError({ + kind: 'unsupported', + message: 'One or more missing external networks cannot be created safely by Sencho.', + networks: resolved.networks, + }); + } + + if (!resolved.autoCreateEnabled) { + throw new MissingExternalNetworksError({ + kind: 'prompt', + message: 'One or more external networks required by this stack are missing on this node.', + networks: resolved.networks, + }); + } + + const docker = DockerController.getInstance(this.nodeId); + const createdNames: string[] = []; + const recordCreatedNetworks = (level: 'info' | 'warning') => { + if (createdNames.length === 0) return; + invalidateNodeCaches(this.nodeId); + recordNetworkAutoCreatedActivity(this.nodeId, stackName, createdNames, level, ctx); + }; + + for (const network of resolved.networks) { + try { + await docker.createNetwork({ Name: network.name, Driver: 'bridge' }); + createdNames.push(network.name); + } catch (createErr) { + // Authoritative re-check: continue only if the network now exists. + let exists = false; + try { + const knownStacks = await FileSystemService.getInstance(this.nodeId).getStacks(); + const snapshot = await docker.getDependencySnapshot(knownStacks); + exists = snapshot.networks.some((n) => n.name === network.name); + } catch (snapErr) { + console.warn( + '[ComposeService] Post-create snapshot failed for %s:', + sanitizeForLog(network.name), + sanitizeForLog(getErrorMessage(snapErr, 'unknown')), + ); + } + if (!exists) { + recordCreatedNetworks('warning'); + throw new MissingExternalNetworksError({ + kind: 'create_failed', + message: `Failed to create external network "${network.name}".`, + networks: resolved.networks, + createdNames, + remainingNames: resolved.networks + .map((missingNetwork) => missingNetwork.name) + .filter((name) => !createdNames.includes(name)), + }); + } + // Race-existing: do not record in createdNames. + } + } + + // Re-resolve before Compose. + const recheck = await resolveMissingExternalNetworks(this.nodeId, stackName); + if (recheck.status !== 'ok' || recheck.networks.length > 0) { + recordCreatedNetworks('warning'); + throw new MissingExternalNetworksError({ + kind: recheck.status === 'ok' ? 'create_failed' : 'unavailable', + message: 'External networks were still missing after automatic creation.', + networks: recheck.networks, + createdNames, + remainingNames: recheck.networks.map((n) => n.name), + }); + } + + recordCreatedNetworks('info'); + } + + async deployStack( + stackName: string, + ws?: WebSocket, + atomic?: boolean, + ctx?: DeployInvocationContext, + ): Promise<{ recoveryId: string | null }> { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + await this.ensureExternalNetworksForDeploy(stackName, ctx); + + const stackDir = path.join(this.baseDir, stackName); + const debug = isDebugEnabled(); + const t0 = Date.now(); + if (debug) console.debug('[ComposeService:debug] deployStack', { stackName, stackDir, atomic }); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + const recoverySvc = atomic ? StackUpdateRecoveryService.getInstance() : null; + let recoveryId: string | null = null; + let handedOff = false; + + if (atomic && recoverySvc) { + sendOutput('=== Capturing rollback generation for atomic deploy ===\n'); + const candidate = await recoverySvc.captureCandidate({ + nodeId: this.nodeId, + stackName, + createdBy: 'atomic-deploy', + operationKind: 'deployment', + }); + recoveryId = candidate.id; + if (!recoverySvc.markAcquired(candidate.id)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to mark recovery generation as acquired'); + } + if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to hand off recovery generation'); + } + handedOff = true; + if (!recoverySvc.markReconciling(candidate.id)) { + throw new Error('Failed to mark recovery generation as reconciling after handoff'); + } + } + + try { + try { + const dockerController = DockerController.getInstance(this.nodeId); + const legacyOrphans = await dockerController.getLegacyOrphanContainersByStack(stackName); + if (legacyOrphans.length > 0) { + sendOutput(`=== Cleaning up legacy orphan containers before deployment ===\n`); + await dockerController.removeContainers(legacyOrphans.map((c) => c.Id)); + } + } catch (e) { + console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e); + } + + await this.withRegistryAuth(async (env) => { + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + + // Post-Deploy Health Probe + await new Promise(resolve => setTimeout(resolve, 3000)); + + const dockerController = DockerController.getInstance(this.nodeId); + const containers = await dockerController.getDocker().listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${stackName}`] } + }); + + for (const containerInfo of containers) { + if (containerInfo.State === 'exited') { + const container = dockerController.getDocker().getContainer(containerInfo.Id); + const inspectData = await container.inspect(); + const exitCode = inspectData.State.ExitCode; + + if (exitCode !== 0) { + throw this.createContainerCrashError(exitCode); + } + } + } + + if (atomic && recoverySvc && recoveryId) { + if (!recoverySvc.markImmediateVerified(recoveryId)) { + console.warn( + '[ComposeService] Could not CAS immediate_verified for recovery %s', + sanitizeForLog(recoveryId), + ); + } + } + if (debug) console.debug(`[ComposeService:debug] deployStack completed in ${Date.now() - t0}ms`, { stackName }); + } catch (deployError) { + if (atomic && recoverySvc && handedOff && recoveryId) { + sendOutput('\n=== Deployment failed - restoring previous runtime from recovery generation ===\n'); + const generationId = recoveryId; + const rolledBack = await compensateOrSwallow(() => + recoverySvc.compensateWithCandidate( + generationId, + (overridePath, invocation) => this.composeUpWithRecoveryOverride( + stackName, + overridePath, + ws, + invocation, + ), + ), + ); + throw new ComposeRollbackError(deployError, true, rolledBack); + } + if (atomic && recoverySvc && recoveryId && !handedOff) { + await recoverySvc.abandon(recoveryId); + } + throw deployError; + } + // Reached only on a successful deploy (the catch above always rethrows). Record + // the drift baseline here so every deploy path gets one, not just the manual + // route: bulk, Git-source, App Store, scheduler, and webhook deploys all funnel + // through this method. Internally guarded; awaited so it cannot race later work. + await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); + // Reconcile the ledger against the just-deployed runtime: findings this deploy + // fixed are resolved and any it left are recorded (and surfaced in the activity + // feed) now, instead of waiting for someone to open the Drift tab. The rollback + // route re-deploys through this method, so it is covered; a failed atomic deploy + // instead restores the previous files and throws above, so that recovery path + // reconciles on its next deploy or scan, not here. Best-effort internally. + await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); + // Refresh the exposure cache so posture reflects the just-deployed model. + // Best-effort: a refresh failure logs a warning but never fails the deploy. + try { + await this.refreshExposureCache(stackName); + } catch (err) { + console.warn('[ComposeService] Exposure refresh failed after deploy for %s:', + sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown'))); + } + return { recoveryId }; + } + + streamLogs(stackName: string, ws: WebSocket) { + let isClosed = false; + let isFirstRun = true; + let isWaitingForActivity = false; + + ws.on('close', () => { isClosed = true; }); + + const startStream = async () => { + if (isClosed || ws.readyState !== WebSocket.OPEN) return; + + try { + const dockerController = DockerController.getInstance(this.nodeId); + const containers = await dockerController.getContainersByStack(stackName); + + if (!containers || containers.length === 0) { + if (!isWaitingForActivity) { + ws.send(`\r\n\x1b[33m[Sencho] No containers found. Waiting for activity...\x1b[0m\r\n`); + isWaitingForActivity = true; + } + setTimeout(startStream, 2000); + return; + } + + const runningContainers = containers.filter((c: any) => c.State === 'running'); + + if (!isFirstRun && runningContainers.length === 0) { + if (!isWaitingForActivity) { + ws.send(`\r\n\x1b[33m[Sencho] Log stream ended. Waiting for container activity...\x1b[0m\r\n`); + isWaitingForActivity = true; + } + setTimeout(startStream, 2000); + return; + } + + const containersToLog = isFirstRun ? containers : runningContainers; + isFirstRun = false; + isWaitingForActivity = false; + + let activeProcesses = 0; + let streamEndedHandled = false; + const localProcesses: ReturnType[] = []; + + const onWsClose = () => { + localProcesses.forEach(cp => { try { cp.kill(); } catch { } }); + }; + + ws.on('close', onWsClose); + + const handleProcessEnd = () => { + activeProcesses--; + if (activeProcesses <= 0 && !streamEndedHandled) { + streamEndedHandled = true; + ws.removeListener('close', onWsClose); + if (!isClosed && ws.readyState === WebSocket.OPEN) { + setTimeout(startStream, 1000); + } + } + }; + + for (const container of containersToLog) { + const rawName = container.Names?.[0]?.replace(/^\//, '') || container.Id; + const displayName = normalizeContainerName(rawName, stackName); + activeProcesses++; + let lineBuffer = ''; + + const sendOutput = (data: Buffer) => { + if (ws.readyState === WebSocket.OPEN) { + lineBuffer += data.toString(); + const lines = lineBuffer.split(/\r?\n/); + lineBuffer = lines.pop() || ''; + for (const line of lines) { + ws.send(LogFormatter.process(`${displayName} | ${line}`) + '\r\n'); + } + } + }; + + const flushBuffer = () => { + if (lineBuffer && ws.readyState === WebSocket.OPEN) { + ws.send(LogFormatter.process(`${displayName} | ${lineBuffer}`) + '\r\n'); + lineBuffer = ''; + } + }; + + const child = spawn('docker', ['logs', '-f', '-t', '--tail', '100', rawName], { + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + } + }); + localProcesses.push(child); + child.stdout.on('data', sendOutput); + child.stderr.on('data', sendOutput); + child.on('error', handleProcessEnd); + child.on('close', () => { + flushBuffer(); + handleProcessEnd(); + }); + } + } catch (err) { + if (!isClosed && ws.readyState === WebSocket.OPEN) { + if (!isWaitingForActivity) { + ws.send(`\r\n\x1b[31m[Sencho] Error tracking containers. Retrying...\x1b[0m\r\n`); + isWaitingForActivity = true; + } + setTimeout(startStream, 2000); + } + } + }; + + startStream(); + } + + + /** + * Pinned compose-up with a recovery override layered last + * (`--pull never --no-build`). Used by manual rollback and deploy/update + * compensation. When `invocation` is set, Compose args come from the + * generation capture instead of the live database-derived invocation. + */ + async composeUpWithRecoveryOverride( + stackName: string, + overridePath: string, + ws?: WebSocket, + invocation?: RollbackInvocationRecord | null, + ): Promise { + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + await this.execute( + 'docker', + await this.buildComposeArgsWithRecoveryOverride( + stackName, + ['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'], + overridePath, + invocation ?? null, + ), + stackDir, + ws, + true, + env, + getComposeStallTimeoutMs(), + ); + }, sendOutput); + } + + /** + * Authored (+ mesh) compose args with an optional recovery override layered LAST. + * When `invocation` is provided, use its validated captured prefix so recovery + * does not mix restored files with the live deploy-spec / project-env selection. + */ + public async buildComposeArgsWithRecoveryOverride( + stackName: string, + action: string[], + recoveryOverridePath: string | null, + invocation?: RollbackInvocationRecord | null, + ): Promise { + const useCaptured = hasUsableCapturedInvocation(invocation); + const out = useCaptured + ? ['compose', ...this.composePrefixFromCapturedInvocation(stackName, invocation)] + : await this.authoredComposeArgsPrefix(stackName); + + if (useCaptured && invocation.meshEnabled) { + await this.appendCapturedMeshLayer(stackName, out); + } + + if (recoveryOverridePath) { + // Captured invocations already encode their file set; live args may need + // an explicit base (+ user override) so docker compose accepts a trailing -f. + await this.ensureExplicitComposeFiles(stackName, out, !useCaptured); + out.push('-f', recoveryOverridePath); + } + out.push(...action); + return out; + } + + /** + * Re-apply the Mesh override layer when the generation captured Mesh as enabled. + * Uses live ensureStackOverride (same absolute override path semantics as deploy). + */ + private async appendCapturedMeshLayer(stackName: string, args: string[]): Promise { + const overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); + if (!overridePath) { + throw new Error( + `Captured Mesh-enabled invocation cannot regenerate mesh override for stack "${stackName}"`, + ); + } + await this.ensureExplicitComposeFiles(stackName, args, true); + args.push('-f', overridePath); + } + + /** Slice the global-flag prefix from authoredComposeArgs (no action tokens). */ + private async authoredComposeArgsPrefix(stackName: string): Promise { + const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']); + const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__'); + const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel; + return [...prefix]; + } + + /** + * When args lack `-f`, pin the stack compose file (and optionally the user + * override) so a trailing override layer is accepted by docker compose. + */ + private async ensureExplicitComposeFiles( + stackName: string, + args: string[], + includeUserOverride: boolean, + ): Promise { + if (args.includes('-f')) return; + const fsSvc = FileSystemService.getInstance(this.nodeId); + args.push('-f', await fsSvc.getComposeFilename(stackName)); + if (!includeUserOverride) return; + try { + const userOverride = await fsSvc.getOverrideFilename(stackName); + if (userOverride) args.push('-f', userOverride); + } catch (err) { + const code = (err as { code?: string }).code; + if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { + throw err; + } + console.warn( + '[ComposeService] could not resolve user compose override while pinning compose files:', + sanitizeForLog((err as Error).message), + ); + } + } + + private resolveValidatedStackDir(stackName: string): string { + const stackDir = path.resolve(this.baseDir, stackName); + if (!isPathWithinBase(stackDir, this.baseDir) || path.resolve(this.baseDir) === stackDir) { + throw new Error('Invalid stack path'); + } + return stackDir; + } + + /** + * Rebuild a spawn-safe compose global-flag prefix from a generation's + * captured invocation. Relative -f / --project-directory paths and absolute + * --env-file paths must stay inside the stack directory. + */ + private composePrefixFromCapturedInvocation( + stackName: string, + invocation: RollbackInvocationRecord, + ): string[] { + const stackDir = this.resolveValidatedStackDir(stackName); + // Empty prefix is valid (single-file auto-discovery at capture time). + const raw = [...invocation.composeArgsPrefix]; + const out: string[] = []; + for (let i = 0; i < raw.length; i++) { + const token = raw[i]; + if (token === '-f' || token === '--file') { + const file = raw[++i]; + if (!file || !isValidRelativeStackPath(file)) { + throw new Error(`Invalid captured compose file path for stack "${stackName}"`); + } + if (!isPathWithinBase(path.resolve(stackDir, file), stackDir)) { + throw new Error(`Captured compose file path escapes stack directory for "${stackName}"`); + } + out.push('-f', file); + continue; + } + if (token === '--env-file') { + const envPath = raw[++i]; + if (!envPath) { + throw new Error(`Invalid captured --env-file for stack "${stackName}"`); + } + const abs = path.resolve(envPath); + if (!isPathWithinBase(abs, stackDir)) { + throw new Error(`Captured env-file path escapes stack directory for "${stackName}"`); + } + out.push('--env-file', abs); + continue; + } + if (token === '--project-directory') { + const dir = raw[++i]; + if (!dir) { + throw new Error(`Invalid captured --project-directory for stack "${stackName}"`); + } + const abs = path.isAbsolute(dir) ? path.resolve(dir) : path.resolve(stackDir, dir); + if (!isPathWithinBase(abs, stackDir)) { + throw new Error(`Captured project-directory escapes stack directory for "${stackName}"`); + } + out.push('--project-directory', abs); + continue; + } + if (token === '-p' || token === '--project-name') { + const name = raw[++i]; + if (!name || name !== (invocation.projectName || stackName)) { + throw new Error(`Captured project name mismatch for stack "${stackName}"`); + } + out.push('-p', stackName); + continue; + } + throw new Error(`Unsupported captured compose flag "${token}" for stack "${stackName}"`); + } + return out; + } + + async updateStack( + stackName: string, + ws?: WebSocket, + atomic?: boolean, + ): Promise<{ recoveryId: string | null }> { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const debug = isDebugEnabled(); + const t0 = Date.now(); + if (debug) console.debug('[ComposeService:debug] updateStack', { stackName, stackDir, atomic }); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + + // Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs). + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + const recoverySvc = StackUpdateRecoveryService.getInstance(); + let recoveryId: string | null = null; + let handedOff = false; + + try { + sendOutput('=== Validating stack for update ===\n'); + sendOutput('=== Capturing rollback generation ===\n'); + const candidate = await recoverySvc.captureCandidate({ + nodeId: this.nodeId, + stackName, + createdBy: null, + operationKind: 'update', + }); + recoveryId = candidate.id; + + const buildServices = await loadStackBuildServices(this.nodeId, stackName); + const buildAware = buildServices.length > 0; + + try { + await this.withRegistryAuth(async (env) => { + if (buildAware) { + sendOutput('=== Building images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['build', '--pull']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + sendOutput('=== Pulling registry images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + } else { + sendOutput('=== Pulling latest images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['pull']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + } + }, sendOutput); + } catch (acquireError) { + // Acquisition failure: abandon candidate; leave runtime untouched. + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw acquireError; + } + + if (!recoverySvc.markAcquired(candidate.id)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to mark recovery generation as acquired'); + } + + const dockerController = DockerController.getInstance(this.nodeId); + sendOutput('=== Classifying legacy orphans ===\n'); + const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName); + if (classified.status === 'classification_failed') { + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw new Error(`Legacy orphan classification failed: ${classified.error}`); + } + + if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw new Error('Failed to hand off recovery generation'); + } + handedOff = true; + if (!recoverySvc.markReconciling(candidate.id)) { + throw new Error('Failed to mark recovery generation as reconciling after handoff'); + } + + if (classified.status === 'orphans') { + sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`); + const results = await dockerController.removeContainers(classified.ids); + const failed = results.filter((r) => !r.success); + if (failed.length > 0) { + throw new Error( + `Failed to remove ${failed.length} legacy orphan container(s) after handoff`, + ); + } + } + + await this.withRegistryAuth(async (env) => { + sendOutput('=== Recreating containers ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + }, sendOutput); + + // Immediate verification probe + await new Promise((resolve) => setTimeout(resolve, 3000)); + + const containers = await dockerController.getDocker().listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${stackName}`] }, + }); + + for (const containerInfo of containers) { + if (containerInfo.State === 'exited') { + const container = dockerController.getDocker().getContainer(containerInfo.Id); + const inspectData = await container.inspect(); + const exitCode = inspectData.State.ExitCode; + if (exitCode !== 0) { + throw this.createContainerCrashError(exitCode); + } + } + } + + if (!recoverySvc.markImmediateVerified(candidate.id)) { + console.warn( + '[ComposeService] Could not CAS immediate_verified for recovery %s', + sanitizeForLog(candidate.id), + ); + } + + sendOutput('=== Stack updated successfully ===\n'); + + // Defer prune until gate retention / gate link: only prune when no active holds + // would be violated. Still honor prune_on_update, but use unified holds so + // candidate/current rollback images are retained. + try { + const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'; + if (pruneOnUpdate) { + const isImageHeld = buildUnifiedHeldImagePredicate(this.nodeId); + const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld); + const reclaimed = result.reclaimedBytes > 0 + ? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB` + : ''; + sendOutput(`=== Pruned dangling images${reclaimed} ===\n`); + } + } catch (pruneError) { + console.warn( + 'Failed to prune dangling images after update for %s:', + sanitizeForLog(stackName), + pruneError, + ); + } + + if (debug) { + console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName }); + } + } catch (updateError) { + if (!handedOff && recoveryId) { + await recoverySvc.abandon(recoveryId); + recoveryId = null; + } + if (handedOff && recoveryId) { + sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n'); + const generationId = recoveryId; + const rolledBack = await compensateOrSwallow(() => + recoverySvc.compensateWithCandidate( + generationId, + (overridePath, invocation) => this.composeUpWithRecoveryOverride( + stackName, + overridePath, + ws, + invocation, + ), + ), + ); + throw new ComposeRollbackError(updateError, true, rolledBack); + } + // Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched. + throw updateError; + } + + await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); + await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); + try { + await this.refreshExposureCache(stackName); + } catch (err) { + console.warn( + '[ComposeService] Exposure refresh failed after update for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(err, 'unknown')), + ); + } + return { recoveryId }; + } + + /** + * Service-scoped update: pull (or `build --pull` for a build-backed service) + * and recreate a single service's replicas in place. Always + * `--no-deps --force-recreate`, never `--remove-orphans`, so sibling services + * keep their container ids and StartedAt. No drift re-baseline and no dangling + * prune here; the orchestrator owns per-service post-update reconciliation. + */ + async updateService(stackName: string, serviceName: string, hasBuild: boolean, ws?: WebSocket): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + if (hasBuild) { + sendOutput(`=== Building ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + } else { + sendOutput(`=== Pulling ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + } + sendOutput(`=== Recreating ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + } + + /** + * Recreate a single service from the image already present locally, without + * pulling or building. Used by service restore after the recovery image id has + * been retagged onto the declared ref (`--pull never --no-build` so Compose + * uses the just-retagged local image). Always `--no-deps --force-recreate`, + * never `--remove-orphans`. + */ + async recreateServiceFromLocal(stackName: string, serviceName: string, ws?: WebSocket): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + const stackDir = path.join(this.baseDir, stackName); + const sendOutput = (data: string) => { + if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); + }; + await this.withRegistryAuth(async (env) => { + sendOutput(`=== Restoring ${serviceName} ===\n`); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--no-deps', '--force-recreate', '--pull', 'never', '--no-build', serviceName]), stackDir, ws, true, env, getComposeStallTimeoutMs()); + }, sendOutput); + } + + public async downStack(stackName: string, options?: { removeVolumes?: boolean }): Promise { + const stackPath = path.join(this.baseDir, stackName); + try { + const args = options?.removeVolumes + ? ['down', '--volumes', '--remove-orphans'] + : ['down', '--remove-orphans']; + await this.execute('docker', await this.authoredComposeArgs(stackName, args), stackPath, undefined, false); + } catch (error) { + console.warn(`[Teardown] Docker down failed or nothing to clean up for ${sanitizeForLog(stackName)}`); + } + } + + /** + * Enumerate image references declared in a stack's compose file. + * + * Used by the pre-deploy policy gate to decide which images to scan before + * `docker compose up` runs. Path traversal is guarded against the node's + * compose base directory; missing / unreadable compose files or `.env` + * interpolation failures surface as a rejected Promise so the gate can + * block the deploy rather than silently allow it. + */ + public async listStackImages( + stackName: string, + invocation?: RollbackInvocationRecord | null, + ): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const stackDir = this.resolveValidatedStackDir(stackName); + // Prefer a generation-captured invocation so restored-target policy does not + // scan with the live deploy-spec / project-env selection. + const useCaptured = hasUsableCapturedInvocation(invocation); + const fileAndEnvPrefix = useCaptured + ? [...this.composePrefixFromCapturedInvocation(stackName, invocation)] + : [ + ...authoredComposeFileArgs(stackName, this.nodeId), + ...(await authoredComposeEnvFileArgs(stackName, this.nodeId)), + ]; + if (useCaptured && invocation.meshEnabled) { + await this.appendCapturedMeshLayer(stackName, fileAndEnvPrefix); + } + const stdout = await this.captureCompose([...fileAndEnvPrefix, 'config', '--images'], stackDir); + const seen = new Set(); + const images: string[] = []; + for (const raw of stdout.split(/\r?\n/)) { + const line = raw.trim(); + if (!line) continue; + if (line.startsWith('sha256:')) continue; + if (seen.has(line)) continue; + seen.add(line); + images.push(line); + } + return images; + } + + /** Render the effective Compose model and cache the per-stack exposure + * descriptor so the Security posture can join exposed images against + * vulnerability findings without re-rendering config on every poll. + * Best-effort: render or parse failure logs a warning and keeps the + * prior cached descriptor, never failing the deploy. */ + private async refreshExposureCache(stackName: string): Promise { + const result = await this.renderConfig(stackName); + if (result.rendered === null) { + console.warn('[ComposeService] Exposure cache skipped for %s: model not renderable', + sanitizeForLog(stackName)); + return; + } + let parsed: unknown; + try { + parsed = JSON.parse(result.rendered); + } catch { + console.warn('[ComposeService] Exposure cache skipped for %s: unparseable model JSON', + sanitizeForLog(stackName)); + return; + } + const model = parseEffectiveModel(parsed, stackName); + const descriptor = deriveStackExposure(model, stackName, Date.now()); + DatabaseService.getInstance().upsertStackExposure( + this.nodeId, + stackName, + JSON.stringify(descriptor), + descriptor.computedAt, + ); + } + + private captureCompose(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn('docker', ['compose', ...args], { + cwd, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }, + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (data: Buffer) => { stdout += data.toString(); }); + child.stderr.on('data', (data: Buffer) => { stderr += data.toString(); }); + child.on('error', (err: NodeJS.ErrnoException) => { + const mapped = describeSpawnError(err, { command: 'docker compose' }); + if (mapped.isLowMemory) { + console.warn('[ComposeService] captureCompose spawn failed under memory pressure:', mapped.message); + } + reject(new Error(mapped.message)); + }); + child.on('close', (code) => { + if (code === 0) resolve(stdout); + else reject(new Error(stderr.trim() || `docker compose ${args.join(' ')} failed with code ${code}`)); + }); + }); + } + + /** + * Render the effective compose model as YAML (the default `docker compose + * config` output) with the exact authored invocation and NO mesh override. + * Used by the Git source detach/export contract: the rendered model becomes + * the stack's single compose.yaml. Throws when the render fails or times + * out, so the detach transaction aborts before anything changes. + */ + public async renderComposeYaml(stackName: string): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + let filePrefix: string[]; + try { + filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + } catch (err) { + throw err instanceof Error ? err : new Error(String(err)); + } + const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); + const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config'], { + cwd: stackDir, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }, + }); + return new Promise((resolve, reject) => { + const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream + const TIMEOUT_MS = 30_000; + // Accumulate Buffer chunks and decode ONCE at the end: chunk-wise + // toString() can split a multi-byte UTF-8 sequence across a chunk + // boundary and mangle non-ASCII values. + const outChunks: Buffer[] = []; + let outBytes = 0; + let stderr = ''; + let capped = false; + let settled = false; + const timer = setTimeout(() => { + settled = true; + clearTimeout(timer); + try { + child.kill('SIGKILL'); + } catch { + // best effort + } + reject(new Error(`docker compose config timed out after ${TIMEOUT_MS / 1000}s`)); + }, TIMEOUT_MS); + const finish = (error: Error | null) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) reject(error); + else resolve(Buffer.concat(outChunks).toString('utf8')); + }; + child.stdout.on('data', (data: Buffer) => { + if (capped) return; + outBytes += data.length; + if (outBytes > MAX_OUTPUT) { + // A truncated model frequently still parses as YAML; overwriting a + // working compose.yaml with it would be silent corruption. The cap is + // an error, not a truncation. + capped = true; + settled = true; + clearTimeout(timer); + try { + child.kill('SIGKILL'); + } catch { + // best effort + } + reject(new Error(`docker compose config output exceeded ${MAX_OUTPUT} bytes`)); + return; + } + outChunks.push(data); + }); + child.stderr.on('data', (data: Buffer) => { + if (stderr.length < MAX_OUTPUT) stderr += data.toString(); + }); + child.on('close', (code) => { + if (capped) return; + if (code === 0) finish(null); + else finish(new Error(stderr.trim() || `docker compose config exited with code ${code}`)); + }); + child.on('error', (err) => finish(err)); + }); + } + + /** + * Render the fully-resolved effective Compose model via `docker compose + * config --format json`. This is the AUTHORED model: it does NOT splice in + * the Sencho Mesh override, so it stays read-only (the override is + * write-generated) and reflects what the user actually edits. The override + * would also add the managed `sencho_mesh` external network and per-service + * mesh attachments, which would make preflight emit a false "external network + * not found" finding, so rendering the authored model is both safer and more + * accurate here. + * Captures stderr (where Compose reports unset variables) and never rejects + * on a non-zero exit, so the Compose Doctor can turn a failed render into a + * finding rather than an exception. Bounded by a timeout and an output cap. + * Rejects only when the docker binary cannot be spawned. + */ + public async renderConfig( + stackName: string, + ): Promise<{ rendered: string | null; stderr: string; code: number | null; timedOut: boolean }> { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + // Canonical inline js/path-injection barrier, kept in the same scope as the + // spawn cwd sink below. CodeQL credits neither the wrapped isPathWithinBase + // helper nor a barrier separated from the sink by the Promise-executor + // closure, so the spawn is hoisted out of the executor. startsWith already + // rejects the base dir itself, since base does not start with base + sep. + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + // Render the authored multi-file model (no mesh override) so the Compose Doctor + // sees every override file; single-file stacks get an empty prefix. The env-file + // flag keeps render resolving the same root .env the validator and deploy use. + let filePrefix: string[]; + try { + filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + } catch (err) { + throw err instanceof Error ? err : new Error(String(err)); + } + const envFileArgs = await authoredComposeEnvFileArgs(stackName, this.nodeId); + const child = spawn('docker', ['compose', ...filePrefix, ...envFileArgs, 'config', '--format', 'json'], { + cwd: stackDir, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + }, + }); + return new Promise((resolve, reject) => { + const MAX_OUTPUT = 5 * 1024 * 1024; // 5 MiB cap on each stream + const TIMEOUT_MS = 20_000; + let stdout = ''; + let stderr = ''; + let timedOut = false; + let capped = false; + let settled = false; + const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, TIMEOUT_MS); + const finish = (result: { rendered: string | null; stderr: string; code: number | null; timedOut: boolean }) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(result); + }; + child.stdout.on('data', (data: Buffer) => { + stdout += data.toString(); + if (stdout.length > MAX_OUTPUT && !capped) { capped = true; child.kill('SIGKILL'); } + }); + child.stderr.on('data', (data: Buffer) => { + if (stderr.length < MAX_OUTPUT) stderr += data.toString(); + }); + child.on('error', (err: NodeJS.ErrnoException) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(new Error(describeSpawnError(err, { command: 'docker compose' }).message)); + }); + child.on('close', (code) => { + if (timedOut) finish({ rendered: null, stderr: stderr.trim() || 'docker compose config timed out', code, timedOut: true }); + else if (capped) finish({ rendered: null, stderr: 'Rendered model exceeded the size limit', code, timedOut: false }); + else if (code === 0) finish({ rendered: stdout, stderr, code, timedOut: false }); + else finish({ rendered: null, stderr: stderr.trim() || `docker compose config failed with code ${code}`, code, timedOut: false }); + }); + }); + } +} diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 0c2998bb..e1f33524 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -18,8 +18,11 @@ import { import { readSnapshotFileRow, type SnapshotFileReadResult, type SnapshotFileRow } from '../helpers/snapshotFileDecrypt'; import { sanitizeForLog } from '../utils/safeLog'; import type { GitSourceManifestState } from '../types/gitProjectManifest'; +import type { RollbackOperationKind } from '../types/rollbackGeneration'; +import { collectImageIds, parseServicesJsonStrict } from './recoveryServicesJson'; export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt'; +export type { RollbackOperationKind } from '../types/rollbackGeneration'; function isPilotMode(): boolean { return process.env.SENCHO_MODE === 'pilot'; @@ -249,6 +252,10 @@ export interface StackUpdateRecoveryGenerationRow { phase: 'captured' | 'acquired' | 'handoff_committed' | 'reconciling' | 'immediate_verified'; is_current: number; backup_slot_id: string | null; + /** Generation content key (often equal to backup_slot_id / generation id). */ + content_path: string | null; + /** Capture trigger: update | deployment | git_apply | manual_backup | unknown. */ + operation_kind: RollbackOperationKind | null; override_path: string | null; services_json: string; health_gate_id: string | null; @@ -1920,6 +1927,9 @@ export class DatabaseService { // pattern used for health_gate_runs below. maybeAddCol('stack_update_recovery_generations', 'released_at', 'INTEGER'); maybeAddCol('stack_update_recovery_generations', 'released_by', 'TEXT'); + // Authored-project generation content key + capture trigger kind. + maybeAddCol('stack_update_recovery_generations', 'content_path', 'TEXT'); + maybeAddCol('stack_update_recovery_generations', 'operation_kind', 'TEXT'); maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER'); // Distributed API model columns @@ -4203,13 +4213,15 @@ export class DatabaseService { public insertStackUpdateRecoveryGeneration(row: StackUpdateRecoveryGenerationRow): void { this.db.prepare( `INSERT INTO stack_update_recovery_generations ( - id, node_id, stack_name, status, phase, is_current, backup_slot_id, override_path, - services_json, health_gate_id, gate_retain_until, artifact_expires_at, - operation_lease_expires_at, created_at, updated_at, created_by, artifacts_retired - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + id, node_id, stack_name, status, phase, is_current, backup_slot_id, content_path, + operation_kind, override_path, services_json, health_gate_id, gate_retain_until, + artifact_expires_at, operation_lease_expires_at, created_at, updated_at, + created_by, artifacts_retired + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current, - row.backup_slot_id, row.override_path, row.services_json, row.health_gate_id, + row.backup_slot_id, row.content_path ?? null, row.operation_kind ?? null, + row.override_path, row.services_json, row.health_gate_id, row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at, row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0, ); @@ -4241,7 +4253,8 @@ export class DatabaseService { id: string, patch: Partial>, + 'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json' | + 'content_path' | 'operation_kind'>>, ): void { const keys = Object.keys(patch) as Array; if (keys.length === 0) return; @@ -4432,26 +4445,13 @@ export class DatabaseService { ).all(nodeId, now, now) as Array<{ services_json: string }>; const ids = new Set(); for (const row of rows) { - try { - const parsed: unknown = JSON.parse(row.services_json); - if (!Array.isArray(parsed)) continue; - for (const item of parsed) { - if (!item || typeof item !== 'object') continue; - const replicas = (item as { replicas?: unknown }).replicas; - if (Array.isArray(replicas)) { - for (const replica of replicas) { - if (replica && typeof replica === 'object' - && typeof (replica as { imageId?: unknown }).imageId === 'string' - && (replica as { imageId: string }).imageId.trim()) { - ids.add((replica as { imageId: string }).imageId); - } - } - } else if (typeof (item as { imageId?: unknown }).imageId === 'string') { - ids.add((item as { imageId: string }).imageId); - } - } - } catch { - // Corrupt JSON: skip. + const parsed = parseServicesJsonStrict(row.services_json); + if (!parsed.ok) { + // Fail closed: corrupt hold metadata must not look like "nothing held". + throw new Error('Malformed stack recovery services_json while listing held images'); + } + for (const id of collectImageIds(parsed.services)) { + ids.add(id); } } return [...ids]; @@ -6323,6 +6323,25 @@ export class DatabaseService { ).run(commitSha, contentHash, Date.now(), stackName); } + /** + * Clear the last-applied revision identity without removing the Git source + * row. Used when compensating to a capture that had a null commit SHA + * (first-apply preimage). + */ + public clearGitSourceAppliedRevision(stackName: string): void { + this.db.prepare( + `UPDATE stack_git_sources SET + last_applied_commit_sha = NULL, + last_applied_content_hash = NULL, + pending_commit_sha = NULL, + pending_compose_content = NULL, + pending_env_content = NULL, + pending_fetched_at = NULL, + updated_at = ? + WHERE stack_name = ?` + ).run(Date.now(), stackName); + } + public touchGitSourceDebounce(stackName: string): void { this.db.prepare('UPDATE stack_git_sources SET last_debounce_at = ? WHERE stack_name = ?') .run(Date.now(), stackName); diff --git a/backend/src/services/DeployedStackDeletionService.ts b/backend/src/services/DeployedStackDeletionService.ts index 9da17242..af92d054 100644 --- a/backend/src/services/DeployedStackDeletionService.ts +++ b/backend/src/services/DeployedStackDeletionService.ts @@ -28,6 +28,7 @@ import { BLUEPRINT_MARKER_FILENAME, parseBlueprintMarker, } from '../helpers/blueprintMarker'; +import { scrapeRollbackTagsLenient } from './recoveryServicesJson'; /** * Directory that may contain recovery override files for a tombstone sweep. @@ -84,23 +85,7 @@ function collectArtifactsFromGenerations( for (const gen of generations) { if (gen.override_path) overridePaths.add(gen.override_path); - try { - const parsed: unknown = JSON.parse(gen.services_json); - if (!Array.isArray(parsed)) continue; - for (const svc of parsed) { - if (!svc || typeof svc !== 'object') continue; - const replicas = (svc as { replicas?: unknown }).replicas; - if (!Array.isArray(replicas)) continue; - for (const replica of replicas) { - const tag = replica && typeof replica === 'object' - ? (replica as { rollbackTag?: unknown }).rollbackTag - : null; - if (typeof tag === 'string' && tag.trim()) tags.add(tag); - } - } - } catch { - // Corrupt capture JSON: skip tags for this generation. - } + for (const tag of scrapeRollbackTagsLenient(gen.services_json)) tags.add(tag); } return { tags: [...tags], overridePaths: [...overridePaths] }; diff --git a/backend/src/services/GitProjectManifestService.ts b/backend/src/services/GitProjectManifestService.ts index 5a1dbe43..ba50ac94 100644 --- a/backend/src/services/GitProjectManifestService.ts +++ b/backend/src/services/GitProjectManifestService.ts @@ -24,6 +24,7 @@ import { StackFileRootsService } from './StackFileRootsService'; import { DatabaseService } from './DatabaseService'; import { ComposeInputDiscoveryService, type ContextCopyPlan, type CopyEntry } from './ComposeInputDiscoveryService'; import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs'; +import { collectManifestFilePaths } from '../helpers/manifestFilePaths'; import { sanitizeForLog } from '../utils/safeLog'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; import type { @@ -411,6 +412,34 @@ export class GitProjectManifestService { await fs.promises.rename(tmp, target); } + /** Raw on-disk manifesto JSON, or null when the file is absent. */ + async readRawManifestText(stackName: string): Promise { + // Inline barrier at the readFile sink (CodeQL path-injection). + const root = path.resolve(this.managedRoot(stackName)); + const target = path.resolve(root, MANIFEST_FILENAME); + if (!target.startsWith(root + path.sep)) { + throw Object.assign(new Error('Path escapes managed project directory'), { code: 'INVALID_PATH' }); + } + try { + return await fs.promises.readFile(target, 'utf8'); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw e; + } + } + + /** Remove the managed manifesto file (first-apply preimage restore). */ + async clearManifestFile(stackName: string): Promise { + // Inline barrier at the rm sink (CodeQL path-injection). + const root = path.resolve(this.managedRoot(stackName)); + const target = path.resolve(root, MANIFEST_FILENAME); + if (!target.startsWith(root + path.sep)) { + throw Object.assign(new Error('Path escapes managed project directory'), { code: 'INVALID_PATH' }); + } + await fs.promises.rm(target, { force: true }); + StackFileRootsService.invalidate(NodeRegistry.getInstance().getDefaultNodeId(), stackName); + } + /** * Public projection for the manifest read endpoint: hashes, size metadata, * provenance, and deletion authority are internal-only, and for @@ -700,19 +729,7 @@ export class GitProjectManifestService { /** Exact file paths owned by one manifest, excluding directory inventory entries. */ private manifestFilePaths(manifest: Pick): string[] { - const paths = new Map(); - for (const entry of manifest.inputs) { - if (entry.ownership !== 'managed' || entry.state !== 'present' || entry.materializedPath === null) continue; - if (entry.dependencyKind === 'build-context' || entry.dependencyKind === 'build-additional-context') continue; - paths.set(entry.materializedPath.toLowerCase(), entry.materializedPath); - } - for (const context of manifest.buildContexts) { - for (const file of context.files) { - const rel = context.repoPath ? `${context.repoPath}/${file.path}` : file.path; - paths.set(rel.toLowerCase(), rel); - } - } - return [...paths.values()].sort((a, b) => a.localeCompare(b)); + return collectManifestFilePaths(manifest); } /** Hash one snapshot file, preserving the distinction between missing and unreadable. */ diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index de35b626..137817b8 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -19,6 +19,7 @@ import { isPathWithinBase, isValidRelativeStackPath } from '../utils/validation' import { gitSourceLocalComposeFiles, PRIMARY_COMPOSE_FILENAME } from '../utils/gitComposeFiles'; import { ComposeInputDiscoveryService, type ContextCopyPlan } from './ComposeInputDiscoveryService'; import { GitProjectManifestService } from './GitProjectManifestService'; +import { StackUpdateRecoveryService } from './StackUpdateRecoveryService'; import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs'; import type { ComposeInputEntry, GitProjectManifest, GitSourceManifestState, InventoryResult, ManifestSummary, RefusalInfo } from '../types/gitProjectManifest'; import type { GitHttpRequest, GitHttpResponse, HttpClient } from 'isomorphic-git/http/node'; @@ -1618,11 +1619,7 @@ export class GitSourceService { for (const old of prevSpec.files) { if (old === PRIMARY_COMPOSE_FILENAME || keep.has(old)) continue; if (!isValidRelativeStackPath(old) || old === '') continue; - try { - await fsSvc.deleteStackPath(stackName, old); - } catch (e) { - console.warn(`[GitSource] stale file cleanup skipped ${sanitizeForLog(old)} for ${sanitizeForLog(stackName)}:`, (e as Error).message); - } + await fsSvc.deleteStackPath(stackName, old); } } @@ -1743,16 +1740,44 @@ export class GitSourceService { stackName: string, commitSha: string, opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean } = {}, - ): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> { - return this.withStackLock(stackName, () => this.applyLocked(stackName, commitSha, opts)); + ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { + return this.withStackLock(stackName, () => this.applyWithSharedLock(stackName, commitSha, opts)); } - /** Body of apply(); assumes the caller already holds the per-stack lock. */ + /** + * Acquire the shared stack-operation lock then run applyLocked. + * Callers that already hold the Git per-stack mutex (public apply, webhook + * auto-apply) use this so capture/promote/handoff/deploy cannot race other + * lifecycle ops. Do not nest withStackLock here. + */ + private async applyWithSharedLock( + stackName: string, + commitSha: string, + opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean }, + ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const lock = await StackOpLockService.getInstance().runExclusive( + nodeId, + stackName, + 'git_apply', + opts.actor ?? 'system:git-source', + () => this.applyLocked(stackName, commitSha, opts), + ); + if (!lock.ran) { + throw new GitSourceError( + 'GIT_ERROR', + `Another operation (${lock.existing.action}) is already in progress for ${stackName}.`, + ); + } + return lock.result; + } + + /** Body of apply(); assumes the caller already holds Git mutex + shared stack lock. */ private async applyLocked( stackName: string, commitSha: string, opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean }, - ): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> { + ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { const diag = isDebugEnabled(); const db = DatabaseService.getInstance(); const src = db.getGitSource(stackName); @@ -1775,6 +1800,9 @@ export class GitSourceService { ? this.crypto.decrypt(src.pending_env_content) : null; const manifestSvc = GitProjectManifestService.getInstance(); + const recoverySvc = StackUpdateRecoveryService.getInstance(); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + let recoveryId: string | undefined; let appliedSpec: GitSourceAppliedSpec | null; if (pending.candidateRelPath !== null && pending.inventory !== null) { @@ -1797,7 +1825,7 @@ export class GitSourceService { // The staged candidate must still exist and be complete; a deleted // candidate (or a node restart that swept it) invalidates the pull. const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data'); - const candidateAbs = path.join(dataDir, 'git-managed', String(NodeRegistry.getInstance().getDefaultNodeId()), stackName, pending.candidateRelPath); + const candidateAbs = path.join(dataDir, 'git-managed', String(nodeId), stackName, pending.candidateRelPath); try { await fsPromises.access(candidateAbs); } catch { @@ -1872,8 +1900,8 @@ export class GitSourceService { : null; const invocation: string[] = []; try { - invocation.push(...(await authoredComposeFileArgs(stackName, NodeRegistry.getInstance().getDefaultNodeId()))); - invocation.push(...(await authoredComposeEnvFileArgs(stackName, NodeRegistry.getInstance().getDefaultNodeId()))); + invocation.push(...(await authoredComposeFileArgs(stackName, nodeId))); + invocation.push(...(await authoredComposeEnvFileArgs(stackName, nodeId))); } catch (e) { console.warn(`[GitSource] invocation build failed for ${stackName}:`, (e as Error).message); } @@ -1903,6 +1931,25 @@ export class GitSourceService { ...(src.applied_deploy_spec?.files ?? [PRIMARY_COMPOSE_FILENAME]), ...(src.sync_env ? ['.env'] : []), ]; + try { + const candidate = await recoverySvc.captureCandidate({ + nodeId, + stackName, + createdBy: opts.actor ?? 'git-source', + operationKind: 'git_apply', + }); + recoveryId = candidate.id; + } catch (captureError) { + const detail = captureError instanceof Error ? captureError.message : String(captureError); + console.error( + `[GitSource] Recovery capture failed before apply of ${sanitizeForLog(stackName)}:`, + detail, + ); + throw new GitSourceError( + 'GIT_ERROR', + `Rollback capture failed before apply; refusing to promote without recovery coverage: ${scrubCredentials(detail)}`, + ); + } try { await manifestSvc.promoteGeneration(stackName, { sha: commitSha, @@ -1917,6 +1964,16 @@ export class GitSourceService { // The original error is logged with its stack for diagnosis, // and the message is scrubbed of credentials and of the // incoming manifest's high-sensitivity paths. + if (recoveryId) { + try { + await recoverySvc.abandon(recoveryId); + } catch (abandonError) { + console.warn( + `[GitSource] Failed to abandon recovery after promote failure for ${sanitizeForLog(stackName)}:`, + abandonError instanceof Error ? abandonError.message : String(abandonError), + ); + } + } if (e instanceof GitSourceError) throw e; const raw = e instanceof Error ? e.message : String(e); console.error(`[GitSource] promotion failed for ${sanitizeForLog(stackName)}:`, e instanceof Error ? e.stack ?? e.message : raw); @@ -1937,9 +1994,51 @@ export class GitSourceService { if (diag) console.log(`[GitSource:diag] apply validation fail stack=${stackName}`); throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`); } + // Capture the true pre-apply project BEFORE materialize writes new files. + try { + const captured = await recoverySvc.captureCandidate({ + nodeId, + stackName, + createdBy: opts.actor ?? 'git-source', + operationKind: 'git_apply', + }); + recoveryId = captured.id; + } catch (captureError) { + const detail = captureError instanceof Error ? captureError.message : String(captureError); + console.error( + `[GitSource] Recovery capture failed before legacy apply of ${sanitizeForLog(stackName)}:`, + detail, + ); + throw new GitSourceError( + 'GIT_ERROR', + `Rollback capture failed before apply; refusing to materialize without recovery coverage: ${scrubCredentials(detail)}`, + ); + } appliedSpec = await this.materialize( stackName, pending.files, pending.contextDir, src.sync_env, envContent, src.applied_deploy_spec, - ); + ).catch(async (materializeError: unknown) => { + if (recoveryId) { + const reverted = await recoverySvc.revertToGenerationContent(recoveryId); + if (!reverted) { + throw new GitSourceError( + 'GIT_ERROR', + `Legacy materialize failed and pre-apply generation restore also failed: ${scrubCredentials( + materializeError instanceof Error ? materializeError.message : String(materializeError), + )}`, + ); + } + try { + await recoverySvc.abandon(recoveryId); + } catch (abandonError) { + console.warn( + `[GitSource] Failed to abandon recovery after legacy materialize failure for ${sanitizeForLog(stackName)}:`, + abandonError instanceof Error ? abandonError.message : String(abandonError), + ); + } + recoveryId = undefined; + } + throw materializeError; + }); // Migration: build the conservative manifest from spec + disk. const migrated = await manifestSvc.buildMigratedManifest(stackName, { repo_url: src.repo_url, @@ -1958,9 +2057,25 @@ export class GitSourceService { const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply; if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy)); + const finalizeRecoveryCurrent = async (id: string, immediateVerified: boolean): Promise => { + if (!recoverySvc.markAcquired(id)) { + await recoverySvc.abandon(id); + throw new Error('Failed to mark recovery generation as acquired'); + } + if (!recoverySvc.handoff(id, nodeId, stackName)) { + await recoverySvc.abandon(id); + throw new Error('Failed to hand off recovery generation'); + } + if (!recoverySvc.markReconciling(id)) { + throw new Error('Failed to mark recovery generation as reconciling'); + } + if (immediateVerified && !recoverySvc.markImmediateVerified(id)) { + console.warn(`[GitSource] Could not CAS immediate_verified for recovery ${sanitizeForLog(id)}`); + } + }; + if (shouldDeploy) { try { - const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); await assertPolicyGateAllows( stackName, nodeId, @@ -1969,34 +2084,73 @@ export class GitSourceService { auditPath: `/api/stacks/${stackName}/git-source/apply`, }), ); - const lock = await StackOpLockService.getInstance().runExclusive( - nodeId, stackName, 'deploy', 'system', - () => ComposeService.getInstance(nodeId).deployStack( - stackName, - undefined, - undefined, - { source: 'git_apply', actor: opts.actor ?? 'system:git-source' }, - ), - ); - if (!lock.ran) { - const busy = `Auto-deploy skipped: another operation (${lock.existing.action}) is already in progress for ${stackName}.`; - console.warn(`[GitSource] ${busy}`); - return { applied: true, deployed: false, deployError: busy }; + if (recoveryId) { + await finalizeRecoveryCurrent(recoveryId, false); + } + // Shared stack lock already held as git_apply for capture→deploy. + await ComposeService.getInstance(nodeId).deployStack( + stackName, + undefined, + undefined, + { source: 'git_apply', actor: opts.actor ?? 'system:git-source' }, + ); + if (recoveryId) { + if (!recoverySvc.markImmediateVerified(recoveryId)) { + console.warn(`[GitSource] Could not CAS immediate_verified for recovery ${sanitizeForLog(recoveryId)}`); + } + } + const healthGateId = HealthGateService.getInstance().beginStack( + nodeId, + stackName, + 'deploy', + 'system:git-source', + ); + if (recoveryId) { + recoverySvc.linkGateOrRetain(recoveryId, healthGateId); } - HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:git-source'); console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`); - return { applied: true, deployed: true }; + return { applied: true, deployed: true, recoveryId }; } catch (e) { - // File is on disk, DB is marked applied. Returning the - // error separately lets the UI flag it as a partial - // success rather than rolling back the disk. + // R1: do not auto-compensate. Keep applied files and leave the + // pre-promote generation is_current for manual rollback. + if (recoveryId) { + const row = recoverySvc.get(recoveryId); + if (row && row.is_current !== 1) { + try { + await finalizeRecoveryCurrent(recoveryId, false); + } catch (handoffError) { + console.warn( + `[GitSource] Failed to hand off recovery after deploy failure for ${sanitizeForLog(stackName)}:`, + handoffError instanceof Error ? handoffError.message : String(handoffError), + ); + } + } + } const scrubbed = scrubCredentials((e as Error).message || String(e)); console.error(`[GitSource] Auto-deploy failed for ${stackName}: ${scrubbed}`); - return { applied: true, deployed: false, deployError: scrubbed }; + return { applied: true, deployed: false, deployError: scrubbed, recoveryId }; + } + } + + if (recoveryId) { + try { + await finalizeRecoveryCurrent(recoveryId, true); + } catch (finalizeError) { + const detail = finalizeError instanceof Error ? finalizeError.message : String(finalizeError); + console.error( + `[GitSource] Failed to finalize recovery for apply-only ${sanitizeForLog(stackName)}:`, + detail, + ); + return { + applied: true, + deployed: false, + deployError: `Recovery finalization failed after apply: ${scrubCredentials(detail)}`, + recoveryId, + }; } } console.log(`[GitSource] Applied ${stackName} at ${commitSha.slice(0, 7)}`); - return { applied: true, deployed: false }; + return { applied: true, deployed: false, recoveryId }; } public dismissPending(stackName: string): void { @@ -2369,7 +2523,10 @@ export class GitSourceService { return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` }; } - const applied = await this.applyLocked(stackName, pullResult.commitSha, { deploy: src.auto_deploy_on_apply }); + const applied = await this.applyWithSharedLock(stackName, pullResult.commitSha, { + deploy: src.auto_deploy_on_apply, + actor: 'system:webhook', + }); if (applied.deployError) { // Apply wrote to disk but deploy failed. Surface it so the // webhook_executions row records a degraded outcome instead diff --git a/backend/src/services/PolicyEnforcement.ts b/backend/src/services/PolicyEnforcement.ts index c14df60d..8ac9cddc 100644 --- a/backend/src/services/PolicyEnforcement.ts +++ b/backend/src/services/PolicyEnforcement.ts @@ -20,6 +20,7 @@ import { applySuppressions } from '../utils/suppression-filter'; import { validateImageRef } from '../utils/image-ref'; import { getErrorMessage } from '../utils/errors'; import { isDebugEnabled } from '../utils/debug'; +import type { RollbackInvocationRecord } from '../types/rollbackGeneration'; import { evaluatePolicyRisk, describePolicyInputs, @@ -58,6 +59,11 @@ export interface PolicyEnforcementOptions { auditMethod?: string; /** Request path of the originating route; used for audit attribution. */ auditPath?: string; + /** + * Rollback compensation: list images via this captured Compose invocation + * instead of the live database-derived args. + */ + composeInvocation?: RollbackInvocationRecord | null; } export interface PolicyEnforcementResult { @@ -286,7 +292,10 @@ export async function enforcePolicyPreDeploy( let imageRefs: string[] = []; try { - imageRefs = await ComposeService.getInstance(nodeId).listStackImages(stackName); + imageRefs = await ComposeService.getInstance(nodeId).listStackImages( + stackName, + opts.composeInvocation ?? null, + ); } catch (err) { const message = getErrorMessage(err, 'compose parse failed'); console.error('[Policy] listStackImages failed for %s:', sanitizeForLog(stackName), sanitizeForLog(message)); diff --git a/backend/src/services/RollbackGenerationStore.ts b/backend/src/services/RollbackGenerationStore.ts new file mode 100644 index 00000000..04a88315 --- /dev/null +++ b/backend/src/services/RollbackGenerationStore.ts @@ -0,0 +1,1163 @@ +/** + * On-disk content store for authored-project rollback generations. + * + * Layout under /backups///generations//: + * generation.json - RollbackGenerationManifest (checksums + metadata) + * files/ - tree mirroring stack-relative paths (ciphertext when encrypted) + * + * The DB row remains the listing / CURRENT pointer authority. This module only + * owns generation content directories. + */ +import { createHash, randomUUID } from 'crypto'; +import { promises as fsPromises } from 'fs'; +import path from 'path'; +import { CryptoService } from './CryptoService'; +import { FileSystemService } from './FileSystemService'; +import { isValidStackName } from '../utils/validation'; +import { + ROLLBACK_GENERATION_SCHEMA_VERSION, + type ResolvedRollbackInventory, + type RollbackGenerationEntry, + type RollbackGenerationManifest, + type RollbackImageIdentity, + type RollbackOperationKind, + type RollbackRestoreIntent, + type RollbackRestoreTransactionMeta, +} from '../types/rollbackGeneration'; +import { GitProjectManifestService } from './GitProjectManifestService'; +import type { GitProjectManifest, InputSensitivity } from '../types/gitProjectManifest'; + +const GENERATION_JSON = 'generation.json'; +const FILES_DIR = 'files'; +const PRE_RESTORE_DIR = 'pre-restore'; +const RESTORE_INTENT_FILE = 'restore-intent.json'; +const GIT_MANIFEST_SNAPSHOT = 'git-manifest.v1.json'; +const PRE_RESTORE_INDEX = 'index.json'; +const PRE_RESTORE_BLOBS = 'blobs'; + +interface PreRestoreIndexEntry { + relativePath: string; + state: 'present' | 'absent'; + blobId?: string; + mode?: number | null; + encrypted?: boolean; +} + +interface PreRestoreIndex { + version: 1; + entries: PreRestoreIndexEntry[]; +} + +function getDataDir(): string { + return process.env.DATA_DIR || path.join(process.cwd(), 'data'); +} + +function getBackupBaseDir(): string { + return path.join(getDataDir(), 'backups'); +} + +function posixRel(rel: string): string { + return rel.replace(/\\/g, '/').replace(/^\.\//, ''); +} + +function shouldEncryptPreRestore(sensitivity: InputSensitivity | undefined): boolean { + // Unknown sensitivity is encrypted fail-closed; only explicit low stays plaintext. + return sensitivity !== 'low'; +} + +async function chmodPrivate(target: string, mode: number): Promise { + if (process.platform === 'win32') return; + await fsPromises.chmod(target, mode); +} + +async function mkdirPrivate(dir: string): Promise { + await fsPromises.mkdir(dir, { recursive: true, mode: 0o700 }); + await chmodPrivate(dir, 0o700); +} + +async function writePrivate(target: string, data: string | Buffer): Promise { + await fsPromises.writeFile(target, data); + await chmodPrivate(target, 0o600); +} + +function assertSafeGenerationId(generationId: string): void { + if ( + !generationId + || generationId.includes('..') + || generationId.includes('/') + || generationId.includes('\\') + || path.isAbsolute(generationId) + ) { + throw Object.assign(new Error('Invalid generation id'), { code: 'INVALID_GENERATION_ID' }); + } +} + +function assertSafeStackName(stackName: string): void { + if (!isValidStackName(stackName)) { + throw Object.assign(new Error('Invalid stack name'), { code: 'INVALID_STACK_NAME' }); + } +} + +function sha256Of(buf: Buffer): string { + return createHash('sha256').update(buf).digest('hex'); +} + +export interface CaptureGenerationOpts { + nodeId: number; + stackName: string; + generationId: string; + inventory: ResolvedRollbackInventory; + operationKind?: RollbackOperationKind; + images?: RollbackImageIdentity[]; + lkgHint?: string | null; + capturedAt?: number; +} + +export class RollbackGenerationStore { + /** /backups///generations */ + static getGenerationsRoot(nodeId: number, stackName: string): string { + assertSafeStackName(stackName); + // Canonical js/path-injection barrier: resolve against a known-safe root + // then check containment with startsWith. CodeQL does not credit helpers. + const backupRoot = path.resolve(getBackupBaseDir()); + const root = path.resolve(backupRoot, String(nodeId), stackName, 'generations'); + if (!root.startsWith(backupRoot + path.sep)) { + throw Object.assign(new Error('Path escapes backup directory'), { code: 'INVALID_PATH' }); + } + return root; + } + + /** Final content directory for one generation id. */ + static getGenerationDir(nodeId: number, stackName: string, generationId: string): string { + assertSafeGenerationId(generationId); + const gensRoot = this.getGenerationsRoot(nodeId, stackName); + // Inline barrier at the generation-id join (same form as FileSystemService). + const gensResolved = path.resolve(gensRoot); + const genDir = path.resolve(gensResolved, generationId); + if (!genDir.startsWith(gensResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + return genDir; + } + + /** + * Stage a new generation under staging-, verify checksums, then rename + * into the final generationId directory. Failures delete only the staging + * directory and leave prior generations intact. + */ + static async captureGeneration(opts: CaptureGenerationOpts): Promise { + const { + nodeId, + stackName, + generationId, + inventory, + operationKind = 'unknown', + images = [], + lkgHint = null, + capturedAt = Date.now(), + } = opts; + + assertSafeStackName(stackName); + assertSafeGenerationId(generationId); + + const gensRoot = this.getGenerationsRoot(nodeId, stackName); + // Inline barrier at the mkdir sink for the generations root. + const gensResolved = path.resolve(gensRoot); + const backupRoot = path.resolve(getBackupBaseDir()); + if (!gensResolved.startsWith(backupRoot + path.sep)) { + throw Object.assign(new Error('Path escapes backup directory'), { code: 'INVALID_PATH' }); + } + await fsPromises.mkdir(gensResolved, { recursive: true }); + + // Inline barrier at the access sink for the final generation directory. + const finalResolved = path.resolve(gensResolved, generationId); + if (!finalResolved.startsWith(gensResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + const alreadyExists = await fsPromises.access(finalResolved).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ); + if (alreadyExists) { + throw Object.assign(new Error(`Generation directory already exists: ${generationId}`), { + code: 'GENERATION_EXISTS', + }); + } + + const composeBase = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir()); + const stackRoot = path.resolve(composeBase, stackName); + if (!stackRoot.startsWith(composeBase + path.sep)) { + throw Object.assign(new Error('Stack name escapes compose directory'), { code: 'INVALID_PATH' }); + } + + const stagingName = `staging-${randomUUID()}`; + const stagingDir = path.resolve(gensResolved, stagingName); + if (!stagingDir.startsWith(gensResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + const stagingFiles = path.resolve(stagingDir, FILES_DIR); + if (!stagingFiles.startsWith(stagingDir + path.sep)) { + throw Object.assign(new Error('Path escapes staging directory'), { code: 'INVALID_PATH' }); + } + + try { + await fsPromises.mkdir(stagingFiles, { recursive: true }); + + const entries: RollbackGenerationEntry[] = []; + const managedRelativePaths: string[] = []; + const managedSeen = new Set(); + const filesRoot = path.resolve(stagingFiles); + + for (const inv of inventory.entries) { + const relativePath = posixRel(inv.relativePath); + if (!managedSeen.has(relativePath)) { + managedSeen.add(relativePath); + managedRelativePaths.push(relativePath); + } + + if (inv.absolutePath === null) { + entries.push({ + relativePath, + dependencyKind: inv.dependencyKind, + provenance: inv.provenance, + state: 'tombstoned', + contentSha256: null, + sizeBytes: null, + sensitivity: inv.sensitivity, + encrypted: false, + mode: null, + }); + continue; + } + + // Inline barrier at the realpath / readFile sinks for live stack sources. + const srcCandidate = path.resolve(stackRoot, relativePath); + if (!srcCandidate.startsWith(stackRoot + path.sep)) { + throw new Error(`Inventory path escapes stack root: ${relativePath}`); + } + + let realPath: string; + try { + realPath = await fsPromises.realpath(srcCandidate); + } catch (e) { + throw new Error( + `Could not resolve ${relativePath} for generation capture: ${(e as Error).message}`, + { cause: e }, + ); + } + if (!realPath.startsWith(stackRoot + path.sep)) { + throw Object.assign( + new Error(`Inventory path escapes stack root via symlink: ${relativePath}`), + { code: 'SYMLINK_ESCAPE' }, + ); + } + + let fileMode: number | null = null; + try { + const st = await fsPromises.lstat(realPath); + if (st.isFile()) fileMode = st.mode & 0o777; + } catch (e) { + console.warn( + `[RollbackGenerationStore] Could not read mode for ${relativePath}:`, + (e as Error).message, + ); + } + + let plaintext: Buffer; + try { + plaintext = await fsPromises.readFile(realPath); + } catch (e) { + throw new Error( + `Could not read ${relativePath} for generation capture: ${(e as Error).message}`, + { cause: e }, + ); + } + + const contentSha256 = sha256Of(plaintext); + const encrypt = inv.sensitivity === 'high' || inv.sensitivity === 'medium'; + // Inline barrier at the write / mkdir sinks under the staging files tree. + const dest = path.resolve(filesRoot, relativePath); + if (!dest.startsWith(filesRoot + path.sep)) { + throw Object.assign(new Error('Path escapes staging files directory'), { code: 'INVALID_PATH' }); + } + const destParent = path.dirname(dest); + if (!destParent.startsWith(filesRoot + path.sep) && destParent !== filesRoot) { + throw Object.assign(new Error('Path escapes staging files directory'), { code: 'INVALID_PATH' }); + } + await fsPromises.mkdir(destParent, { recursive: true }); + + if (encrypt) { + const cipher = CryptoService.getInstance().encrypt(plaintext.toString('base64')); + await fsPromises.writeFile(dest, cipher, 'utf8'); + if (process.platform !== 'win32') { + // Inline barrier at the chmod sink on the same dest just written. + if (!dest.startsWith(filesRoot + path.sep)) { + throw Object.assign(new Error('Path escapes staging files directory'), { code: 'INVALID_PATH' }); + } + try { + await fsPromises.chmod(dest, 0o600); + } catch (e) { + console.warn( + `[RollbackGenerationStore] Could not set 0o600 on ${path.basename(relativePath)}:`, + (e as Error).message, + ); + } + } + } else { + await fsPromises.writeFile(dest, plaintext); + } + + entries.push({ + relativePath, + dependencyKind: inv.dependencyKind, + provenance: inv.provenance, + state: 'present', + contentSha256, + sizeBytes: plaintext.length, + sensitivity: inv.sensitivity, + encrypted: encrypt, + mode: fileMode, + }); + } + + managedRelativePaths.sort((a, b) => a.localeCompare(b)); + + let gitManifestCaptured = false; + if (inventory.git) { + const read = await GitProjectManifestService.getInstance().readManifest( + stackName, + inventory.git.repoUrl, + inventory.git.branch, + ); + // Never snapshot a corrupt or missing manifesto; restore must not + // reinstate unvalidated Git state as if it were authoritative. + if (read !== null && !('corrupt' in read)) { + // Inline barrier at the manifesto snapshot write sink. + const snapPath = path.resolve(stagingDir, GIT_MANIFEST_SNAPSHOT); + if (!snapPath.startsWith(stagingDir + path.sep)) { + throw Object.assign(new Error('Path escapes staging directory'), { code: 'INVALID_PATH' }); + } + await fsPromises.writeFile(snapPath, JSON.stringify(read, null, 2), 'utf8'); + gitManifestCaptured = true; + } + } + + const manifest: RollbackGenerationManifest = { + schemaVersion: ROLLBACK_GENERATION_SCHEMA_VERSION, + capabilityVersion: 1, + generationId, + nodeId, + stackName, + capturedAt, + operationKind, + entries, + managedRelativePaths, + invocation: inventory.invocation, + git: inventory.git, + priorRecords: { + appliedDeploySpec: inventory.appliedDeploySpec, + lkgHint, + lastAppliedContentHash: inventory.lastAppliedContentHash, + manifestState: inventory.manifestState, + manifestGeneration: inventory.manifestGeneration, + gitManifestCaptured, + }, + images, + }; + + // Inline barrier at the manifest write sink. + const stagingResolved = path.resolve(stagingDir); + const manifestPath = path.resolve(stagingResolved, GENERATION_JSON); + if (!manifestPath.startsWith(stagingResolved + path.sep)) { + throw Object.assign(new Error('Path escapes staging directory'), { code: 'INVALID_PATH' }); + } + await fsPromises.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf8'); + + await this.verifyManifestContent(stagingResolved, manifest); + + // Inline barriers at both sides of the rename sink. + const renameFrom = path.resolve(gensResolved, stagingName); + const renameTo = path.resolve(gensResolved, generationId); + if (!renameFrom.startsWith(gensResolved + path.sep) || !renameTo.startsWith(gensResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + await fsPromises.rename(renameFrom, renameTo); + return manifest; + } catch (e) { + // Inline barrier at the cleanup rm sink. + const cleanupRoot = path.resolve(gensResolved); + const cleanupDir = path.resolve(cleanupRoot, stagingName); + if (!cleanupDir.startsWith(cleanupRoot + path.sep)) { + throw e; + } + await fsPromises.rm(cleanupDir, { recursive: true, force: true }).catch((cleanupErr) => { + console.warn( + '[RollbackGenerationStore] Failed to clean staging directory after capture error:', + (cleanupErr as Error).message, + ); + }); + throw e; + } + } + + /** + * Persist finalized runtime image identity on an already-captured generation. + * Capture writes files first; image inspect happens after, so the manifest + * images array is patched in place once IDs, digests, and platform are known. + */ + static async attachImages( + nodeId: number, + stackName: string, + generationId: string, + images: RollbackImageIdentity[], + ): Promise { + assertSafeStackName(stackName); + assertSafeGenerationId(generationId); + const genDir = this.getGenerationDir(nodeId, stackName, generationId); + const genResolved = path.resolve(genDir); + const manifest = await this.readAndVerifyGeneration(genResolved); + const seen = new Set(); + for (const image of images) { + const name = image.serviceName.trim(); + if (!name) { + throw Object.assign( + new Error('Recovery image identity is missing a service name'), + { code: 'INVALID_IMAGE_IDENTITY' }, + ); + } + if (seen.has(name)) { + throw Object.assign( + new Error(`Duplicate recovery image identity for service "${name}"`), + { code: 'INVALID_IMAGE_IDENTITY' }, + ); + } + seen.add(name); + if (image.platform !== null && image.platform.trim() === '') { + throw Object.assign( + new Error(`Empty platform for service "${name}"`), + { code: 'INVALID_IMAGE_IDENTITY' }, + ); + } + } + manifest.images = images; + const manifestPath = path.resolve(genResolved, GENERATION_JSON); + if (!manifestPath.startsWith(genResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + await fsPromises.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf8'); + await this.readAndVerifyGeneration(genResolved); + } + + /** + * If a prior restore left a durable intent + pre-restore snapshot, revert the + * live managed set, Git DB projection, and managed manifesto to the + * pre-restore state. Used by startup reconciliation and failed compensate. + */ + static async reconcileInterruptedRestore( + nodeId: number, + stackName: string, + generationId: string, + ): Promise { + assertSafeStackName(stackName); + assertSafeGenerationId(generationId); + const genDir = this.getGenerationDir(nodeId, stackName, generationId); + const genResolved = path.resolve(genDir); + const intentPath = path.resolve(genResolved, RESTORE_INTENT_FILE); + if (!intentPath.startsWith(genResolved + path.sep)) return false; + let intentRaw: string; + try { + intentRaw = await fsPromises.readFile(intentPath, 'utf8'); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw e; + } + + let intent: RollbackRestoreIntent; + try { + intent = JSON.parse(intentRaw) as RollbackRestoreIntent; + } catch (e) { + throw new Error( + `Corrupt restore-intent.json for generation ${generationId}: ${(e as Error).message}`, + { cause: e }, + ); + } + + await this.revertFromPreRestoreSnapshot(nodeId, stackName, genResolved); + await this.restoreGitSideStateFromIntent(stackName, intent); + await fsPromises.rm(intentPath, { force: true }); + return true; + } + + /** + * Drop the durable restore intent and pre-restore snapshot after a successful + * compensation (files restored, policy passed, compose up + probe ok). + */ + static async commitRestoreTransaction( + nodeId: number, + stackName: string, + generationId: string, + ): Promise { + assertSafeStackName(stackName); + assertSafeGenerationId(generationId); + const genDir = this.getGenerationDir(nodeId, stackName, generationId); + const genResolved = path.resolve(genDir); + const intentPath = path.resolve(genResolved, RESTORE_INTENT_FILE); + const preRoot = path.resolve(genResolved, PRE_RESTORE_DIR); + if (intentPath.startsWith(genResolved + path.sep)) { + await fsPromises.rm(intentPath, { force: true }); + } + if (preRoot.startsWith(genResolved + path.sep)) { + await fsPromises.rm(preRoot, { recursive: true, force: true }); + } + } + + /** + * Restore a generation into the live stack. Verifies generation.json and + * content checksums before any live mutation. Writes present entries; + * deletes live paths that are in managedRelativePaths or liveManagedPaths + * but not present in the generation (tombstones and post-capture additions + * inside the managed set). Paths outside both sets are left untouched. + * + * Durability: copies the affected live paths into pre-restore/ and writes + * restore-intent.json (including Git DB + manifesto preimage) before mutation. + * Failure or restart reverts from that snapshot so a hybrid managed project + * cannot remain. + */ + static async restoreGeneration( + nodeId: number, + stackName: string, + generationId: string, + liveManagedPaths: string[], + transactionMeta?: RollbackRestoreTransactionMeta, + ): Promise { + assertSafeStackName(stackName); + assertSafeGenerationId(generationId); + + const genDir = this.getGenerationDir(nodeId, stackName, generationId); + const genResolved = path.resolve(genDir); + const manifest = await this.readAndVerifyGeneration(genDir); + + const presentByKey = new Map(); + for (const entry of manifest.entries) { + if (entry.state === 'present') presentByKey.set(posixRel(entry.relativePath), entry); + } + + const deleteRelByKey = new Map(); + const considerDelete = (relRaw: string): void => { + const rel = posixRel(relRaw); + if (presentByKey.has(rel) || deleteRelByKey.has(rel)) return; + deleteRelByKey.set(rel, rel); + }; + + for (const rel of manifest.managedRelativePaths) considerDelete(rel); + for (const entry of manifest.entries) { + if (entry.state === 'tombstoned') considerDelete(entry.relativePath); + } + for (const rel of liveManagedPaths) considerDelete(rel); + + const restores: Array<{ + relativePath: string; + content: Buffer; + sensitivity: RollbackGenerationEntry['sensitivity']; + mode: number | null; + }> = []; + for (const entry of presentByKey.values()) { + restores.push({ + relativePath: entry.relativePath, + content: await this.readPresentEntryBytes(genDir, entry), + sensitivity: entry.sensitivity, + mode: entry.mode ?? null, + }); + } + + const affectedPaths = [ + ...restores.map((r) => r.relativePath), + ...deleteRelByKey.values(), + ]; + + const fsSvc = FileSystemService.getInstance(nodeId); + const scope = { protectedEnabled: false as const }; + + // Refuse directory-at-file-path collisions before any snapshot or mutation. + for (const rel of affectedPaths) { + const kind = await fsSvc.pathKind(stackName, rel, scope); + if (kind === 'directory') { + throw Object.assign( + new Error( + `Managed path "${rel}" is a directory; refusing restore that would replace or delete it as a file`, + ), + { code: 'DIRECTORY_COLLISION' }, + ); + } + } + + const sensitivityByPath = new Map( + manifest.entries.map((entry) => [posixRel(entry.relativePath), entry.sensitivity]), + ); + await this.capturePreRestoreSnapshot( + nodeId, + stackName, + genResolved, + affectedPaths, + sensitivityByPath, + ); + const intentPath = path.resolve(genResolved, RESTORE_INTENT_FILE); + if (!intentPath.startsWith(genResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + const intent: RollbackRestoreIntent = { + generationId, + stackName, + nodeId, + paths: affectedPaths, + at: Date.now(), + }; + if (transactionMeta) { + intent.gitSide = { + gitDbBefore: transactionMeta.gitDbBefore, + managedManifestBefore: transactionMeta.managedManifestBefore, + }; + } + await fsPromises.writeFile(intentPath, JSON.stringify(intent), 'utf8'); + + try { + for (const item of restores) { + await fsSvc.writeStackFile(stackName, item.relativePath, item.content); + const sensitive = item.sensitivity === 'high' || item.sensitivity === 'medium'; + // Sensitive content is always owner-only; other files restore captured mode. + const targetMode = sensitive ? 0o600 : (item.mode ?? null); + if (targetMode === null || process.platform === 'win32') continue; + try { + await fsSvc.chmodStackPath(stackName, item.relativePath, targetMode, scope); + } catch (e) { + if (sensitive) { + throw Object.assign( + new Error( + `Could not apply permissions on sensitive restored path "${item.relativePath}": ${(e as Error).message}`, + ), + { code: 'RESTORE_CHMOD_FAILED', cause: e }, + ); + } + console.warn( + '[RollbackGenerationStore] Could not restore mode on entry:', + (e as Error).message, + ); + } + } + + for (const rel of deleteRelByKey.values()) { + try { + const kind = await fsSvc.pathKind(stackName, rel, scope); + if (kind === null) continue; + if (kind === 'directory') { + throw Object.assign( + new Error(`Managed path "${rel}" is a directory; refusing delete during restore`), + { code: 'DIRECTORY_COLLISION' }, + ); + } + await fsSvc.deleteStackPath(stackName, rel, false, scope); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') continue; + throw e; + } + } + + // Leave restore-intent.json + pre-restore/ until commitRestoreTransaction + // so a later policy/compose failure can still revert the managed set. + return manifest; + } catch (e) { + try { + await this.revertFromPreRestoreSnapshot(nodeId, stackName, genResolved); + await this.restoreGitSideStateFromIntent(stackName, intent); + await fsPromises.rm(intentPath, { force: true }); + } catch (revertErr) { + console.error( + '[RollbackGenerationStore] Failed to revert interrupted restore:', + (revertErr as Error).message, + ); + } + throw e; + } + } + + /** True when restore-intent.json still exists for this generation. */ + static async hasPendingRestoreIntent( + nodeId: number, + stackName: string, + generationId: string, + ): Promise { + assertSafeStackName(stackName); + assertSafeGenerationId(generationId); + const genDir = this.getGenerationDir(nodeId, stackName, generationId); + const genResolved = path.resolve(genDir); + const intentPath = path.resolve(genResolved, RESTORE_INTENT_FILE); + if (!intentPath.startsWith(genResolved + path.sep)) return false; + try { + await fsPromises.access(intentPath); + return true; + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw e; + } + } + + /** Restore captured managed manifesto from a generation content directory. */ + static async restoreCapturedGitManifest( + stackName: string, + generationDir: string, + generation: RollbackGenerationManifest, + ): Promise { + const genResolved = path.resolve(generationDir); + const snapPath = path.resolve(genResolved, GIT_MANIFEST_SNAPSHOT); + const capturedFlag = generation.priorRecords?.gitManifestCaptured; + const svc = GitProjectManifestService.getInstance(); + + // Inline barrier immediately before the read sink (no access-then-read TOCTOU). + if (!snapPath.startsWith(genResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + + if (capturedFlag === true) { + const raw = await fsPromises.readFile(snapPath, 'utf8'); + const parsed = JSON.parse(raw) as GitProjectManifest; + await svc.writeManifest(stackName, parsed); + return; + } + + if (capturedFlag === undefined) { + // Legacy generations: restore only when a snapshot file is present. + try { + const raw = await fsPromises.readFile(snapPath, 'utf8'); + const parsed = JSON.parse(raw) as GitProjectManifest; + await svc.writeManifest(stackName, parsed); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + } + return; + } + + // Explicit first-apply preimage only. Legacy generations without the flag + // and without a snapshot must not wipe a live manifesto that was never + // part of the capture contract. + if (capturedFlag === false && generation.git) { + await svc.clearManifestFile(stackName); + } + } + + private static async restoreGitSideStateFromIntent( + stackName: string, + intent: RollbackRestoreIntent, + ): Promise { + if (!intent.gitSide) return; + + const { DatabaseService } = await import('./DatabaseService'); + const db = DatabaseService.getInstance(); + const before = intent.gitSide.gitDbBefore; + if (before) { + db.setGitSourceAppliedSpec(stackName, before.appliedDeploySpec); + if (before.lastAppliedCommitSha) { + db.markGitSourceApplied( + stackName, + before.lastAppliedCommitSha, + before.lastAppliedContentHash || '', + ); + } else { + db.clearGitSourceAppliedRevision(stackName); + } + db.setGitSourceManifestState( + stackName, + before.manifestVersion, + before.manifestState, + before.manifestGeneration, + ); + } + + const svc = GitProjectManifestService.getInstance(); + if (intent.gitSide.managedManifestBefore === null) { + await svc.clearManifestFile(stackName); + } else { + const parsed = JSON.parse(intent.gitSide.managedManifestBefore) as GitProjectManifest; + await svc.writeManifest(stackName, parsed); + } + } + + private static async lstatManagedPath( + nodeId: number, + stackName: string, + rel: string, + ): Promise<'missing' | 'file' | 'directory' | 'symlink' | 'other'> { + const composeBase = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir()); + const stackRoot = path.resolve(composeBase, stackName); + if (!stackRoot.startsWith(composeBase + path.sep)) { + throw Object.assign(new Error('Stack name escapes compose directory'), { code: 'INVALID_PATH' }); + } + const candidate = path.resolve(stackRoot, posixRel(rel)); + if (!candidate.startsWith(stackRoot + path.sep)) { + throw Object.assign(new Error('Path escapes stack root'), { code: 'INVALID_PATH' }); + } + try { + const st = await fsPromises.lstat(candidate); + if (st.isSymbolicLink()) return 'symlink'; + if (st.isDirectory()) return 'directory'; + if (st.isFile()) return 'file'; + return 'other'; + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return 'missing'; + throw e; + } + } + + private static async capturePreRestoreSnapshot( + nodeId: number, + stackName: string, + genResolved: string, + relativePaths: string[], + sensitivityByPath: Map, + ): Promise { + const preRoot = path.resolve(genResolved, PRE_RESTORE_DIR); + if (!preRoot.startsWith(genResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + await fsPromises.rm(preRoot, { recursive: true, force: true }); + await mkdirPrivate(preRoot); + const blobsRoot = path.resolve(preRoot, PRE_RESTORE_BLOBS); + if (!blobsRoot.startsWith(preRoot + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + await mkdirPrivate(blobsRoot); + + const composeBase = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir()); + const stackRoot = path.resolve(composeBase, stackName); + if (!stackRoot.startsWith(composeBase + path.sep)) { + throw Object.assign(new Error('Stack name escapes compose directory'), { code: 'INVALID_PATH' }); + } + + const seen = new Set(); + const index: PreRestoreIndex = { version: 1, entries: [] }; + + for (const relRaw of relativePaths) { + const rel = posixRel(relRaw); + if (!rel || seen.has(rel)) continue; + seen.add(rel); + const sensitivity = sensitivityByPath.get(rel); + const srcCandidate = path.resolve(stackRoot, rel); + if (!srcCandidate.startsWith(stackRoot + path.sep)) continue; + try { + const realSrc = await fsPromises.realpath(srcCandidate); + if (!realSrc.startsWith(stackRoot + path.sep)) continue; + const st = await fsPromises.lstat(realSrc); + if (!st.isFile()) { + throw Object.assign( + new Error( + `Managed path "${rel}" is not a regular file; refusing to snapshot it as absent`, + ), + { code: 'DIRECTORY_COLLISION' }, + ); + } + const realForRead = await fsPromises.realpath(srcCandidate); + if (!realForRead.startsWith(stackRoot + path.sep)) continue; + const bytes = await fsPromises.readFile(realForRead); + const blobId = randomUUID(); + const blobPath = path.resolve(blobsRoot, blobId); + if (!blobPath.startsWith(blobsRoot + path.sep)) continue; + const encrypted = shouldEncryptPreRestore(sensitivity) && bytes.length > 0; + if (encrypted) { + const crypto = CryptoService.getInstance(); + const cipher = crypto.encrypt(bytes.toString('base64')); + if (!crypto.isEncrypted(cipher)) { + throw new Error(`Could not encrypt pre-restore content for ${rel}`); + } + await writePrivate(blobPath, cipher); + } else { + await writePrivate(blobPath, bytes); + } + index.entries.push({ + relativePath: rel, + state: 'present', + blobId, + mode: st.mode & 0o777, + encrypted, + }); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') { + index.entries.push({ relativePath: rel, state: 'absent' }); + continue; + } + throw e; + } + } + + const indexPath = path.resolve(preRoot, PRE_RESTORE_INDEX); + if (!indexPath.startsWith(preRoot + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + await writePrivate(indexPath, JSON.stringify(index)); + } + + private static async revertFromPreRestoreSnapshot( + nodeId: number, + stackName: string, + genResolved: string, + ): Promise { + const preRoot = path.resolve(genResolved, PRE_RESTORE_DIR); + if (!preRoot.startsWith(genResolved + path.sep)) return; + try { + await fsPromises.access(preRoot); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return; + throw e; + } + + const indexPath = path.resolve(preRoot, PRE_RESTORE_INDEX); + if (!indexPath.startsWith(preRoot + path.sep)) return; + + let index: PreRestoreIndex; + try { + const raw = await fsPromises.readFile(indexPath, 'utf8'); + index = JSON.parse(raw) as PreRestoreIndex; + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') { + // Legacy suffix-tombstone snapshots are unsupported after this change. + throw Object.assign( + new Error('Pre-restore index missing; cannot safely revert interrupted restore'), + { code: 'PRE_RESTORE_INDEX_MISSING', cause: e }, + ); + } + throw e; + } + if (!index || index.version !== 1 || !Array.isArray(index.entries)) { + throw Object.assign( + new Error('Pre-restore index is malformed'), + { code: 'PRE_RESTORE_INDEX_INVALID' }, + ); + } + + const blobsRoot = path.resolve(preRoot, PRE_RESTORE_BLOBS); + const fsSvc = FileSystemService.getInstance(nodeId); + const scope = { protectedEnabled: false as const }; + + for (const entry of index.entries) { + const rel = posixRel(entry.relativePath); + if (!rel) continue; + if (entry.state === 'absent') { + const liveKind = await this.lstatManagedPath(nodeId, stackName, rel); + if (liveKind === 'missing') continue; + if (liveKind !== 'file') { + throw Object.assign( + new Error( + `Managed path "${rel}" is a ${liveKind}; refusing to delete it while reverting an absent-file preimage`, + ), + { code: 'DIRECTORY_COLLISION' }, + ); + } + try { + await fsSvc.deleteStackPath(stackName, rel, false, scope); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + } + continue; + } + if (entry.state !== 'present' || !entry.blobId) { + throw Object.assign( + new Error(`Pre-restore entry for "${rel}" is incomplete`), + { code: 'PRE_RESTORE_INDEX_INVALID' }, + ); + } + const blobPath = this.resolvePreRestoreBlobPath(blobsRoot, entry.blobId); + let buf = await fsPromises.readFile(blobPath); + if (entry.encrypted === true) { + const cipherText = buf.toString('utf8'); + const crypto = CryptoService.getInstance(); + if (!crypto.isEncrypted(cipherText)) { + throw new Error(`Pre-restore content for ${rel} is marked encrypted but is not ciphertext`); + } + try { + buf = Buffer.from(crypto.decrypt(cipherText), 'base64'); + } catch (e) { + throw new Error( + `Could not decrypt pre-restore content for ${rel}: ${(e as Error).message}`, + { cause: e }, + ); + } + } + await fsSvc.writeStackFile(stackName, rel, buf); + if (typeof entry.mode === 'number' && process.platform !== 'win32') { + try { + await fsSvc.chmodStackPath(stackName, rel, entry.mode & 0o777, scope); + } catch (e) { + throw Object.assign( + new Error( + `Could not restore pre-restore permissions on "${rel}": ${(e as Error).message}`, + ), + { code: 'RESTORE_CHMOD_FAILED', cause: e }, + ); + } + } + } + await fsPromises.rm(preRoot, { recursive: true, force: true }); + } + + /** Resolve a pre-restore blob id under blobsRoot; rejects traversal. */ + private static resolvePreRestoreBlobPath(blobsRoot: string, blobId: string): string { + if (blobId.includes('..') || blobId.includes('/') || blobId.includes('\\')) { + throw Object.assign(new Error('Invalid pre-restore blob id'), { code: 'INVALID_PATH' }); + } + const blobPath = path.resolve(blobsRoot, blobId); + if (!blobPath.startsWith(blobsRoot + path.sep)) { + throw Object.assign(new Error('Path escapes pre-restore blobs'), { code: 'INVALID_PATH' }); + } + return blobPath; + } + + /** Remove one generation content directory. Missing dirs are a no-op. */ + static async retireGenerationContent( + nodeId: number, + stackName: string, + generationId: string, + ): Promise { + assertSafeStackName(stackName); + assertSafeGenerationId(generationId); + const gensRoot = this.getGenerationsRoot(nodeId, stackName); + // Inline barrier at the retire rm sink. + const gensResolved = path.resolve(gensRoot); + const dir = path.resolve(gensResolved, generationId); + if (!dir.startsWith(gensResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + await fsPromises.rm(dir, { recursive: true, force: true }); + } + + /** + * Verify generation.json exists and every present entry matches its checksum. + * Returns false on missing or corrupt content; does not throw for expected failures. + */ + static async verifyGenerationContent( + nodeId: number, + stackName: string, + generationId: string, + ): Promise { + try { + assertSafeStackName(stackName); + assertSafeGenerationId(generationId); + const genDir = this.getGenerationDir(nodeId, stackName, generationId); + await this.readAndVerifyGeneration(genDir); + return true; + } catch (e) { + const code = (e as { code?: string }).code; + if ( + code === 'ENOENT' + || code === 'INVALID_GENERATION_ID' + || code === 'INVALID_STACK_NAME' + || code === 'INVALID_PATH' + ) { + return false; + } + console.warn( + '[RollbackGenerationStore] verifyGenerationContent failed:', + (e as Error).message, + ); + return false; + } + } + + private static async readAndVerifyGeneration(genDir: string): Promise { + // Inline barrier at the manifest read sink. + const genResolved = path.resolve(genDir); + const manifestPath = path.resolve(genResolved, GENERATION_JSON); + if (!manifestPath.startsWith(genResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + let raw: string; + try { + raw = await fsPromises.readFile(manifestPath, 'utf8'); + } catch (e) { + throw new Error( + `Could not read generation manifest: ${(e as Error).message}`, + { cause: e }, + ); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (e) { + throw new Error(`Generation manifest is not valid JSON: ${(e as Error).message}`, { cause: e }); + } + const manifest = parsed as RollbackGenerationManifest; + if (manifest.schemaVersion !== ROLLBACK_GENERATION_SCHEMA_VERSION) { + throw new Error(`Unsupported generation schemaVersion ${String(manifest.schemaVersion)}`); + } + if (!Array.isArray(manifest.entries) || !Array.isArray(manifest.managedRelativePaths)) { + throw new Error('Generation manifest is missing entries or managedRelativePaths'); + } + await this.verifyManifestContent(genResolved, manifest); + return manifest; + } + + private static async verifyManifestContent( + genDir: string, + manifest: RollbackGenerationManifest, + ): Promise { + for (const entry of manifest.entries) { + if (entry.state !== 'present') continue; + if (!entry.contentSha256) { + throw new Error(`Present entry ${entry.relativePath} is missing contentSha256`); + } + const buf = await this.readPresentEntryBytes(genDir, entry); + const hash = sha256Of(buf); + if (hash !== entry.contentSha256) { + throw new Error( + `Checksum mismatch for ${entry.relativePath}: expected ${entry.contentSha256}, got ${hash}`, + ); + } + if (entry.sizeBytes !== null && buf.length !== entry.sizeBytes) { + throw new Error( + `Size mismatch for ${entry.relativePath}: expected ${entry.sizeBytes}, got ${buf.length}`, + ); + } + } + } + + private static async readPresentEntryBytes( + genDir: string, + entry: RollbackGenerationEntry, + ): Promise { + // Inline barrier at the content readFile sink. + const genResolved = path.resolve(genDir); + const filesRoot = path.resolve(genResolved, FILES_DIR); + if (!filesRoot.startsWith(genResolved + path.sep)) { + throw Object.assign(new Error('Path escapes generation directory'), { code: 'INVALID_PATH' }); + } + const relativePath = posixRel(entry.relativePath); + const storedPath = path.resolve(filesRoot, relativePath); + if (!storedPath.startsWith(filesRoot + path.sep)) { + throw Object.assign(new Error('Path escapes generation files directory'), { code: 'INVALID_PATH' }); + } + let stored: Buffer; + try { + stored = await fsPromises.readFile(storedPath); + } catch (e) { + throw new Error( + `Missing generation content for ${entry.relativePath}: ${(e as Error).message}`, + { cause: e }, + ); + } + if (!entry.encrypted) return stored; + + const cipherText = stored.toString('utf8'); + let b64: string; + try { + b64 = CryptoService.getInstance().decrypt(cipherText); + } catch (e) { + throw new Error( + `Could not decrypt generation content for ${entry.relativePath}: ${(e as Error).message}`, + { cause: e }, + ); + } + return Buffer.from(b64, 'base64'); + } +} + +export { getBackupBaseDir, getDataDir }; diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index 19393abf..a412a7b0 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -8,6 +8,7 @@ import { ComposeService } from './ComposeService'; import { StackUpdateOrchestrator } from './StackUpdateOrchestrator'; import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOpLockService'; import { FileSystemService } from './FileSystemService'; +import { StackUpdateRecoveryService } from './StackUpdateRecoveryService'; import { HealthGateService } from './HealthGateService'; import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService'; import { @@ -559,17 +560,21 @@ export class SchedulerService { this.assertStackTarget(task, 'Auto-backup'); if (this.isRemoteNode(task.node_id)) { await this.postToRemoteStack(task.node_id, `${encodeURIComponent(task.target_id)}/backup`); - return `Backed up stack "${task.target_id}" files on remote node`; + return `Captured a recovery generation for stack "${task.target_id}" on remote node`; } const localNodeId = task.node_id ?? NodeRegistry.getInstance().getDefaultNodeId(); const lock = await StackOpLockService.getInstance().runExclusive( localNodeId, task.target_id, 'backup', 'system', - () => FileSystemService.getInstance(localNodeId).backupStackFiles(task.target_id), + () => StackUpdateRecoveryService.getInstance().captureCurrentBackup({ + nodeId: localNodeId, + stackName: task.target_id, + createdBy: 'system:scheduler', + }), ); // Throw (not return) so the skip records as a failed run instead of a // silent success; the next scheduled tick retries once the lock frees. if (!lock.ran) throw new Error(skipMessage(task.target_id, lock.existing.action)); - return `Backed up stack "${task.target_id}" files`; + return `Captured a recovery generation for stack "${task.target_id}"`; } private async executeAutoStop(task: ScheduledTask): Promise { diff --git a/backend/src/services/StackOpLockService.ts b/backend/src/services/StackOpLockService.ts index fcb49926..49475db2 100644 --- a/backend/src/services/StackOpLockService.ts +++ b/backend/src/services/StackOpLockService.ts @@ -1,16 +1,17 @@ import { DatabaseService } from './DatabaseService'; /** * Tracks in-flight stack lifecycle operations (deploy, down, restart, stop, - * start, update, rollback, backup) per (nodeId, stackName). A second request to + * start, update, rollback, backup, delete, git_apply) per (nodeId, stackName). A second request to * the same stack while the first is still running returns 409 instead of racing * the first. Backup is included because it rewrites the shared rollback slot, so * it must not interleave with a deploy/update/rollback on the same stack. + * Git apply holds this lock across capture, promote, handoff, and optional deploy. * * State is intentionally process-local: a Sencho restart clears all locks, * which matches the lifecycle of any in-flight `docker compose` child process. */ -export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete'; +export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete' | 'git_apply'; /** * Note returned by a background path that skipped its operation because a manual diff --git a/backend/src/services/StackUpdateRecoveryService.ts b/backend/src/services/StackUpdateRecoveryService.ts index dc65ef82..015609c1 100644 --- a/backend/src/services/StackUpdateRecoveryService.ts +++ b/backend/src/services/StackUpdateRecoveryService.ts @@ -11,7 +11,6 @@ * - Services that were fully stopped at capture are kept at `scale: 0`. * - Authored replica counts are not used when they diverge from observed state. */ -import { randomUUID } from 'crypto'; import fs from 'fs/promises'; import path from 'path'; import { @@ -23,14 +22,211 @@ import { FileSystemService } from './FileSystemService'; import { buildEffectiveServiceModel } from './effectiveServiceModel'; import { classifyReferenceKind, - type ImageReferenceKind, resolveComposeProjectContext, + resolveComposeProjectContextForGeneration, } from './composeProjectContext'; +import { + collectImageIds, + collectRollbackTags, + parseServicesJsonStrict, + scrapeRollbackTagsLenient, + type StackRecoveryReplicaCapture, + type StackRecoveryServiceCapture, +} from './recoveryServicesJson'; import { getComposeCommandTimeoutMs } from './ComposeService'; +import { assessGenerationEligibility } from './rollbackEligibility'; +import { enforcePolicyForImageRefs, type PolicyEnforcementOptions } from './PolicyEnforcement'; +import { describePolicyBlock } from '../helpers/policyGate'; +import type { GitSourceAppliedSpec } from './DatabaseService'; +import type { GitSourceManifestState } from '../types/gitProjectManifest'; +import type { + RollbackGenerationManifest, + RollbackGitDbSnapshot, + RollbackImageIdentity, + RollbackInvocationRecord, + RollbackOperationKind, + RollbackRestoreTransactionMeta, +} from '../types/rollbackGeneration'; +import { getBackupBaseDir, RollbackGenerationStore } from './RollbackGenerationStore'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; import { isValidStackName } from '../utils/validation'; +export type { StackRecoveryReplicaCapture, StackRecoveryServiceCapture } from './recoveryServicesJson'; +export { parseServicesJsonStrict } from './recoveryServicesJson'; + +const GENERATION_UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +const STAGING_MAX_AGE_MS = 60 * 60 * 1000; + +function looksLikeGenerationUuid(value: string): boolean { + return GENERATION_UUID_RE.test(value); +} + +function isComposeOneOff(labels: Record | undefined): boolean { + const value = labels?.['com.docker.compose.oneoff']; + return typeof value === 'string' && value.toLowerCase() === 'true'; +} + +function isHeldRecoveryImageMissing(error: unknown): boolean { + const message = getErrorMessage(error, ''); + if (/no such image/i.test(message)) return true; + return /sencho-rb\//i.test(message) + && /manifest unknown|repository does not exist|failed to resolve reference|pull access denied/i.test(message); +} + +function formatImagePlatform(inspect: { + Os?: string; + Architecture?: string; + Variant?: string; +}): string | null { + const os = typeof inspect.Os === 'string' ? inspect.Os.trim() : ''; + const arch = typeof inspect.Architecture === 'string' ? inspect.Architecture.trim() : ''; + if (!os || !arch) return null; + const variant = typeof inspect.Variant === 'string' ? inspect.Variant.trim() : ''; + return variant ? `${os}/${arch}/${variant}` : `${os}/${arch}`; +} + +/** New content-store generations always set content_path. Never infer from UUID shape. */ +function expectsGenerationContent(row: StackUpdateRecoveryGenerationRow): boolean { + return typeof row.content_path === 'string' && row.content_path.length > 0; +} + +async function generationContentPresent( + nodeId: number, + stackName: string, + generationId: string, +): Promise { + try { + const genDir = RollbackGenerationStore.getGenerationDir(nodeId, stackName, generationId); + await fs.access(path.join(genDir, 'generation.json')); + return true; + } catch { + return false; + } +} + +async function resolveRestoreContext(row: StackUpdateRecoveryGenerationRow) { + if (expectsGenerationContent(row)) { + const contentKey = row.content_path!; + if (!looksLikeGenerationUuid(contentKey)) { + throw Object.assign( + new Error('Recovery generation content key is missing or invalid'), + { code: 'GENERATION_CONTENT_MISSING' }, + ); + } + const present = await generationContentPresent(row.node_id, row.stack_name, contentKey); + if (!present) { + throw Object.assign( + new Error('Recovery generation content is missing or incomplete'), + { code: 'GENERATION_CONTENT_MISSING' }, + ); + } + return resolveComposeProjectContextForGeneration( + row.node_id, + row.stack_name, + contentKey, + ); + } + // Pre-migration rows: content_path null (even when backup_slot_id is a UUID). + return resolveComposeProjectContext(row.node_id, row.stack_name); +} + +async function restoreCapturedGitDatabaseState( + stackName: string, + manifest: RollbackGenerationManifest, +): Promise { + const db = DatabaseService.getInstance(); + const src = db.getGitSource(stackName); + if (!src) return; + + const rawSpec = manifest.priorRecords?.appliedDeploySpec; + if (rawSpec === null) { + db.setGitSourceAppliedSpec(stackName, null); + } else if (typeof rawSpec === 'string' && rawSpec.length > 0) { + try { + const parsed = JSON.parse(rawSpec) as GitSourceAppliedSpec; + if (parsed && Array.isArray(parsed.files)) { + db.setGitSourceAppliedSpec(stackName, parsed); + } + } catch (e) { + throw new Error( + `Stored applied deploy specification is corrupt: ${(e as Error).message}`, + { cause: e }, + ); + } + } + + if (!manifest.git) return; + + const commitSha = manifest.git.commitSha?.trim() || null; + const contentHash = + typeof manifest.priorRecords?.lastAppliedContentHash === 'string' + ? manifest.priorRecords.lastAppliedContentHash + : null; + + if (commitSha) { + db.markGitSourceApplied(stackName, commitSha, contentHash ?? ''); + } else { + // First-apply preimage: clear any SHA written after capture. + db.clearGitSourceAppliedRevision(stackName); + } + + const capturedGeneration = + typeof manifest.priorRecords?.manifestGeneration === 'string' + ? manifest.priorRecords.manifestGeneration + : null; + const manifestStateRaw = manifest.priorRecords?.manifestState; + db.setGitSourceManifestState( + stackName, + manifest.git.manifestVersion ?? null, + (typeof manifestStateRaw === 'string' + ? manifestStateRaw + : null) as GitSourceManifestState | null, + capturedGeneration, + ); +} + +function snapshotGitDb(src: NonNullable>): RollbackGitDbSnapshot { + return { + appliedDeploySpec: src.applied_deploy_spec + ? { + files: [...src.applied_deploy_spec.files], + contextDir: src.applied_deploy_spec.contextDir, + } + : null, + lastAppliedCommitSha: src.last_applied_commit_sha, + lastAppliedContentHash: src.last_applied_content_hash, + manifestVersion: src.manifest_version, + manifestState: src.manifest_state, + manifestGeneration: src.manifest_generation, + }; +} + +async function captureGitSidePreimage(stackName: string): Promise { + const priorGit = DatabaseService.getInstance().getGitSource(stackName); + if (!priorGit) { + return { gitDbBefore: null, managedManifestBefore: null }; + } + const { GitProjectManifestService } = await import('./GitProjectManifestService'); + return { + gitDbBefore: snapshotGitDb(priorGit), + managedManifestBefore: await GitProjectManifestService.getInstance().readRawManifestText(stackName), + }; +} + +async function applyRestoredGenerationGitSide( + stackName: string, + nodeId: number, + contentPath: string, + restoredManifest: RollbackGenerationManifest, +): Promise { + const genDir = RollbackGenerationStore.getGenerationDir(nodeId, stackName, contentPath); + await RollbackGenerationStore.restoreCapturedGitManifest(stackName, genDir, restoredManifest); + await restoreCapturedGitDatabaseState(stackName, restoredManifest); +} + const SWEEP_INTERVAL_MS = 5 * 60_000; const INITIAL_SWEEP_DELAY_MS = 30_000; const MIN_RECOVERY_WINDOW_SECONDS = 90; @@ -38,28 +234,12 @@ const RECOVERY_TTL_BUFFER_MS = 30 * 60_000; const GATE_RETAIN_DEFAULT_MS = 2 * 60 * 60_000; const RECOVERY_PROBE_DELAY_MS = 3_000; -export interface StackRecoveryReplicaCapture { - containerId: string | null; - imageId: string | null; - repoDigest: string | null; - state: 'running' | 'stopped' | 'none'; - rollbackTag: string | null; -} - -export interface StackRecoveryServiceCapture { - serviceName: string; - /** Observed running replica count at capture (supported restore scale). */ - scale: number; - hasBuild: boolean; - declaredImageRef: string | null; - referenceKind: ImageReferenceKind; - replicas: StackRecoveryReplicaCapture[]; -} - export interface CaptureStackUpdateInput { nodeId: number; stackName: string; createdBy: string | null; + /** Capture trigger; defaults to 'update'. */ + operationKind?: RollbackOperationKind; } function yamlQuote(value: string): string { @@ -79,36 +259,6 @@ function opaqueRollbackTag(generationId: string, serviceName: string): string { return `sencho-rb/${shortGenerationId(generationId)}/${sanitizeServiceSlug(serviceName)}:hold`; } -function parseServicesJson(raw: string): StackRecoveryServiceCapture[] { - try { - const parsed: unknown = JSON.parse(raw); - if (!Array.isArray(parsed)) return []; - return parsed as StackRecoveryServiceCapture[]; - } catch { - return []; - } -} - -export function collectImageIdsFromServicesJson(servicesJson: string): string[] { - const ids = new Set(); - for (const svc of parseServicesJson(servicesJson)) { - for (const replica of svc.replicas ?? []) { - if (replica.imageId && replica.imageId.trim()) ids.add(replica.imageId); - } - } - return [...ids]; -} - -function collectRollbackTags(services: StackRecoveryServiceCapture[]): string[] { - const tags = new Set(); - for (const svc of services) { - for (const replica of svc.replicas ?? []) { - if (replica.rollbackTag) tags.add(replica.rollbackTag); - } - } - return [...tags]; -} - export class StackUpdateRecoveryService { private static instance: StackUpdateRecoveryService; private started = false; @@ -134,6 +284,8 @@ export class StackUpdateRecoveryService { public start(): void { this.started = true; if (this.initialTimer || this.intervalId) return; + // Startup already awaits reconcileInterruptedRestoresAtStartup before + // HTTP/mutators. Periodic full reconcile (abandon/TTL) starts after delay. this.initialTimer = setTimeout(() => { void this.reconcileIncomplete(); this.intervalId = setInterval(() => { @@ -142,6 +294,93 @@ export class StackUpdateRecoveryService { }, INITIAL_SWEEP_DELAY_MS); } + /** + * Revert any crash-interrupted generation restores (files + Git side state) + * before background mutators or HTTP accept traffic. + */ + public async reconcileInterruptedRestoresAtStartup(): Promise { + await this.sweepInterruptedRestores(DatabaseService.getInstance()); + } + + private async sweepInterruptedRestores(db: DatabaseService): Promise { + const failures: string[] = []; + for (const node of db.getNodes()) { + for (const row of db.listStackUpdateRecoveryGenerationsForNode(node.id)) { + if (!row.content_path) continue; + try { + const reverted = await RollbackGenerationStore.reconcileInterruptedRestore( + row.node_id, + row.stack_name, + row.content_path, + ); + if (reverted) { + db.updateStackUpdateRecoveryGeneration(row.id, { status: 'recovery_required' }); + } + if (await RollbackGenerationStore.hasPendingRestoreIntent( + row.node_id, + row.stack_name, + row.content_path, + )) { + failures.push(row.id); + } + } catch (e) { + console.warn( + '[StackUpdateRecovery] Interrupted restore reconcile failed for %s: %s', + sanitizeForLog(row.id), + sanitizeForLog(getErrorMessage(e, 'unknown')), + ); + this.markGenerationRecoveryRequiredBestEffort(db, row.id); + failures.push(row.id); + } + } + } + if (failures.length > 0) { + throw new Error( + `Unresolved interrupted restore intent(s) remain for generation(s): ${failures.join(', ')}`, + ); + } + } + + private markGenerationRecoveryRequiredBestEffort( + db: DatabaseService, + generationId: string, + ): void { + try { + db.updateStackUpdateRecoveryGeneration(generationId, { status: 'recovery_required' }); + } catch (updateErr) { + console.warn( + '[StackUpdateRecovery] Failed to mark recovery_required after reconcile error: %s', + sanitizeForLog(getErrorMessage(updateErr, 'unknown')), + ); + } + } + + /** + * Block mutations while a restore intent is still on disk for any generation + * of this stack (mirrors deletion-intent gating). + */ + public async assertNoBlockingRestoreIntent(nodeId: number, stackName: string): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack name'); + } + const db = DatabaseService.getInstance(); + for (const row of db.listStackUpdateRecoveryGenerationsForNode(nodeId)) { + if (row.stack_name !== stackName || !row.content_path) continue; + const pending = await RollbackGenerationStore.hasPendingRestoreIntent( + nodeId, + stackName, + row.content_path, + ); + if (!pending) continue; + throw Object.assign( + new Error( + `Stack "${stackName}" has an interrupted restore in progress; resolve recovery before mutating`, + ), + { code: 'RESTORE_INTENT_BLOCKING' }, + ); + } + } + public stop(): void { this.started = false; if (this.initialTimer) { @@ -160,9 +399,11 @@ export class StackUpdateRecoveryService { */ public async captureCandidate(input: CaptureStackUpdateInput): Promise { const { nodeId, stackName, createdBy } = input; + const operationKind: RollbackOperationKind = input.operationKind ?? 'update'; if (!isValidStackName(stackName)) { throw new Error('Invalid stack name'); } + await this.assertNoBlockingRestoreIntent(nodeId, stackName); const context = await resolveComposeProjectContext(nodeId, stackName); await context.validateForMutation(); @@ -171,17 +412,17 @@ export class StackUpdateRecoveryService { const { ComposeService } = await import('./ComposeService'); await ComposeService.getInstance(nodeId).validateExactComposeInvocation(stackName); - const backupSlotId = await context.backupFromContext('update'); + // Content-store generation id becomes the row id, backup_slot_id, and content_path. + const generationId = await context.backupFromContext(operationKind); const model = await buildEffectiveServiceModel(nodeId, stackName); if (!model.renderable) { throw new Error(model.error || 'Effective Compose model failed to render'); } - - const generationId = randomUUID(); const docker = DockerController.getInstance(nodeId).getDocker(); const services: StackRecoveryServiceCapture[] = []; const createdTags: string[] = []; + const platformByImageId = new Map(); let overridePath: string | null = null; try { @@ -200,6 +441,8 @@ export class StackUpdateRecoveryService { const replicas: StackRecoveryReplicaCapture[] = []; for (const info of listed) { + const labels = (info.Labels ?? {}) as Record; + if (isComposeOneOff(labels)) continue; try { const inspect = await docker.getContainer(info.Id).inspect(); const status = inspect.State?.Status; @@ -219,6 +462,9 @@ export class StackUpdateRecoveryService { const image = await docker.getImage(imageId).inspect(); const digests = (image.RepoDigests ?? []) as string[]; repoDigest = digests.length > 0 ? digests[0] : null; + if (!platformByImageId.has(imageId)) { + platformByImageId.set(imageId, formatImagePlatform(image)); + } } catch { repoDigest = null; } @@ -247,6 +493,36 @@ export class StackUpdateRecoveryService { }); } + for (const svc of services) { + const imageIds = new Set(); + for (const replica of svc.replicas) { + if ((replica.state === 'running' || replica.state === 'stopped') && replica.imageId?.trim()) { + imageIds.add(replica.imageId); + } + } + if (imageIds.size > 1) { + throw Object.assign( + new Error( + `Service "${svc.serviceName}" has mixed replica images; refusing recovery capture that cannot restore exact prior identity`, + ), + { code: 'MIXED_REPLICA_IMAGES' }, + ); + } + } + + const images: RollbackImageIdentity[] = services.map((svc) => { + const primary = svc.replicas.find((r) => r.imageId) ?? null; + const imageId = primary?.imageId ?? null; + return { + serviceName: svc.serviceName, + imageId, + repoDigest: primary?.repoDigest ?? null, + platform: imageId ? (platformByImageId.get(imageId) ?? null) : null, + declaredImageRef: svc.declaredImageRef, + }; + }); + await RollbackGenerationStore.attachImages(nodeId, stackName, generationId, images); + const taggedIds = new Set(); for (const svc of services) { const primary = svc.replicas.find((r) => r.imageId) ?? null; @@ -277,7 +553,9 @@ export class StackUpdateRecoveryService { status: 'candidate', phase: 'captured', is_current: 0, - backup_slot_id: backupSlotId, + backup_slot_id: generationId, + content_path: generationId, + operation_kind: operationKind, override_path: overridePath, services_json: JSON.stringify(services), health_gate_id: null, @@ -302,10 +580,80 @@ export class StackUpdateRecoveryService { // Best-effort mid-capture cleanup. } } + try { + await RollbackGenerationStore.retireGenerationContent(nodeId, stackName, generationId); + } catch (retireError) { + console.warn( + '[StackUpdateRecovery] Failed to retire staged generation content after capture error: %s', + sanitizeForLog(getErrorMessage(retireError, 'unknown')), + ); + } throw error; } } + /** + * Capture the live authored project as the current recovery generation. + * Shares captureCandidate with deploy/update (files, holds, override), then + * hands off immediately without compose or a runtime probe. + */ + public async captureCurrentBackup(input: Omit): Promise { + const current = this.getCurrent(input.nodeId, input.stackName); + if (current?.health_gate_id) { + const gate = DatabaseService.getInstance().getHealthGateRun( + current.node_id, + current.stack_name, + current.health_gate_id, + ); + if (gate?.status === 'observing') { + throw Object.assign( + new Error('Cannot replace the current recovery generation while a health gate is observing'), + { code: 'HEALTH_GATE_OBSERVING' }, + ); + } + } + + const row = await this.captureCandidate({ + ...input, + operationKind: 'manual_backup', + }); + try { + if (!this.markAcquired(row.id)) { + throw new Error('Could not acquire the backup generation'); + } + if (!this.handoff(row.id, row.node_id, row.stack_name)) { + throw new Error('Could not hand off the backup generation'); + } + } catch (error) { + try { + await this.abandon(row.id); + } catch (abandonErr) { + console.warn( + '[StackUpdateRecovery] Failed to abandon backup generation after handoff error: %s', + sanitizeForLog(getErrorMessage(abandonErr, 'unknown')), + ); + } + throw error; + } + + if (!this.markReconciling(row.id)) { + console.warn( + '[StackUpdateRecovery] Backup generation handed off but reconciling CAS failed for %s', + sanitizeForLog(row.id), + ); + } else if (!this.markImmediateVerified(row.id)) { + console.warn( + '[StackUpdateRecovery] Backup generation handed off but immediate_verified CAS failed for %s', + sanitizeForLog(row.id), + ); + } + const verified = this.get(row.id); + if (!verified) { + throw new Error('Backup generation missing after handoff'); + } + return verified; + } + private async writeRecoveryOverride( nodeId: number, stackName: string, @@ -406,8 +754,10 @@ export class StackUpdateRecoveryService { /** * Informational mirror of releaseStackUpdateRecoveryGeneration's WHERE - * clause, for the list endpoint to grey out a row it already knows is - * ineligible. Not authoritative: releaseGeneration revalidates for real. + * clause, plus a strict services_json parse. Malformed recovery image + * state is refused here (and in releaseGeneration as malformed_services) + * before the DB update; the SQL WHERE clause does not encode that check. + * Not authoritative: releaseGeneration revalidates for real. */ public isReleaseEligible(row: StackUpdateRecoveryGenerationRow): boolean { if (row.released_at !== null || row.artifacts_retired !== 0) return false; @@ -417,6 +767,7 @@ export class StackUpdateRecoveryService { const gate = DatabaseService.getInstance().getHealthGateRun(row.node_id, row.stack_name, row.health_gate_id); if (gate?.status === 'observing') return false; } + if (!parseServicesJsonStrict(row.services_json).ok) return false; return true; } @@ -436,11 +787,14 @@ export class StackUpdateRecoveryService { releasedBy: string | null, ): Promise< | { ok: true; row: StackUpdateRecoveryGenerationRow; artifactsCleaned: boolean } - | { ok: false; reason: 'not_found' | 'already_released' | 'not_eligible' } + | { ok: false; reason: 'not_found' | 'already_released' | 'not_eligible' | 'malformed_services' } > { const before = this.get(id); if (!before) return { ok: false, reason: 'not_found' }; if (before.released_at !== null) return { ok: false, reason: 'already_released' }; + if (!parseServicesJsonStrict(before.services_json).ok) { + return { ok: false, reason: 'malformed_services' }; + } const released = DatabaseService.getInstance().releaseStackUpdateRecoveryGeneration(id, releasedBy); if (!released) return { ok: false, reason: 'not_eligible' }; @@ -540,29 +894,130 @@ export class StackUpdateRecoveryService { */ public async compensateWithCandidate( generationId: string, - composeUp: (overridePath: string) => Promise, + composeUp: ( + overridePath: string, + invocation: RollbackInvocationRecord | null, + ) => Promise, + policyOptions?: PolicyEnforcementOptions, ): Promise { const row = this.get(generationId); if (!row) return false; + // Finish any leftover restore transaction for this generation before a new + // restore can overwrite pre-restore/ with an already-restored tree. + if (row.content_path) { + try { + const reverted = await RollbackGenerationStore.reconcileInterruptedRestore( + row.node_id, + row.stack_name, + row.content_path, + ); + if (reverted) { + DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, { + status: 'recovery_required', + }); + } + } catch (e) { + console.warn( + '[StackUpdateRecovery] Pre-compensate reconcile failed for %s: %s', + sanitizeForLog(generationId), + sanitizeForLog(getErrorMessage(e, 'unknown')), + ); + } + } + await this.assertNoBlockingRestoreIntent(row.node_id, row.stack_name); + + const db = DatabaseService.getInstance(); + const transactionMeta = await captureGitSidePreimage(row.stack_name); + const generationContentPath = + expectsGenerationContent(row) && row.content_path ? row.content_path : null; + + let filesRestored = false; try { - const context = await resolveComposeProjectContext(row.node_id, row.stack_name); - await context.restoreFromContext(); + // Eligibility (integrity + held images + security posture) before mutation. + const integrityVerdict = await assessGenerationEligibility(row); + if (integrityVerdict === 'prohibited') { + throw Object.assign( + new Error('Rollback is prohibited for this generation'), + { code: 'ROLLBACK_PROHIBITED' }, + ); + } + + // Validate recovery image state before mutating the live project. + const servicesParsed = parseServicesJsonStrict(row.services_json); + if (!servicesParsed.ok) { + throw Object.assign( + new Error('Recovery generation has malformed services state; refusing rollback'), + { code: 'ROLLBACK_PROHIBITED' }, + ); + } + const rollbackTags = collectRollbackTags(servicesParsed.services); + const heldRefs = rollbackTags.length > 0 + ? rollbackTags + : collectImageIds(servicesParsed.services); + if (heldRefs.length === 0 && servicesParsed.services.some((s) => s.scale > 0)) { + throw Object.assign( + new Error('Recovery generation has no held image references for running services'), + { code: 'ROLLBACK_PROHIBITED' }, + ); + } + + const context = await resolveRestoreContext(row); + const restoredManifest = await context.restoreFromContext(transactionMeta); + filesRestored = true; + + // Evaluate current policy against the exact held images rollback will launch + // (opaque tags / image ids), not the restored authored moving tags. + const restoredInvocation = restoredManifest?.invocation ?? null; + const gate = await enforcePolicyForImageRefs(row.stack_name, row.node_id, heldRefs, { + bypass: policyOptions?.bypass ?? false, + actor: policyOptions?.actor ?? 'recovery-compensate', + ip: policyOptions?.ip, + auditMethod: policyOptions?.auditMethod ?? 'POST', + auditPath: policyOptions?.auditPath ?? '/api/stacks/rollback', + }); + if (!gate.ok) { + throw Object.assign( + new Error(describePolicyBlock(gate.policy, gate.violations, 'rollback')), + { code: 'ROLLBACK_PROHIBITED', policy: gate.policy, violations: gate.violations }, + ); + } + if (!row.override_path) { throw new Error('Recovery generation has no override path'); } - await composeUp(row.override_path); + await composeUp(row.override_path, restoredInvocation); const probeOk = await this.probeRecoveredStack( row.node_id, row.stack_name, row.services_json, ); if (!probeOk) { - DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, { - status: 'recovery_required', - }); - return false; + // Revert files via the existing catch path. Do not apply Git or commit + // the restore transaction; a leftover crash-intent would later undo + // files while recovered containers stay running. + throw Object.assign(new Error('Recovery health probe failed'), { code: 'RECOVERY_PROBE_FAILED' }); } - DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, { + + if (restoredManifest) { + if (generationContentPath) { + await applyRestoredGenerationGitSide( + row.stack_name, + row.node_id, + generationContentPath, + restoredManifest, + ); + } else { + await restoreCapturedGitDatabaseState(row.stack_name, restoredManifest); + } + } + if (generationContentPath) { + await RollbackGenerationStore.commitRestoreTransaction( + row.node_id, + row.stack_name, + generationContentPath, + ); + } + db.updateStackUpdateRecoveryGeneration(generationId, { status: 'restored_current', phase: 'immediate_verified', is_current: 1, @@ -570,23 +1025,104 @@ export class StackUpdateRecoveryService { }); return true; } catch (error) { + const code = (error as { code?: string }).code; + if (filesRestored && generationContentPath) { + try { + await RollbackGenerationStore.reconcileInterruptedRestore( + row.node_id, + row.stack_name, + generationContentPath, + ); + } catch (revertErr) { + console.error( + '[StackUpdateRecovery] Failed to revert files after compensation error: %s', + sanitizeForLog(getErrorMessage(revertErr, 'unknown')), + ); + } + } + if (code === 'ROLLBACK_PROHIBITED') { + throw error; + } console.error( '[StackUpdateRecovery] Compensation failed for %s: %s', sanitizeForLog(generationId), sanitizeForLog(getErrorMessage(error, 'unknown')), ); - DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, { + db.updateStackUpdateRecoveryGeneration(generationId, { status: 'recovery_required', }); + if ( + code === 'GENERATION_CONTENT_MISSING' + || code === 'HELD_IMAGE_MISSING' + || code === 'RECOVERY_PROBE_FAILED' + ) { + throw error; + } + if (isHeldRecoveryImageMissing(error)) { + throw Object.assign( + new Error('Held recovery image is missing'), + { code: 'HELD_IMAGE_MISSING' }, + ); + } return false; } } /** - * Verify recovered runtime against the captured generation. - * Rejects absent, restarting, dead, exited, or unhealthy expected replicas, - * image-id mismatches vs capture, and any running replica of a scale-0 service. + * Restore generation project files + Git manifesto/DB without compose-up. + * Used when legacy Git materialize fails mid-write and must reinstate the + * pre-apply capture before abandoning the recovery candidate. */ + public async revertToGenerationContent(generationId: string): Promise { + const row = this.get(generationId); + if (!row || !expectsGenerationContent(row) || !row.content_path) return false; + await this.assertNoBlockingRestoreIntent(row.node_id, row.stack_name); + + const transactionMeta = await captureGitSidePreimage(row.stack_name); + + try { + const context = await resolveComposeProjectContextForGeneration( + row.node_id, + row.stack_name, + row.content_path, + ); + const restoredManifest = await context.restoreFromContext(transactionMeta); + if (restoredManifest) { + await applyRestoredGenerationGitSide( + row.stack_name, + row.node_id, + row.content_path, + restoredManifest, + ); + } + await RollbackGenerationStore.commitRestoreTransaction( + row.node_id, + row.stack_name, + row.content_path, + ); + return true; + } catch (error) { + try { + await RollbackGenerationStore.reconcileInterruptedRestore( + row.node_id, + row.stack_name, + row.content_path, + ); + } catch (revertErr) { + console.error( + '[StackUpdateRecovery] Failed to revert after revertToGenerationContent error: %s', + sanitizeForLog(getErrorMessage(revertErr, 'unknown')), + ); + } + console.error( + '[StackUpdateRecovery] revertToGenerationContent failed for %s: %s', + sanitizeForLog(generationId), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return false; + } + } + public async probeRecoveredStack( nodeId: number, stackName: string, @@ -594,14 +1130,16 @@ export class StackUpdateRecoveryService { ): Promise { await new Promise((resolve) => setTimeout(resolve, RECOVERY_PROBE_DELAY_MS)); try { - const expected = parseServicesJson(servicesJson); + const expectedParsed = parseServicesJsonStrict(servicesJson); + if (!expectedParsed.ok) return false; + const expected = expectedParsed.services; const expectedRunning = new Map(); const expectedImageIds = new Map>(); const scaleZeroServices = new Set(); for (const svc of expected) { const imageIds = new Set(); - for (const replica of svc.replicas ?? []) { + for (const replica of svc.replicas) { if (replica.imageId?.trim()) imageIds.add(replica.imageId); } if (svc.scale > 0) { @@ -627,6 +1165,7 @@ export class StackUpdateRecoveryService { const runningByService = new Map(); for (const containerInfo of containers) { const labels = (containerInfo.Labels ?? {}) as Record; + if (isComposeOneOff(labels)) continue; const serviceName = labels['com.docker.compose.service']; const state = (containerInfo.State || '').toLowerCase(); @@ -673,10 +1212,14 @@ export class StackUpdateRecoveryService { } for (const [serviceName, need] of expectedRunning) { - if ((runningByService.get(serviceName) ?? 0) < need) { + if ((runningByService.get(serviceName) ?? 0) !== need) { return false; } } + // Scale-zero services never enter runningByService (failed above if running). + for (const serviceName of runningByService.keys()) { + if (!expectedRunning.has(serviceName)) return false; + } return true; } catch (error) { console.warn( @@ -695,8 +1238,17 @@ export class StackUpdateRecoveryService { */ public async retireGenerationArtifacts(row: StackUpdateRecoveryGenerationRow): Promise { if (row.artifacts_retired === 1) return true; - const services = parseServicesJson(row.services_json); - const tagsOk = await this.removeRollbackTags(row.node_id, collectRollbackTags(services)); + const servicesParsed = parseServicesJsonStrict(row.services_json); + const tags = servicesParsed.ok + ? collectRollbackTags(servicesParsed.services) + : scrapeRollbackTagsLenient(row.services_json); + if (!servicesParsed.ok) { + console.warn( + '[StackUpdateRecovery] Malformed services_json for %s; using best-effort tag scrape before override/content retirement', + sanitizeForLog(row.id), + ); + } + const tagsOk = await this.removeRollbackTags(row.node_id, tags); let overrideOk = true; if (row.override_path) { try { @@ -712,7 +1264,25 @@ export class StackUpdateRecoveryService { } } } - if (!tagsOk || !overrideOk) return false; + let contentOk = true; + const contentKey = row.content_path; + if (contentKey) { + try { + await RollbackGenerationStore.retireGenerationContent( + row.node_id, + row.stack_name, + contentKey, + ); + } catch (error) { + contentOk = false; + console.warn( + '[StackUpdateRecovery] Failed to retire generation content %s: %s', + sanitizeForLog(contentKey), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + } + if (!tagsOk || !overrideOk || !contentOk) return false; try { DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id); } catch (error) { @@ -739,6 +1309,15 @@ export class StackUpdateRecoveryService { try { const db = DatabaseService.getInstance(); const now = Date.now(); + // Revert any crash-interrupted content-store restores (intent still present). + try { + await this.sweepInterruptedRestores(db); + } catch (e) { + console.warn( + '[StackUpdateRecovery] Interrupted restore sweep failed: %s', + sanitizeForLog(getErrorMessage(e, 'unknown')), + ); + } let abandoned = 0; for (const row of db.listStaleStackUpdateRecoveryCandidates(now)) { if (await this.abandon(row.id)) abandoned += 1; @@ -779,10 +1358,30 @@ export class StackUpdateRecoveryService { if (row.is_current === 1 || row.status === 'recovery_required') continue; if (await this.retireGenerationArtifacts(row)) retired += 1; } - if (abandoned > 0 || flagged > 0 || capped > 0 || retired > 0) { + + let orphansRetired = 0; + let incompleteFlagged = 0; + try { + ({ orphansRetired, incompleteFlagged } = await this.reconcileGenerationContentDirs(db)); + } catch (contentError) { + console.warn( + '[StackUpdateRecovery] Generation content reconcile failed: %s', + sanitizeForLog(getErrorMessage(contentError, 'unknown')), + ); + } + + if ( + abandoned > 0 + || flagged > 0 + || capped > 0 + || retired > 0 + || orphansRetired > 0 + || incompleteFlagged > 0 + ) { console.log( `[StackUpdateRecovery] Reconciled ${abandoned} stale candidate(s), ` - + `${flagged} stuck generation(s), ${capped} generation(s) over cap, retired ${retired} artifact set(s)`, + + `${flagged} stuck generation(s), ${capped} generation(s) over cap, retired ${retired} artifact set(s), ` + + `${orphansRetired} orphan dir(s), ${incompleteFlagged} incomplete generation(s)`, ); } } catch (error) { @@ -793,6 +1392,106 @@ export class StackUpdateRecoveryService { } } + /** + * Retire orphan generation content dirs with no live row, and flag active rows + * whose content dir is missing or incomplete as recovery_required. + */ + private async reconcileGenerationContentDirs( + db: DatabaseService, + ): Promise<{ orphansRetired: number; incompleteFlagged: number }> { + let orphansRetired = 0; + let incompleteFlagged = 0; + const backupsRoot = getBackupBaseDir(); + let nodeEntries: string[] = []; + try { + nodeEntries = await fs.readdir(backupsRoot); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return { orphansRetired, incompleteFlagged }; + } + throw error; + } + + const incompleteStatuses = new Set(['active', 'restored_current', 'candidate']); + + for (const nodeEntry of nodeEntries) { + const nodeId = Number(nodeEntry); + if (!Number.isFinite(nodeId)) continue; + const nodeDir = path.join(backupsRoot, nodeEntry); + let stackEntries: string[] = []; + try { + stackEntries = await fs.readdir(nodeDir); + } catch { + continue; + } + for (const stackName of stackEntries) { + if (!isValidStackName(stackName)) continue; + const gensRoot = path.join(nodeDir, stackName, 'generations'); + let genIds: string[] = []; + try { + genIds = await fs.readdir(gensRoot); + } catch { + continue; + } + const rows = db.listStackUpdateRecoveryForStack(nodeId, stackName); + const liveKeys = new Set(); + for (const row of rows) { + if (row.artifacts_retired !== 0) continue; + if (row.content_path) liveKeys.add(row.content_path); + if (row.backup_slot_id) liveKeys.add(row.backup_slot_id); + liveKeys.add(row.id); + } + const nowMs = Date.now(); + for (const genId of genIds) { + if (genId.startsWith('staging-')) { + const stagingDir = path.join(gensRoot, genId); + try { + const st = await fs.stat(stagingDir); + if (nowMs - st.mtimeMs > STAGING_MAX_AGE_MS) { + await fs.rm(stagingDir, { recursive: true, force: true }); + orphansRetired += 1; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; + console.warn( + '[StackUpdateRecovery] Failed to retire stale staging dir %s/%s: %s', + sanitizeForLog(stackName), + sanitizeForLog(genId), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + continue; + } + if (!looksLikeGenerationUuid(genId) || liveKeys.has(genId)) continue; + try { + await RollbackGenerationStore.retireGenerationContent(nodeId, stackName, genId); + orphansRetired += 1; + } catch (error) { + console.warn( + '[StackUpdateRecovery] Failed to retire orphan generation %s/%s: %s', + sanitizeForLog(stackName), + sanitizeForLog(genId), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + } + for (const row of rows) { + if (row.artifacts_retired !== 0) continue; + if (row.status === 'abandoned' || row.status === 'superseded') continue; + if (!incompleteStatuses.has(row.status) && row.is_current !== 1) continue; + const contentKey = row.content_path; + if (!contentKey || !looksLikeGenerationUuid(contentKey)) continue; + const present = await generationContentPresent(row.node_id, row.stack_name, contentKey); + if (!present && row.status !== 'recovery_required') { + db.updateStackUpdateRecoveryGeneration(row.id, { status: 'recovery_required' }); + incompleteFlagged += 1; + } + } + } + } + return { orphansRetired, incompleteFlagged }; + } + /** Returns true when every tag is removed or already absent. */ private async removeRollbackTags(nodeId: number, tags: string[]): Promise { if (tags.length === 0) return true; diff --git a/backend/src/services/UpdateGuardService.ts b/backend/src/services/UpdateGuardService.ts index 40f0d002..9c663ae9 100644 --- a/backend/src/services/UpdateGuardService.ts +++ b/backend/src/services/UpdateGuardService.ts @@ -241,6 +241,35 @@ export class UpdateGuardService { withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness container probe')), ]); + const { StackUpdateRecoveryService, shortGenerationId } = await import('./StackUpdateRecoveryService'); + const { assessGenerationEligibility } = await import('./rollbackEligibility'); + const recoverySvc = StackUpdateRecoveryService.getInstance(); + const currentGen = recoverySvc.getCurrent(nodeId, stackName); + + const recoveryGeneration = currentGen + ? { exists: true as const, shortId: shortGenerationId(currentGen.id) } + : { exists: false as const }; + + const policyEligibility = currentGen + ? await this.collect('rollback eligibility', stackName, () => assessGenerationEligibility(currentGen)) + : null; + + const managedInputs = await this.collect('managed inputs', stackName, async () => { + const { resolveRollbackInventory } = await import('./rollbackInventory'); + const inventory = await resolveRollbackInventory(nodeId, stackName); + if (inventory.exactCoverage) { + return { + covered: true, + detail: `Exact authored-project coverage includes ${inventory.entries.length} managed path(s).`, + }; + } + return { + covered: false, + detail: inventory.coverageRefusal + || 'Exact authored-project coverage is incomplete for this stack.', + }; + }); + const items = buildRollbackItems({ backup, envSummary, @@ -255,15 +284,18 @@ export class UpdateGuardService { }, lastDeployAt, containers, + recoveryGeneration, + policyEligibility: policyEligibility === 'error' ? 'error' : policyEligibility, + managedInputs: managedInputs === 'error' ? 'error' : managedInputs, }, now); - // Partial-revert disclosure for Git-managed stacks: rollback restores only - // compose files and .env; the rest of the materialized project is not - // reverted by the backup slot. State the scope rather than imply a - // complete revert. + // Partial-revert disclosure for Git-managed stacks when exact generation + // coverage is not available. Prefer generation-backed wording when present. let note: string | undefined; const gitSource = db.getGitSource(stackName); - if (gitSource && (gitSource.manifest_state === 'active' || gitSource.manifest_state === 'partial' || gitSource.manifest_state === 'migrated')) { + if (currentGen) { + note = undefined; + } else if (gitSource && (gitSource.manifest_state === 'active' || gitSource.manifest_state === 'partial' || gitSource.manifest_state === 'migrated')) { note = 'This stack is Git-managed. Rollback restores compose files and .env; other materialized inputs are not reverted. Re-apply the previous revision from Git to restore them.'; } diff --git a/backend/src/services/WebhookService.ts b/backend/src/services/WebhookService.ts index 22ac714a..8f310de5 100644 --- a/backend/src/services/WebhookService.ts +++ b/backend/src/services/WebhookService.ts @@ -151,20 +151,25 @@ export class WebhookService { nodeId, stackName, lockAction, 'system', async () => { switch (action) { - case 'deploy': + case 'deploy': { await assertPolicyGateAllows( stackName, nodeId, buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }), ); - await compose.deployStack( + const deployResult = await compose.deployStack( stackName, undefined, atomic, { source: 'webhook', actor: 'system:webhook' }, ); - HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook'); + const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook'); + if (deployResult.recoveryId) { + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + StackUpdateRecoveryService.getInstance().linkGateOrRetain(deployResult.recoveryId, healthGateId); + } break; + } case 'restart': await compose.runCommand(stackName, 'restart'); break; diff --git a/backend/src/services/composeProjectContext.ts b/backend/src/services/composeProjectContext.ts index bfeafe07..ac568a74 100644 --- a/backend/src/services/composeProjectContext.ts +++ b/backend/src/services/composeProjectContext.ts @@ -1,9 +1,9 @@ /** - * Thin Compose project context for safe full-stack updates. + * Shared Compose project context for safe full-stack mutations. * - * Wraps the current authored compose argument path and atomic file backup/restore. - * When a richer shared Compose project context lands, migrate callers to that type; - * this module must not become a competing full-manifest resolver. + * Resolves the authored inventory (Git managed-project manifest or live stack + * discovery), captures/restores generation content through RollbackGenerationStore, + * and builds the exact Compose invocation used for deploy/update/rollback. */ import path from 'path'; import { randomUUID } from 'crypto'; @@ -11,9 +11,18 @@ import { ComposeService } from './ComposeService'; import { FileSystemService } from './FileSystemService'; import { buildEffectiveServiceModel } from './effectiveServiceModel'; import { getErrorMessage } from '../utils/errors'; +import { resolveRollbackInventory } from './rollbackInventory'; +import { RollbackGenerationStore } from './RollbackGenerationStore'; +import type { + RollbackGenerationManifest, + RollbackOperationKind, + RollbackRestoreTransactionMeta, +} from '../types/rollbackGeneration'; export type ImageReferenceKind = 'moving_tag' | 'digest_pinned' | 'none'; +export type BackupOperation = RollbackOperationKind; + const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i; export function classifyReferenceKind(declaredImageRef: string | null): ImageReferenceKind { @@ -34,11 +43,18 @@ export interface ComposeProjectContext { readonly nodeId: number; readonly stackName: string; readonly stackDir: string; + /** Generation id used as content-store key and backup_slot_id on the DB row. */ backupSlotId: string | null; toComposeArgs(action: string[]): Promise; validateForMutation(): Promise; - backupFromContext(operation: 'update' | 'deployment'): Promise; - restoreFromContext(): Promise; + /** + * Capture a staged generation. When exactCoverage is required and inventory + * refuses it, throws before writing any generation content. + */ + backupFromContext(operation: BackupOperation): Promise; + restoreFromContext( + transactionMeta?: RollbackRestoreTransactionMeta, + ): Promise; resolveServiceImageMap(): Promise>; } @@ -60,15 +76,63 @@ class AuthoredComposeProjectContext implements ComposeProjectContext { await requireRenderableModel(this.nodeId, this.stackName); } - async backupFromContext(_operation: 'update' | 'deployment'): Promise { - await FileSystemService.getInstance(this.nodeId).backupStackFiles(this.stackName); - const slotId = randomUUID(); - this.backupSlotId = slotId; - return slotId; + async backupFromContext(operation: BackupOperation): Promise { + const inventory = await resolveRollbackInventory(this.nodeId, this.stackName); + if (!inventory.exactCoverage) { + throw Object.assign( + new Error( + inventory.coverageRefusal + || 'Exact authored-project rollback coverage is unavailable for this stack', + ), + { code: 'ROLLBACK_COVERAGE_UNAVAILABLE' }, + ); + } + + // Do not refresh the legacy single-slot backup. Clearing that reused slot + // would destroy a pre-migration recovery point if a later capture step fails. + + const generationId = randomUUID(); + await RollbackGenerationStore.captureGeneration({ + nodeId: this.nodeId, + stackName: this.stackName, + generationId, + inventory, + operationKind: operation, + }); + this.backupSlotId = generationId; + return generationId; } - async restoreFromContext(): Promise { - await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName); + async restoreFromContext( + transactionMeta?: RollbackRestoreTransactionMeta, + ): Promise { + const generationId = this.backupSlotId; + if (!generationId) { + // Legacy pre-migration restore: only when no generation id is bound. + await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName); + return; + } + + const present = await RollbackGenerationStore.verifyGenerationContent( + this.nodeId, + this.stackName, + generationId, + ); + if (!present) { + throw Object.assign( + new Error('Recovery generation content is missing or incomplete'), + { code: 'GENERATION_CONTENT_MISSING' }, + ); + } + + const inventory = await resolveRollbackInventory(this.nodeId, this.stackName); + return RollbackGenerationStore.restoreGeneration( + this.nodeId, + this.stackName, + generationId, + inventory.entries.map((e) => e.relativePath), + transactionMeta, + ); } async resolveServiceImageMap(): Promise> { @@ -89,6 +153,17 @@ export async function resolveComposeProjectContext( return new AuthoredComposeProjectContext(nodeId, stackName, stackDir); } +/** Bind an existing generation id onto a fresh context for restore. */ +export async function resolveComposeProjectContextForGeneration( + nodeId: number, + stackName: string, + generationId: string, +): Promise { + const ctx = await resolveComposeProjectContext(nodeId, stackName); + ctx.backupSlotId = generationId; + return ctx; +} + export function describeContextError(error: unknown): string { return getErrorMessage(error, 'Compose project context failed'); } diff --git a/backend/src/services/recoveryServicesJson.ts b/backend/src/services/recoveryServicesJson.ts new file mode 100644 index 00000000..05e04aa6 --- /dev/null +++ b/backend/src/services/recoveryServicesJson.ts @@ -0,0 +1,175 @@ +/** + * Structural validation for stack-update recovery services_json payloads. + * Fail closed on any unexpected shape so eligibility, compensate, and probe + * share one authoritative parser. + */ +import type { ImageReferenceKind } from './composeProjectContext'; + +export interface StackRecoveryReplicaCapture { + containerId: string | null; + imageId: string | null; + repoDigest: string | null; + state: 'running' | 'stopped' | 'none'; + rollbackTag: string | null; +} + +export interface StackRecoveryServiceCapture { + serviceName: string; + /** Observed running replica count at capture (supported restore scale). */ + scale: number; + hasBuild: boolean; + declaredImageRef: string | null; + referenceKind: ImageReferenceKind; + replicas: StackRecoveryReplicaCapture[]; +} + +export type ParsedServicesJson = + | { ok: true; services: StackRecoveryServiceCapture[] } + | { ok: false }; + +const REPLICA_STATES = new Set(['running', 'stopped', 'none']); +const REFERENCE_KINDS = new Set(['moving_tag', 'digest_pinned', 'none']); + +function isNonEmptyString(v: unknown): v is string { + return typeof v === 'string' && v.trim().length > 0; +} + +/** Accepts null/undefined/string; rejects any other type. */ +function isNullableString(v: unknown): v is string | null | undefined { + return v === null || v === undefined || typeof v === 'string'; +} + +function asNullableString(v: unknown): string | null { + return typeof v === 'string' ? v : null; +} + +function parseReplicaCapture(raw: unknown): StackRecoveryReplicaCapture | null { + if (!raw || typeof raw !== 'object') return null; + const r = raw as Record; + if (!isNullableString(r.containerId) + || !isNullableString(r.imageId) + || !isNullableString(r.repoDigest) + || !isNullableString(r.rollbackTag)) { + return null; + } + if (typeof r.state !== 'string' || !REPLICA_STATES.has(r.state)) return null; + + const state = r.state as StackRecoveryReplicaCapture['state']; + const imageId = asNullableString(r.imageId); + // Running/stopped replicas must carry a protectable image identity. + if ((state === 'running' || state === 'stopped') && !imageId?.trim()) { + return null; + } + return { + containerId: asNullableString(r.containerId), + imageId, + repoDigest: asNullableString(r.repoDigest), + state, + rollbackTag: asNullableString(r.rollbackTag), + }; +} + +function parseServiceCapture(raw: unknown): StackRecoveryServiceCapture | null { + if (!raw || typeof raw !== 'object') return null; + const s = raw as Record; + if (!isNonEmptyString(s.serviceName)) return null; + if (typeof s.scale !== 'number' || !Number.isInteger(s.scale) || s.scale < 0) return null; + if (typeof s.hasBuild !== 'boolean') return null; + if (!isNullableString(s.declaredImageRef)) return null; + if (typeof s.referenceKind !== 'string' || !REFERENCE_KINDS.has(s.referenceKind as ImageReferenceKind)) { + return null; + } + if (!Array.isArray(s.replicas)) return null; + + const replicas: StackRecoveryReplicaCapture[] = []; + for (const item of s.replicas) { + const replica = parseReplicaCapture(item); + if (!replica) return null; + replicas.push(replica); + } + return { + serviceName: s.serviceName, + scale: s.scale, + hasBuild: s.hasBuild, + declaredImageRef: asNullableString(s.declaredImageRef), + referenceKind: s.referenceKind as ImageReferenceKind, + replicas, + }; +} + +/** Strict structural validation for recovery services_json (fail closed). */ +export function parseServicesJsonStrict(raw: string): ParsedServicesJson { + try { + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) return { ok: false }; + const services: StackRecoveryServiceCapture[] = []; + for (const item of parsed) { + const svc = parseServiceCapture(item); + if (!svc) return { ok: false }; + services.push(svc); + } + return { ok: true, services }; + } catch { + return { ok: false }; + } +} + +/** Lenient parse for callers that treat empty as no services (legacy). Prefer strict. */ +export function parseServicesJson(raw: string): StackRecoveryServiceCapture[] { + const parsed = parseServicesJsonStrict(raw); + return parsed.ok ? parsed.services : []; +} + +export function collectImageIds(services: StackRecoveryServiceCapture[]): string[] { + const ids = new Set(); + for (const svc of services) { + for (const replica of svc.replicas) { + if (replica.imageId?.trim()) ids.add(replica.imageId); + } + } + return [...ids]; +} + +export function collectImageIdsFromServicesJson(servicesJson: string): string[] { + return collectImageIds(parseServicesJson(servicesJson)); +} + +export function collectRollbackTags(services: StackRecoveryServiceCapture[]): string[] { + const tags = new Set(); + for (const svc of services) { + for (const replica of svc.replicas) { + if (replica.rollbackTag?.trim()) tags.add(replica.rollbackTag); + } + } + return [...tags]; +} + +/** + * Best-effort rollbackTag scrape for cleanup paths. Prefer strict parse; on + * structural failure walk nested objects for string rollbackTag fields so + * opaque holds are still removed when possible. + */ +export function scrapeRollbackTagsLenient(raw: string): string[] { + const strict = parseServicesJsonStrict(raw); + if (strict.ok) return collectRollbackTags(strict.services); + try { + const parsed: unknown = JSON.parse(raw); + const tags = new Set(); + const walk = (value: unknown): void => { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item); + return; + } + const obj = value as Record; + if (typeof obj.rollbackTag === 'string' && obj.rollbackTag.trim()) { + tags.add(obj.rollbackTag); + } + for (const nested of Object.values(obj)) walk(nested); + }; + walk(parsed); + return [...tags]; + } catch { + return []; + } +} diff --git a/backend/src/services/rollbackEligibility.ts b/backend/src/services/rollbackEligibility.ts new file mode 100644 index 00000000..966d4ce3 --- /dev/null +++ b/backend/src/services/rollbackEligibility.ts @@ -0,0 +1,178 @@ +/** + * Rollback restore eligibility (fail closed on known-bad evidence). + * + * Pure evaluateRollbackEligibility maps known signals to a verdict. + * assessGenerationEligibility gathers best-effort evidence for a recovery row. + */ +import type { StackUpdateRecoveryGenerationRow } from './DatabaseService'; +import DockerController from './DockerController'; +import { enforcePolicyForImageRefs } from './PolicyEnforcement'; +import { + collectImageIds, + collectRollbackTags, + parseServicesJsonStrict, +} from './recoveryServicesJson'; +import { RollbackGenerationStore } from './RollbackGenerationStore'; +import { getErrorMessage } from '../utils/errors'; +import { sanitizeForLog } from '../utils/safeLog'; + +type HeldImagesParse = + | { ok: true; ids: string[]; rollbackTags: string[] } + | { ok: false }; + +function parseHeldImageState(servicesJson: string): HeldImagesParse { + const parsed = parseServicesJsonStrict(servicesJson); + if (!parsed.ok) return { ok: false }; + return { + ok: true, + ids: collectImageIds(parsed.services), + rollbackTags: collectRollbackTags(parsed.services), + }; +} + +export type RollbackEligibilityVerdict = + | 'eligible' + | 'eligible_with_warning' + | 'prohibited' + | 'unknown'; + +export interface RollbackEligibilityInput { + /** null = unknown */ + generationIntegrityOk: boolean | null; + heldImagesPresent: boolean | null; + /** true when known blocked; null = unknown */ + securityPostureBlocked: boolean | null; +} + +/** + * Rules (fail closed on known bad): + * - securityPostureBlocked === true → prohibited + * - generationIntegrityOk === false → prohibited + * - heldImagesPresent === false → eligible_with_warning + * - any remaining null → unknown (unless already prohibited) + * - else eligible + */ +export function evaluateRollbackEligibility( + input: RollbackEligibilityInput, +): RollbackEligibilityVerdict { + if (input.securityPostureBlocked === true || input.generationIntegrityOk === false) { + return 'prohibited'; + } + if (input.heldImagesPresent === false) return 'eligible_with_warning'; + if ( + input.generationIntegrityOk === null + || input.heldImagesPresent === null + || input.securityPostureBlocked === null + ) { + return 'unknown'; + } + return 'eligible'; +} + +async function checkGenerationIntegrity( + row: StackUpdateRecoveryGenerationRow, +): Promise { + // Only explicit content_path generations use the content store. + const contentKey = row.content_path; + if (!contentKey) return null; + try { + return await RollbackGenerationStore.verifyGenerationContent( + row.node_id, + row.stack_name, + contentKey, + ); + } catch (error) { + console.warn( + '[RollbackEligibility] Integrity check failed for %s: %s', + sanitizeForLog(row.id), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return null; + } +} + +async function inspectImagePresent( + docker: ReturnType['getDocker']>, + ref: string, +): Promise { + try { + await docker.getImage(ref).inspect(); + return true; + } catch (error) { + const status = (error as { statusCode?: number }).statusCode; + const message = getErrorMessage(error, '').toLowerCase(); + if (status === 404 || message.includes('no such image') || message.includes('not found')) { + return false; + } + throw error; + } +} + +/** + * Held recovery launch requires both underlying image ids and opaque rollback + * tags used by the recovery override to still resolve locally. + */ +async function checkHeldImagesPresent( + row: StackUpdateRecoveryGenerationRow, + held: HeldImagesParse, +): Promise { + if (!held.ok) return null; + const refs = [...new Set([...held.ids, ...held.rollbackTags])]; + if (refs.length === 0) return true; + try { + const docker = DockerController.getInstance(row.node_id).getDocker(); + for (const ref of refs) { + if (!(await inspectImagePresent(docker, ref))) return false; + } + return true; + } catch (error) { + console.warn( + '[RollbackEligibility] Docker check failed for %s: %s', + sanitizeForLog(row.id), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return null; + } +} + +async function checkSecurityPostureBlocked( + row: StackUpdateRecoveryGenerationRow, + held: HeldImagesParse, +): Promise { + if (!held.ok) return null; + const refs = [...new Set([...held.ids, ...held.rollbackTags])]; + if (refs.length === 0) return false; + try { + const gate = await enforcePolicyForImageRefs(row.stack_name, row.node_id, refs, { + bypass: false, + actor: 'rollback-eligibility', + auditMethod: 'GET', + auditPath: '/api/stacks/rollback-eligibility', + }); + return !gate.ok; + } catch (error) { + console.warn( + '[RollbackEligibility] Security posture check failed for %s: %s', + sanitizeForLog(row.id), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return null; + } +} + +/** Best-effort eligibility for a recovery generation row. */ +export async function assessGenerationEligibility( + row: StackUpdateRecoveryGenerationRow, +): Promise { + const held = parseHeldImageState(row.services_json); + // Malformed recovery state cannot be assessed safely; refuse restore. + if (!held.ok) return 'prohibited'; + const generationIntegrityOk = await checkGenerationIntegrity(row); + const heldImagesPresent = await checkHeldImagesPresent(row, held); + const securityPostureBlocked = await checkSecurityPostureBlocked(row, held); + return evaluateRollbackEligibility({ + generationIntegrityOk, + heldImagesPresent, + securityPostureBlocked, + }); +} diff --git a/backend/src/services/rollbackInventory.ts b/backend/src/services/rollbackInventory.ts new file mode 100644 index 00000000..83986655 --- /dev/null +++ b/backend/src/services/rollbackInventory.ts @@ -0,0 +1,565 @@ +/** + * Resolve the authored-project file set that an atomic rollback generation + * must capture. Git-managed stacks consume the managed-project manifest; + * authored stacks rediscover against the live stack directory. + */ +import { promises as fsPromises, readFileSync } from 'fs'; +import path from 'path'; +import { DatabaseService, type StackGitSource } from './DatabaseService'; +import { FileSystemService } from './FileSystemService'; +import { GitProjectManifestService } from './GitProjectManifestService'; +import { collectManifestFilePaths } from '../helpers/manifestFilePaths'; +import { isHostAbsolutePath, parseDeclaredInputs } from '../helpers/composeInputParse'; +import { isValidRelativeStackPath, isValidStackName } from '../utils/validation'; +import { authoredComposeEnvFileArgs, authoredComposeFileArgs } from '../utils/authoredComposeArgs'; +import type { + ComposeInputEntry, + GitProjectManifest, + InputSensitivity, +} from '../types/gitProjectManifest'; +import type { + ResolvedRollbackInventory, + RollbackEntryKind, + RollbackEntryProvenance, + RollbackInvocationRecord, +} from '../types/rollbackGeneration'; + +const ROOT_COMPOSE_FILENAMES = [ + 'compose.yaml', + 'compose.yml', + 'docker-compose.yaml', + 'docker-compose.yml', +] as const; + +const DOT_ENV = '.' + 'env'; + +const COMPOSE_INVOCATION_KINDS = new Set([ + 'compose-root', + 'implicit-override', + 'explicit', + 'include', + 'extends', +]); + +type InventoryAccum = { + relativePath: string; + dependencyKind: RollbackEntryKind; + provenance: RollbackEntryProvenance; + sensitivity: InputSensitivity; + absolutePath: string | null; +}; + +function posixRel(rel: string): string { + return rel.replace(/\\/g, '/').replace(/^\.\//, ''); +} + +function foldedKey(rel: string): string { + return posixRel(rel).toLowerCase(); +} + +/** + * Prefer exact POSIX paths as map keys so case-distinct Linux paths are kept. + * When two different paths collide under case-folding, record a refusal note + * instead of silently merging them. + */ +function upsertEntry( + map: Map, + foldedOwners: Map, + entry: InventoryAccum, + caseCollisions: string[], +): void { + const exact = posixRel(entry.relativePath); + const folded = foldedKey(exact); + const owner = foldedOwners.get(folded); + if (owner !== undefined && owner !== exact) { + caseCollisions.push(`Case-colliding managed paths "${owner}" and "${exact}"`); + return; + } + const prev = map.get(exact); + if (!prev) { + map.set(exact, { ...entry, relativePath: exact }); + foldedOwners.set(folded, exact); + return; + } + if (prev.absolutePath !== null && entry.absolutePath === null) return; + map.set(exact, { + ...entry, + relativePath: exact, + absolutePath: entry.absolutePath ?? prev.absolutePath, + dependencyKind: prev.dependencyKind === 'compose-root' && entry.dependencyKind !== 'compose-root' + ? entry.dependencyKind + : entry.dependencyKind, + sensitivity: higherSensitivity(entry.sensitivity, prev.sensitivity), + }); +} + +function resolveStackRoot(fsSvc: FileSystemService, stackName: string): string { + if (!isValidStackName(stackName)) { + throw Object.assign(new Error('Invalid stack name'), { code: 'INVALID_STACK_NAME' }); + } + // Canonical js/path-injection barrier: resolve + startsWith. + // CodeQL does not credit isPathWithinBase helpers at later sinks. + const base = path.resolve(fsSvc.getBaseDir()); + const stackRoot = path.resolve(base, stackName); + if (!stackRoot.startsWith(base + path.sep)) { + throw Object.assign(new Error('Stack name escapes compose directory'), { code: 'INVALID_PATH' }); + } + return stackRoot; +} + +function resolveStackRel( + stackRoot: string, + relRaw: string, +): { relativePath: string; absolutePath: string } | null { + const relativePath = posixRel(relRaw); + if (!relativePath || !isValidRelativeStackPath(relativePath)) return null; + // Join-time containment; callers still re-check at each fs sink. + const baseResolved = path.resolve(stackRoot); + const absolutePath = path.resolve(baseResolved, relativePath); + if (!absolutePath.startsWith(baseResolved + path.sep)) return null; + return { relativePath, absolutePath }; +} + +async function pathExistsAsFile(stackRoot: string, relativePath: string): Promise { + // Inline barrier at the lstat sink. + const baseResolved = path.resolve(stackRoot); + const abs = path.resolve(baseResolved, relativePath); + if (!abs.startsWith(baseResolved + path.sep)) return false; + try { + const st = await fsPromises.lstat(abs); + return st.isFile() || st.isSymbolicLink(); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw e; + } +} + +function authoredSensitivity(kind: RollbackEntryKind): InputSensitivity { + if (kind === 'secret' || kind === 'config' || kind === 'build-secret') return 'high'; + if ( + kind === 'env_file' + || kind === 'include-env' + || kind === 'interpolation-env' + || kind === 'sync-env' + || kind === 'project-env' + || kind === 'label_file' + ) { + return 'medium'; + } + return 'low'; +} + +function higherSensitivity(a: InputSensitivity, b: InputSensitivity): InputSensitivity { + if (a === 'high' || b === 'high') return 'high'; + if (a === 'medium' || b === 'medium') return 'medium'; + return 'low'; +} + +function appliedDeploySpecString( + spec: { files: string[]; contextDir: string | null } | null | undefined, +): string | null { + if (!spec) return null; + return JSON.stringify(spec); +} + +function refusedGitInventory( + gitSource: StackGitSource, + emptyInvocation: RollbackInvocationRecord, + coverageRefusal: string, + manifestVersion: number | null = gitSource.manifest_version, +): ResolvedRollbackInventory { + return { + entries: [], + invocation: emptyInvocation, + git: { + repoUrl: gitSource.repo_url, + branch: gitSource.branch, + commitSha: gitSource.last_applied_commit_sha || '', + manifestVersion, + }, + appliedDeploySpec: appliedDeploySpecString(gitSource.applied_deploy_spec), + lastAppliedContentHash: gitSource.last_applied_content_hash, + manifestState: gitSource.manifest_state, + manifestGeneration: gitSource.manifest_generation, + exactCoverage: false, + coverageRefusal, + }; +} + +function isGitManifest( + value: GitProjectManifest | { corrupt: string } | null, +): value is GitProjectManifest { + return value !== null && !('corrupt' in value); +} + +function sensitivityForManifestPath( + manifest: GitProjectManifest, + rel: string, +): { kind: RollbackEntryKind; sensitivity: InputSensitivity; provenance: RollbackEntryProvenance } { + const key = foldedKey(rel); + const input = manifest.inputs.find( + (i: ComposeInputEntry) => i.materializedPath !== null && foldedKey(i.materializedPath) === key, + ); + if (input) { + return { + kind: input.dependencyKind, + sensitivity: input.sensitivity, + provenance: input.provenance, + }; + } + return { kind: 'other', sensitivity: 'low', provenance: 'fetch' }; +} + +async function resolveGitInventory( + nodeId: number, + stackName: string, + stackRoot: string, +): Promise { + const gitSource = DatabaseService.getInstance().getGitSource(stackName); + if (!gitSource) return null; + + const emptyInvocation: RollbackInvocationRecord = { + composeArgsPrefix: [], + projectDirectory: null, + projectName: stackName, + explicitComposeFiles: [], + meshOverrideRelativePath: null, + meshEnabled: false, + }; + + const read = await GitProjectManifestService.getInstance().readManifest( + stackName, + gitSource.repo_url, + gitSource.branch, + ); + if (!isGitManifest(read)) { + const corrupt = Boolean(read && 'corrupt' in read); + const reason = corrupt + ? `Managed-project manifest is unreadable (${(read as { corrupt: string }).corrupt}). Fix or re-link the Git source before capturing rollback coverage.` + : 'Managed-project manifest is missing. Pull or re-link the Git source before capturing rollback coverage.'; + + const established = Boolean( + gitSource.applied_deploy_spec + || gitSource.last_applied_content_hash + || gitSource.last_applied_commit_sha, + ); + + // Established missing/corrupt manifesto: fail closed. applied_deploy_spec + // alone cannot claim exact managed-input coverage (includes, extends, env, + // labels, configs, secrets, and build inputs are omitted). + if (established || corrupt) { + return refusedGitInventory(gitSource, emptyInvocation, reason); + } + + // First apply (no applied revision yet): signal incomplete Git coverage so + // resolveRollbackInventory can merge authored disk files with this identity. + return refusedGitInventory(gitSource, emptyInvocation, reason, null); + } + + const map = new Map(); + const foldedOwners = new Map(); + const caseCollisions: string[] = []; + for (const rel of collectManifestFilePaths(read)) { + const resolved = resolveStackRel(stackRoot, rel); + if (!resolved) continue; + const meta = sensitivityForManifestPath(read, resolved.relativePath); + const exists = await pathExistsAsFile(stackRoot, resolved.relativePath); + upsertEntry(map, foldedOwners, { + relativePath: resolved.relativePath, + dependencyKind: meta.kind, + provenance: meta.provenance, + sensitivity: meta.sensitivity, + absolutePath: exists ? resolved.absolutePath : null, + }, caseCollisions); + } + + const refused = read.refusals.length > 0 + || read.counts.refused > 0 + || read.state === 'unsupported' + || read.state === 'partial' + || caseCollisions.length > 0; + const coverageRefusal = refused + ? (caseCollisions[0] + ?? read.refusals[0]?.reason + ?? `Managed-project manifest state "${read.state}" does not claim exact coverage`) + : null; + + let meshEnabled = false; + let meshReadFailed: string | null = null; + try { + meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName); + } catch (e) { + meshReadFailed = `Could not read Mesh enablement: ${(e as Error).message}`; + } + + const invocation: RollbackInvocationRecord = { + composeArgsPrefix: [...read.project.invocation], + projectDirectory: read.project.effectiveProjectDir, + projectName: read.project.projectName || stackName, + explicitComposeFiles: [...read.project.composeFiles], + meshOverrideRelativePath: null, + meshEnabled, + }; + + const meshRefused = meshReadFailed !== null; + return { + entries: [...map.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)), + invocation, + git: { + repoUrl: read.repo.url, + branch: read.repo.branch, + commitSha: read.resolvedRevision.commitSha || gitSource.last_applied_commit_sha || '', + manifestVersion: read.manifestVersion, + }, + appliedDeploySpec: appliedDeploySpecString(gitSource.applied_deploy_spec), + lastAppliedContentHash: gitSource.last_applied_content_hash, + manifestState: gitSource.manifest_state, + manifestGeneration: gitSource.manifest_generation, + exactCoverage: !refused && !meshRefused, + coverageRefusal: coverageRefusal ?? meshReadFailed, + }; +} + +async function resolveAuthoredInventory( + nodeId: number, + stackName: string, + stackRoot: string, + fsSvc: FileSystemService, +): Promise { + const map = new Map(); + const foldedOwners = new Map(); + const coverageNotes: string[] = []; + const caseCollisions: string[] = []; + + const composePaths: string[] = []; + for (const name of ROOT_COMPOSE_FILENAMES) { + const resolved = resolveStackRel(stackRoot, name); + if (!resolved) continue; + if (!(await pathExistsAsFile(stackRoot, resolved.relativePath))) continue; + composePaths.push(resolved.relativePath); + upsertEntry(map, foldedOwners, { + relativePath: resolved.relativePath, + dependencyKind: 'compose-root', + provenance: 'authored', + sensitivity: 'low', + absolutePath: resolved.absolutePath, + }, caseCollisions); + } + + const overrideName = await fsSvc.getOverrideFilename(stackName); + if (overrideName) { + const resolved = resolveStackRel(stackRoot, overrideName); + if (resolved && await pathExistsAsFile(stackRoot, resolved.relativePath)) { + if (!composePaths.includes(resolved.relativePath)) { + composePaths.push(resolved.relativePath); + } + upsertEntry(map, foldedOwners, { + relativePath: resolved.relativePath, + dependencyKind: 'implicit-override', + provenance: 'authored', + sensitivity: 'low', + absolutePath: resolved.absolutePath, + }, caseCollisions); + } + } + + const envCandidates = new Set([DOT_ENV]); + for (const f of DatabaseService.getInstance().getStackProjectEnvFiles(nodeId, stackName)) { + envCandidates.add(posixRel(f)); + } + for (const envFile of envCandidates) { + const resolved = resolveStackRel(stackRoot, envFile); + if (!resolved) continue; + if (!(await pathExistsAsFile(stackRoot, resolved.relativePath))) continue; + upsertEntry(map, foldedOwners, { + relativePath: resolved.relativePath, + dependencyKind: envFile === DOT_ENV ? 'interpolation-env' : 'project-env', + provenance: 'authored', + sensitivity: authoredSensitivity(envFile === DOT_ENV ? 'interpolation-env' : 'project-env'), + absolutePath: resolved.absolutePath, + }, caseCollisions); + } + + const readCallback = (repoPath: string): string | null => { + const relativePath = posixRel(repoPath); + if (!relativePath || !isValidRelativeStackPath(relativePath)) return null; + // Inline barrier at the readFileSync sink. + const baseResolved = path.resolve(stackRoot); + const abs = path.resolve(baseResolved, relativePath); + if (!abs.startsWith(baseResolved + path.sep)) return null; + try { + return readFileSync(abs, 'utf8'); + } catch { + return null; + } + }; + + if (composePaths.length > 0) { + const orderedContents: Array<{ path: string; content: string }> = []; + for (const rel of composePaths) { + if (!isValidRelativeStackPath(rel)) continue; + // Inline barrier at the readFile sink (same form as readCallback). + const baseResolved = path.resolve(stackRoot); + const abs = path.resolve(baseResolved, rel); + if (!abs.startsWith(baseResolved + path.sep)) continue; + try { + const content = await fsPromises.readFile(abs, 'utf8'); + orderedContents.push({ path: rel, content }); + } catch (e) { + coverageNotes.push(`Could not read compose file ${rel}: ${(e as Error).message}`); + } + } + + if (orderedContents.length > 0) { + const parsed = parseDeclaredInputs(orderedContents, { + projectRoot: null, + read: readCallback, + }); + + if (parsed.parseErrors.length > 0) { + coverageNotes.push(...parsed.parseErrors); + } + if (parsed.dynamic.length > 0) { + coverageNotes.push( + `${parsed.dynamic.length} dynamic path declaration(s) cannot be captured exactly`, + ); + } + + for (const input of parsed.inputs) { + const hostAbs = (input.sourcePath !== null && isHostAbsolutePath(input.sourcePath)) + || input.baseDir === 'host' + || input.materializedPath === null; + + if (hostAbs) { + if (input.kind === 'include' || input.kind === 'extends') { + coverageNotes.push( + `Host-absolute ${input.kind} path cannot be captured for exact rollback`, + ); + } + continue; + } + + const candidate = input.materializedPath ?? input.sourcePath; + if (!candidate) continue; + const resolved = resolveStackRel(stackRoot, candidate); + if (!resolved) continue; + if (!(await pathExistsAsFile(stackRoot, resolved.relativePath))) continue; + + const kind = input.kind; + upsertEntry(map, foldedOwners, { + relativePath: resolved.relativePath, + dependencyKind: kind, + provenance: 'authored', + sensitivity: authoredSensitivity(kind), + absolutePath: resolved.absolutePath, + }, caseCollisions); + } + } + } + + let composeArgsPrefix: string[] = []; + try { + composeArgsPrefix = [ + ...authoredComposeFileArgs(stackName, nodeId), + ...(await authoredComposeEnvFileArgs(stackName, nodeId)), + ]; + } catch (e) { + coverageNotes.push(`Could not build compose invocation args: ${(e as Error).message}`); + } + + const explicitComposeFiles = [...map.values()] + .filter((e) => COMPOSE_INVOCATION_KINDS.has(e.dependencyKind)) + .map((e) => e.relativePath) + .sort((a, b) => a.localeCompare(b)); + + if (caseCollisions.length > 0) { + coverageNotes.push(caseCollisions[0]); + } + + let meshEnabled = false; + try { + meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(nodeId, stackName); + } catch (e) { + coverageNotes.push(`Could not read Mesh enablement: ${(e as Error).message}`); + } + + const exactCoverage = coverageNotes.length === 0 && composePaths.length > 0; + const coverageRefusal = exactCoverage + ? null + : (coverageNotes[0] + ?? (composePaths.length === 0 + ? 'No compose file found in the stack directory' + : 'Exact coverage unavailable')); + + return { + entries: [...map.values()].sort((a, b) => a.relativePath.localeCompare(b.relativePath)), + invocation: { + composeArgsPrefix, + projectDirectory: null, + projectName: stackName, + explicitComposeFiles: explicitComposeFiles.length > 0 ? explicitComposeFiles : composePaths, + meshOverrideRelativePath: null, + meshEnabled, + }, + git: null, + appliedDeploySpec: null, + lastAppliedContentHash: null, + manifestState: null, + manifestGeneration: null, + exactCoverage, + coverageRefusal, + }; +} + +/** + * Prefer an exact Git-managed inventory when available. When the manifesto is + * missing on a true first apply (no applied revision yet), merge authored disk + * discovery with the Git identity fields so capture preserves nullable Git + * state. Established missing/corrupt manifesto cases fail closed via + * resolveGitInventory and are not overwritten by authored exactCoverage. + */ +export async function resolveRollbackInventory( + nodeId: number, + stackName: string, +): Promise { + const fsSvc = FileSystemService.getInstance(nodeId); + const stackRoot = resolveStackRoot(fsSvc, stackName); + + try { + const gitInventory = await resolveGitInventory(nodeId, stackName, stackRoot); + if (gitInventory?.exactCoverage) return gitInventory; + + const authored = await resolveAuthoredInventory(nodeId, stackName, stackRoot, fsSvc); + + if (gitInventory && !gitInventory.exactCoverage) { + const established = Boolean( + gitInventory.appliedDeploySpec + || gitInventory.lastAppliedContentHash + || gitInventory.git?.commitSha, + ); + const firstApplyCorrupt = Boolean(gitInventory.coverageRefusal?.includes('unreadable')); + // Established or corrupt first-apply: fail closed. Otherwise merge Git + // identity onto authored exact coverage for a true first apply. + if (established || firstApplyCorrupt || !authored.exactCoverage) { + return gitInventory; + } + return { + ...authored, + git: gitInventory.git, + appliedDeploySpec: gitInventory.appliedDeploySpec, + lastAppliedContentHash: gitInventory.lastAppliedContentHash, + manifestState: gitInventory.manifestState, + manifestGeneration: gitInventory.manifestGeneration, + }; + } + + if (authored.exactCoverage) return authored; + return gitInventory ?? authored; + } catch (e) { + console.error( + '[rollbackInventory] Failed to resolve inventory:', + (e as Error).message, + ); + throw e; + } +} diff --git a/backend/src/services/updateGuard/failureClassifier.ts b/backend/src/services/updateGuard/failureClassifier.ts index 9ccaefe9..47b1e06a 100644 --- a/backend/src/services/updateGuard/failureClassifier.ts +++ b/backend/src/services/updateGuard/failureClassifier.ts @@ -91,6 +91,18 @@ const RULES: ClassifierRule[] = [ suggestion: 'Review the compose file syntax (Compose Doctor can pinpoint the issue), then retry.', pattern: /yaml:|mapping values are not allowed|cannot unmarshal|additional propert|undefined volume|undefined network|invalid compose/i, }, + { + reason: 'mixed_replica_images', + label: 'Mixed replica images', + suggestion: 'Bring every replica of the service onto the same image, then retry.', + pattern: /mixed replica images/i, + }, + { + reason: 'rollback_coverage_unavailable', + label: 'Exact rollback coverage unavailable', + suggestion: 'Remove host-absolute include or extends paths, or capture from a project Sencho can enumerate completely, then retry.', + pattern: /cannot be captured for exact rollback|rollback coverage is unavailable/i, + }, ]; const UNKNOWN_FAILURE: FailureClassification = { diff --git a/backend/src/services/updateGuard/readiness.ts b/backend/src/services/updateGuard/readiness.ts index 2a04d1d7..17f30dca 100644 --- a/backend/src/services/updateGuard/readiness.ts +++ b/backend/src/services/updateGuard/readiness.ts @@ -331,13 +331,55 @@ export interface RollbackInputs { /** Timestamp of the most recent deploy_success activity event, if any. */ lastDeployAt: number | null | Errored; containers: ContainerProbe[] | Errored; + /** + * Current recovery generation, when one exists. When present, compose_source + * readiness is driven by this generation rather than the legacy backup slot. + */ + recoveryGeneration: { exists: boolean; shortId?: string } | Errored | null; + /** Eligibility verdict for the current generation, when assessed. */ + policyEligibility: 'eligible' | 'eligible_with_warning' | 'prohibited' | 'unknown' | Errored | null; + /** Whether managed authored inputs are covered by exact inventory. */ + managedInputs: { covered: boolean; detail: string } | Errored | null; } export function buildRollbackItems(inputs: RollbackInputs, now: number): RollbackReadinessItem[] { const items: RollbackReadinessItem[] = []; + const recoveryRaw = inputs.recoveryGeneration; + const recoveryInfo = recoveryRaw !== null && recoveryRaw !== 'error' ? recoveryRaw : null; + const recoveryExists = !!recoveryInfo?.exists; const backupExists = inputs.backup !== 'error' && inputs.backup.exists; - if (inputs.backup === 'error') { + + if (recoveryRaw === 'error') { + items.push({ id: 'recovery_generation', state: 'unknown', label: 'Recovery generation', detail: 'The recovery generation could not be read.' }); + } else if (recoveryInfo?.exists) { + const short = recoveryInfo.shortId; + items.push({ + id: 'recovery_generation', + state: 'ready', + label: 'Recovery generation', + detail: short + ? `Current recovery generation ${short} is available for exact restore.` + : 'A current recovery generation is available for exact restore.', + }); + } else { + items.push({ + id: 'recovery_generation', + state: 'missing', + label: 'Recovery generation', + detail: 'No recovery generation is current yet. One is created by the next update, atomic deploy, or Git apply.', + }); + } + + // Supersede rule: when a recovery generation exists, compose_source tracks it. + if (recoveryExists) { + items.push({ + id: 'compose_source', + state: 'ready', + label: 'Previous compose file', + detail: 'Authored project files are covered by the current recovery generation.', + }); + } else if (inputs.backup === 'error') { items.push({ id: 'compose_source', state: 'unknown', label: 'Previous compose file', detail: 'The backup slot could not be read.' }); } else if (backupExists) { const age = inputs.backup.timestamp ? ` from ${formatAge(inputs.backup.timestamp, now)}` : ''; @@ -385,6 +427,30 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac items.push({ id: 'healthchecks', state: 'missing', label: 'Healthchecks', detail: 'No service defines a healthcheck; rollback verification relies on run state only.' }); } + if (inputs.policyEligibility === 'error') { + items.push({ id: 'policy_eligibility', state: 'unknown', label: 'Rollback eligibility', detail: 'Eligibility could not be assessed.' }); + } else if (inputs.policyEligibility === null) { + items.push({ id: 'policy_eligibility', state: 'ready', label: 'Rollback eligibility', detail: 'No recovery generation to assess yet.' }); + } else if (inputs.policyEligibility === 'eligible') { + items.push({ id: 'policy_eligibility', state: 'ready', label: 'Rollback eligibility', detail: 'Restore is eligible with known-good generation integrity and held images.' }); + } else if (inputs.policyEligibility === 'eligible_with_warning') { + items.push({ id: 'policy_eligibility', state: 'warning', label: 'Rollback eligibility', detail: 'Restore is possible but held images may be missing; moving-tag recoverability is weak.' }); + } else if (inputs.policyEligibility === 'prohibited') { + items.push({ id: 'policy_eligibility', state: 'blocked', label: 'Rollback eligibility', detail: 'Restore is prohibited until generation integrity or security posture is repaired.' }); + } else { + items.push({ id: 'policy_eligibility', state: 'unknown', label: 'Rollback eligibility', detail: 'Eligibility is not fully known yet.' }); + } + + if (inputs.managedInputs === 'error') { + items.push({ id: 'managed_inputs', state: 'unknown', label: 'Managed inputs', detail: 'Managed input coverage could not be read.' }); + } else if (inputs.managedInputs === null) { + items.push({ id: 'managed_inputs', state: 'unknown', label: 'Managed inputs', detail: 'Managed input coverage has not been assessed.' }); + } else if (inputs.managedInputs.covered) { + items.push({ id: 'managed_inputs', state: 'ready', label: 'Managed inputs', detail: inputs.managedInputs.detail }); + } else { + items.push({ id: 'managed_inputs', state: 'warning', label: 'Managed inputs', detail: inputs.managedInputs.detail }); + } + const mounts = inputs.containers === 'error' ? [] : [...new Set(inputs.containers.flatMap(c => c.mounts))]; @@ -393,7 +459,7 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac id: 'volume_data', state: 'not_covered', label: 'Application data', - detail: `Named volumes and bind-mounted data are not included in file backups. Rolling back restores compose and env files only; application data keeps its current state.${mountDetail}`, + detail: `Named volumes and bind-mounted data are not included in recovery generations. Rolling back restores the managed authored project (compose files, overrides, includes/extends, env and related inputs), Git identity when captured, and prior image holds; application data keeps its current state.${mountDetail}`, }); return items; @@ -406,9 +472,16 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac */ export function aggregateRollbackOverall(items: RollbackReadinessItem[]): RollbackOverall { const byId = new Map(items.map(i => [i.id, i.state])); + if (byId.get('policy_eligibility') === 'blocked') { + return 'not_ready'; + } if (byId.get('compose_source') !== 'ready') { return byId.get('compose_source') === 'unknown' ? 'partial' : 'not_ready'; } + const policy = byId.get('policy_eligibility'); + if (policy === 'unknown' || policy === 'warning') { + return 'partial'; + } if (byId.get('env_keys') === 'ready' && byId.get('previous_images') === 'ready') { return 'ready'; } diff --git a/backend/src/services/updateGuard/types.ts b/backend/src/services/updateGuard/types.ts index 7ad60b43..a0ab6dbd 100644 --- a/backend/src/services/updateGuard/types.ts +++ b/backend/src/services/updateGuard/types.ts @@ -35,10 +35,10 @@ export interface UpdateReadinessReport { } /** State of one rollback readiness item. */ -export type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered'; +export type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered' | 'blocked' | 'warning'; export interface RollbackReadinessItem { - id: 'compose_source' | 'env_keys' | 'previous_images' | 'last_deploy' | 'healthchecks' | 'volume_data'; + id: 'compose_source' | 'env_keys' | 'previous_images' | 'last_deploy' | 'healthchecks' | 'volume_data' | 'policy_eligibility' | 'managed_inputs' | 'recovery_generation'; state: RollbackItemState; label: string; /** Names only for env coverage; values never appear here. */ @@ -117,6 +117,8 @@ export type FailureReason = | 'healthcheck_failed' | 'dependency_unavailable' | 'node_unreachable' + | 'mixed_replica_images' + | 'rollback_coverage_unavailable' | 'unknown'; /** diff --git a/backend/src/types/rollbackGeneration.ts b/backend/src/types/rollbackGeneration.ts new file mode 100644 index 00000000..d57018c1 --- /dev/null +++ b/backend/src/types/rollbackGeneration.ts @@ -0,0 +1,169 @@ +/** + * Authored-project rollback generation content schema. + * + * The DB row in stack_update_recovery_generations remains the listing / + * lifecycle authority. Files under the generation content directory are the + * referenced content store (paths, checksums, invocation, tombstones). + */ +import type { GitSourceManifestState, InputDependencyKind, InputSensitivity, ManifestProvenance } from './gitProjectManifest'; + +export const ROLLBACK_GENERATION_SCHEMA_VERSION = 1 as const; + +export type RollbackOperationKind = + | 'update' + | 'deployment' + | 'git_apply' + | 'manual_backup' + | 'unknown'; + +export type RollbackEntryKind = + | InputDependencyKind + | 'compose-root' + | 'project-env' + | 'invocation-meta' + | 'other'; + +export type RollbackEntryProvenance = ManifestProvenance | 'authored' | 'sencho-generated'; + +export interface RollbackGenerationEntry { + /** Stack-relative POSIX path. */ + relativePath: string; + dependencyKind: RollbackEntryKind; + provenance: RollbackEntryProvenance; + /** present = file captured; tombstoned = must be absent after restore. */ + state: 'present' | 'tombstoned'; + contentSha256: string | null; + sizeBytes: number | null; + sensitivity: InputSensitivity; + /** When true, content is encrypted at rest in the generation store. */ + encrypted: boolean; + /** + * POSIX permission bits (mode & 0o777) at capture. Null on legacy generations + * or platforms where mode could not be read. + */ + mode: number | null; +} + +export interface RollbackInvocationRecord { + /** Ordered compose -f / project-directory / env-file args used for mutation. */ + composeArgsPrefix: string[]; + projectDirectory: string | null; + projectName: string | null; + explicitComposeFiles: string[]; + /** Stack-relative mesh override path when Mesh was part of the capture invocation. */ + meshOverrideRelativePath?: string | null; + /** True when Mesh was enabled for the stack at capture time. */ + meshEnabled?: boolean; +} + +export interface RollbackGitIdentity { + repoUrl: string; + branch: string; + commitSha: string; + manifestVersion: number | null; +} + +export interface RollbackImageIdentity { + serviceName: string; + imageId: string | null; + repoDigest: string | null; + platform: string | null; + declaredImageRef: string | null; +} + +/** On-disk generation.json for one recovery content directory. */ +export interface RollbackGenerationManifest { + schemaVersion: typeof ROLLBACK_GENERATION_SCHEMA_VERSION; + capabilityVersion: 1; + generationId: string; + nodeId: number; + stackName: string; + capturedAt: number; + operationKind: RollbackOperationKind; + entries: RollbackGenerationEntry[]; + /** + * Full managed relative-path set at capture time. Restore may delete live + * files in this set that are not present entries (tombstones / absent-at- + * capture). Paths outside this set are never touched by restore unless they + * also appear in the caller-supplied liveManagedPaths discovery set. + */ + managedRelativePaths: string[]; + invocation: RollbackInvocationRecord; + git: RollbackGitIdentity | null; + /** Prior applied/deployed/LKG refs when known (opaque strings). */ + priorRecords: { + appliedDeploySpec: string | null; + lkgHint: string | null; + /** Git DB snapshot at capture (restored with files so Compose args match). */ + lastAppliedContentHash: string | null; + manifestState: string | null; + manifestGeneration: string | null; + /** + * True when git-manifest.v1.json was snapshotted into the generation. + * False means the capture-time managed manifesto was absent (first apply + * preimage); restore must clear any manifesto written after capture. + * Omitted on older generations: inferred from whether the snapshot file exists. + */ + gitManifestCaptured?: boolean; + }; + images: RollbackImageIdentity[]; +} + +/** Durable Git database projection stored in restore-intent.json. */ +export interface RollbackGitDbSnapshot { + appliedDeploySpec: { files: string[]; contextDir: string | null } | null; + lastAppliedCommitSha: string | null; + lastAppliedContentHash: string | null; + manifestVersion: number | null; + manifestState: GitSourceManifestState | null; + manifestGeneration: string | null; +} + +/** + * Git DB + managed-manifesto preimage recorded before a restore mutates the + * live stack. Used as restoreGeneration transactionMeta and as + * RollbackRestoreIntent.gitSide. + */ +export interface RollbackRestoreTransactionMeta { + gitDbBefore: RollbackGitDbSnapshot | null; + /** Raw manifest.v1.json text before restore, or null when absent. */ + managedManifestBefore: string | null; +} + +/** + * Crash-safe restore intent: pre-restore filesystem snapshot plus the Git DB + * and managed-manifesto state that must be reinstated if the process dies + * before commitRestoreTransaction. + */ +export interface RollbackRestoreIntent { + generationId: string; + stackName: string; + nodeId: number; + paths: string[]; + at: number; + /** + * Present when compensate recorded Git side-state. Absent on older intents + * (filesystem-only reversion). + */ + gitSide?: RollbackRestoreTransactionMeta; +} + +export interface ResolvedRollbackInventory { + entries: Array<{ + relativePath: string; + dependencyKind: RollbackEntryKind; + provenance: RollbackEntryProvenance; + sensitivity: InputSensitivity; + /** Absolute path on the live stack when present; null if absent (tombstone candidate). */ + absolutePath: string | null; + }>; + invocation: RollbackInvocationRecord; + git: RollbackGitIdentity | null; + appliedDeploySpec: string | null; + lastAppliedContentHash: string | null; + manifestState: string | null; + manifestGeneration: string | null; + /** True when inventory claims exact atomic coverage. */ + exactCoverage: boolean; + coverageRefusal: string | null; +} diff --git a/backend/src/utils/snapshot-capture.ts b/backend/src/utils/snapshot-capture.ts index 2aed6e3d..a782b465 100644 --- a/backend/src/utils/snapshot-capture.ts +++ b/backend/src/utils/snapshot-capture.ts @@ -104,6 +104,21 @@ export const MAX_SNAPSHOT_FILE_BYTES = 1_000_000; /** The cap rendered in MB for operator-facing warning text. */ const MAX_SNAPSHOT_FILE_MB = MAX_SNAPSHOT_FILE_BYTES / 1_000_000; +/** Snapshot files Fleet restore is allowed to write. */ +export const FLEET_SNAPSHOT_APPLY_FILENAMES = ['compose.yaml', '.env'] as const; +export type FleetSnapshotApplyFilename = typeof FLEET_SNAPSHOT_APPLY_FILENAMES[number]; + +/** + * JSON body limit for POST /api/stacks/:name/fleet-snapshot-apply. Capture + * allows 1 MB per allowed file; the apply POST sends those files in one body, + * so the default 100 KB parser would 413 a legal captured pair. + */ +export const FLEET_SNAPSHOT_APPLY_BODY_LIMIT = + FLEET_SNAPSHOT_APPLY_FILENAMES.length * MAX_SNAPSHOT_FILE_BYTES + 256_000; + +/** Hub wait for remote capture-then-write. Capture inspects images before any write. */ +export const FLEET_SNAPSHOT_APPLY_TIMEOUT_MS = 300_000; + /** * Minimal node shape accepted by capture functions. * `mode` is required so remote dispatch can emit a tunnel-aware error when diff --git a/docs/features/atomic-deployments.mdx b/docs/features/atomic-deployments.mdx index 07954e5e..9533da8f 100644 --- a/docs/features/atomic-deployments.mdx +++ b/docs/features/atomic-deployments.mdx @@ -1,21 +1,21 @@ --- title: Atomic Deployments sidebarTitle: Atomic deploys -description: Wrap every deploy and update in a backup, a 3-second health probe, and an automatic rollback when a container crashes. +description: Wrap every deploy and update in a managed authored-project backup, prior-image holds, a 3-second health probe, and automatic rollback when a container crashes. --- -Sencho wraps every protected deploy in a four-step safety net: it backs up the current compose file, `.env`, and any configured project env files, runs the compose action, waits 3 seconds for containers to settle, then checks for a non-zero exit code. If any container crashed, Sencho restores the backup and re-deploys automatically. +Sencho wraps every protected deploy in a four-step safety net: it backs up the managed authored project (compose inventory, invocation metadata, and Git revision state when the stack is Git-linked), captures a recovery generation with prior image identity (opaque holds), runs the compose action, waits 3 seconds for containers to settle, then checks for a non-zero exit code. If any container crashed after handoff, Sencho restores from that generation and re-deploys automatically. -The same backup also powers the **Rollback** action in the stack editor, so you can roll a stack back to its last good configuration on demand. To see in advance whether that rollback would actually help, and to watch container health for longer than the 3-second probe, see [Health-Gated Updates](/features/health-gated-updates). +The same recovery generation powers the **Rollback** action in the stack editor, so you can roll a stack back to its last good configuration on demand. To see in advance whether that rollback would actually help, and to watch container health for longer than the 3-second probe, see [Health-Gated Updates](/features/health-gated-updates). ## How it works -1. **Backup.** Before the action runs, Sencho copies `compose.yaml` (or `compose.yml` / `docker-compose.yaml` / `docker-compose.yml`), `.env` if present, and any project env files configured for the stack (for example, `stack.env` or `.env.production`) into the backup directory. The deploy progress modal streams `=== Backup created for atomic deployment ===` once the copy completes, before any `docker compose` output. +1. **Backup.** Before the action runs, Sencho captures the managed authored project into a recovery generation: ordered compose files, overrides, discoverable `include:` / `extends` inputs, and env, label, config, and secret files when they can be enumerated. For Git-linked stacks it also records the managed-project manifesto and Git revision identity so a later restore can converge with the next pull. It records prior image identity as opaque holds. The deploy progress modal streams `=== Capturing rollback generation for atomic deploy ===` (or the matching update line) once capture completes, before any `docker compose` output. 2. **Run the action.** Sencho executes the requested compose action: `up -d` for a deploy, or a pull-then-`up -d` recreate for an update. -3. **Health probe.** Sencho waits 3 seconds, then lists every container with the `com.docker.compose.project=` label and checks each one for a non-zero exit code. Any container that has exited with a non-zero status counts as a crash. -4. **Auto-rollback on failure.** When a crash is detected, Sencho streams `=== Deployment failed - restoring previous compose and env files ===`, restores the backed-up files, and re-runs `docker compose up -d` with the restored configuration. On success it streams `=== Restored previous compose and env files ===`. The restore reverts the compose and `.env` configuration. An image on a moving tag (such as `latest`) is not reverted, because the local tag still resolves to the newly pulled digest. The original deploy error is preserved as the deploy result, so a failed-then-rolled-back deploy still registers as a failure. +3. **Health probe.** Sencho waits 3 seconds, then lists every container with the `com.docker.compose.project=` label and checks each one for a non-zero exit code. Any container that has exited with a non-zero status counts as a crash. Recovery capture and the post-rollback compensation probe skip Compose one-off (`docker compose run`) containers so they are not counted as service replicas. +4. **Auto-rollback on failure.** When a crash is detected after handoff, Sencho streams `=== Deployment failed - restoring previous runtime from recovery generation ===` (or the matching update line), restores the captured authored inventory, and re-runs `docker compose up -d` with the restored configuration. Where the generation holds a prior image ID, restore retargets that exact image for moving tags and for supported local builds. Named volumes and bind-mounted application data are not restored. The original deploy error is preserved as the deploy result, so a failed-then-rolled-back deploy still registers as a failure. -If the rollback itself fails (for example, the re-deploy step cannot pull a previously available image, or the file restore is blocked by filesystem permissions), Sencho streams `=== Rollback failed. Manual intervention may be required ===`. The backup files remain at `/backups///` so you can copy them back manually. +If the rollback itself fails (for example, a held image is missing, or the file restore is blocked by filesystem permissions), Sencho streams `=== Rollback failed. Manual intervention may be required ===`. The recovery generation remains available so you can retry or recover manually. ## Which operations are protected @@ -26,69 +26,65 @@ Atomic deployments wrap: - **Webhook** triggers for deploy and pull actions. - **Image auto-updates** triggered by an auto-update policy. -A scheduled image-update task uses the same atomic wrapper as a manual update, so a recurring update still takes a backup and rolls back automatically when a container crashes. Scheduled lifecycle actions (start, stop, restart) change no stack configuration and run `docker compose` directly without a backup. +A scheduled image-update task uses the same atomic wrapper as a manual update, so a recurring update still takes a backup and rolls back automatically when a container crashes. Scheduled lifecycle actions (start, stop, restart) change no stack configuration and run `docker compose` directly without a backup. On-demand **Backup** and scheduled stack backup capture a current recovery generation of the managed authored project. Backup refuses while a health gate is still observing after a deploy or update, so the pre-operation generation stays available for rollback. [Blueprint](/features/blueprint-model) deploys are a separate, fleet-wide desired-state path and do not use this atomic wrapper: a failed Blueprint apply is not backed up or rolled back automatically. ## Manual rollback -The stack editor's action bar has a **More actions** overflow menu (the three-dot icon next to **Update**). **Rollback** sits at the top, with the most recent backup's timestamp beneath the label. Selecting it restores the backed-up files and re-runs `docker compose up -d` non-atomically, so the rollback does not nest inside another atomic wrapper and overwrite the good backup with the just-failed state. +The stack editor's action bar has a **More actions** overflow menu (the three-dot icon next to **Update**). **Rollback** sits at the top, with the most recent recovery generation's timestamp beneath the label. Selecting it prefers the current recovery generation: it restores the captured authored inventory (and prior image IDs where held) and re-runs `docker compose up -d` non-atomically, so the rollback does not nest inside another atomic wrapper and overwrite the good generation with the just-failed state. Stack editor header for the plex stack, with the More actions overflow menu open. Rollback sits at the top with the backup timestamp beneath the label, followed by Scan config, a Mute submenu for the stack's notifications, and Delete. -The menu entry is hidden when no backup exists for the stack, for example on a freshly created stack that has never been deployed. It is also hidden for users who lack the `stack:deploy` permission; the backend enforces that check as the authoritative guard. The **Mute** entry next to it controls notification suppression for the stack and is unrelated to atomic deployments; see [Alerts & Notifications](/features/alerts-notifications). +The menu entry is hidden when no recovery generation (or fallback backup) exists for the stack, for example on a freshly created stack that has never been deployed. It is also hidden for users who lack the `stack:deploy` permission; the backend enforces that check as the authoritative guard. The **Mute** entry next to it controls notification suppression for the stack and is unrelated to atomic deployments; see [Alerts & Notifications](/features/alerts-notifications). After a failed deploy or update, the stack page also surfaces a **Roll back** button in the recovery panel alongside Retry, Restart, and Refresh. This is the same rollback action, triggered in response to a failure rather than invoked on demand. See [Deploy Progress](/features/deploy-progress#recovery-actions) for the full recovery actions reference. ## Where backups are stored -Backups live under `/backups///`, in the same writable volume Sencho uses for its database and other persisted state. They are intentionally kept outside the user's compose folder, so the operation works even when a container has chowned its bind-mounted stack directory to root. +Recovery generations and file backups live under Sencho's writable data volume (outside the user's compose folder), so the operation works even when a container has chowned its bind-mounted stack directory to root. Generations are also listed in **Resources → Rollback**; see [Health-Gated Updates](/features/health-gated-updates#automatic-rollback-images). -Each backup is a flat copy of the compose file Sencho found, plus `.env` and any configured project env files if they exist, plus two markers: a `.timestamp` recording when the backup was taken and a `.checksums` integrity manifest holding a SHA-256 for each backed-up file. There is one backup slot per stack: every protected deploy or update overwrites the previous backup, so the **Rollback** menu always reverts to the configuration that was on disk immediately before the most recent run. +Each capture stores the managed authored inventory for the stack: ordered compose files, overrides, discoverable include/extends inputs, and env/label/config/secret inputs when discoverable, plus integrity metadata. Opaque image holds record prior image identity when containers were inspectable. A stack keeps a current generation for manual rollback; superseded generations follow the retention settings on that page. -A restore is a faithful revert, not an overlay. Sencho replaces the compose file and `.env` with the backed-up copies and removes any compose variant or `.env` that was added after the backup was taken, so the stack returns to exactly the file set it had before the run. For example, if a deploy switched the stack from `compose.yaml` to `docker-compose.yml` or introduced a new `.env`, a rollback undoes both. Files Sencho does not manage are left untouched. +A restore reverts the managed inventory recorded in the generation, not an overlay of unmanaged files. Sencho restores the captured authored set (and, for Git-linked stacks, the matching managed-project manifesto and revision identity) and, where held, retargets exact prior image IDs. Files Sencho does not manage are left untouched. Named volumes and bind-mounted application data are never part of the restore. -Before a restore overwrites anything, Sencho re-hashes each backed-up file and compares it against the `.checksums` manifest. If a file no longer matches (for example, a backup truncated by an out-of-disk write), Sencho aborts the restore with a clear error and leaves the stack exactly as it was, rather than copying the corrupt content back over a working configuration. +Before a restore overwrites anything, Sencho verifies generation integrity. If content no longer matches (for example, a truncated write), Sencho aborts the restore with a clear error and leaves the stack exactly as it was, rather than copying corrupt content back over a working configuration. ## Rollback readiness -The **Stack Dossier** includes a **Rollback readiness** panel: a pre-flight read on whether rolling back will actually fix the problem. It carries an overall verdict (**Ready**, **Partial**, or **Not ready**) and evaluates five signals: whether a previous compose file exists and how old it is, whether a previous `.env` was captured, whether the previous image tag is pinned or moving (moving tags are not reverted), the age of the last successful deploy, and whether healthchecks are defined to verify recovery. A sixth, always-shown note covers application data: named volumes and bind-mounted data (database rows, uploaded files, anything outside the compose and env files) are outside the scope of any revert. +The **Stack Dossier** includes a **Rollback readiness** panel: a pre-flight read on whether rolling back will actually fix the problem. It carries an overall verdict (**Ready**, **Partial**, or **Not ready**) and reports managed-input coverage, whether a current recovery generation exists, rollback policy eligibility, plus the existing disclosures for compose/env coverage, previous image identity, last successful deploy, and healthchecks. Application data is always marked not covered: named volumes and bind-mounted data (database rows, uploaded files, anything outside the managed authored files) stay outside the scope of any revert. - - Rollback readiness panel in the Stack Dossier for the plex stack, showing a Partial verdict badge and five signal rows: Previous compose file (ready), Previous env file (ready), Previous image tag (a moving tag warning naming the rollback target), Last successful deploy (ready), and Healthchecks (missing). - - -Check this panel in the dossier before rolling back a stack that has been running for a while. A rollback reverts only the compose and env files; if the problem is in a volume or in a database migration that already ran, rolling back the compose file alone will not help. See [Stack Dossier](/features/stack-dossier) for the full readout. +Check this panel in the dossier before rolling back a stack that has been running for a while. A rollback restores the managed authored inventory and held prior images where available; if the problem is in a volume or in a database migration that already ran, rolling back files alone will not help. See [Stack Dossier](/features/stack-dossier) for the full readout. ## Troubleshooting - Sencho hides the entry whenever a rollback is not possible. The most common reason is that the stack has never been deployed, so no backup file exists yet. Run **Deploy** or **Update** once and the entry will appear. + Sencho hides the entry whenever a rollback is not possible. The most common reason is that the stack has never been deployed, so no recovery generation exists yet. Run **Deploy**, **Update**, or **Backup** once and the entry will appear. The entry is also hidden for users who lack the `stack:deploy` permission. Ask an admin to grant `stack:deploy` through **Settings · Access** if the entry does not appear. - The health probe is a 3-second window after `docker compose up -d` returns. Crashes after that window are out of scope for atomic rollback, because Sencho cannot tell a late exit apart from a normal restart. The [health gate](/features/health-gated-updates) covers exactly this period: it observes the stack for a configurable window after the update, records a verdict on the stack timeline, and offers a manual rollback when containers do not stay healthy. For ongoing health beyond that, use [Auto-Heal Policies](/features/auto-heal-policies) to restart unhealthy containers automatically and [per-stack alert rules](/features/alerts-notifications#per-stack-alert-rules) to page you when a container exits unexpectedly. + The health probe is a 3-second window after `docker compose up -d` returns. Crashes after that window are out of scope for atomic rollback, because Sencho cannot tell a late exit apart from a normal restart. The [health gate](/features/health-gated-updates) covers exactly this period: it observes the stack for a configurable window after the update, records a verdict on the stack timeline, and offers a manual rollback when containers do not stay healthy. The gate does not auto-compensate. For ongoing health beyond that, use [Auto-Heal Policies](/features/auto-heal-policies) to restart unhealthy containers automatically and [per-stack alert rules](/features/alerts-notifications#per-stack-alert-rules) to page you when a container exits unexpectedly. - This message means the auto-rollback attempted to restore the backup and re-deploy, but the restore step or the re-deploy itself errored out. The backup files are still at `/backups///`. To recover: + This message means the auto-rollback attempted to restore from the recovery generation and re-deploy, but the restore step or the re-deploy itself errored out. The generation remains listed under **Resources → Rollback**. To recover: - 1. Copy `compose.yaml` (or the variant Sencho backed up), `.env`, and any project env files from `/backups///` back into the stack directory. - 2. Open the stack in the editor and click **Deploy** to re-run with the restored configuration. + 1. Prefer **Rollback** from the stack editor (it uses the current recovery generation when one exists). + 2. If that still fails, open the stack in the editor, restore the authored files you need, and click **Deploy**. - The most common causes are filesystem permissions on the stack directory and a missing image in a private registry that the original deploy could not pull. + The most common causes are filesystem permissions on the stack directory and a missing held image that the restore expected to retarget. - The backup slot holds a file whose contents no longer match the checksum recorded when the backup was taken, usually because the disk filled up or the write was interrupted while the backup was being written. Sencho refuses to copy that file back, so your live stack is left untouched rather than overwritten with corrupt content. + The recovery generation holds content whose integrity no longer matches what was recorded at capture, usually because the disk filled up or the write was interrupted. Sencho refuses to copy that content back, so your live stack is left untouched rather than overwritten with corrupt content. - Edit the compose file or `.env` directly in the editor to the configuration you want, then click **Deploy**. The next protected deploy writes a fresh, verified backup, and **Rollback** works again from that point. + Edit the authored files directly in the editor to the configuration you want, then click **Deploy**. The next protected deploy writes a fresh, verified generation, and **Rollback** works again from that point. - Sencho keeps a single backup per stack. If you ran two atomic deploys back to back, the second deploy overwrote the first backup with the broken configuration before it failed. **Rollback** then restores that broken configuration, because as far as Sencho is concerned it is the most recent known state. + Manual rollback prefers the current recovery generation. If a later protected run already handed off a generation that captured the broken authored state, **Rollback** restores that state, because it is the generation Sencho treats as current. - To recover, edit the compose file or `.env` directly in the editor, fix the bad change, and click **Deploy**. The next protected deploy will write a fresh backup of the now-good configuration. + To recover, edit the authored files directly in the editor, fix the bad change, and click **Deploy**. The next protected deploy captures a fresh generation of the good configuration. diff --git a/docs/features/fleet-backups.mdx b/docs/features/fleet-backups.mdx index bd8a5c17..7a78aab6 100644 --- a/docs/features/fleet-backups.mdx +++ b/docs/features/fleet-backups.mdx @@ -89,12 +89,10 @@ By default, restoring a stack only writes back its files and leaves the stack's Restore plex on Local confirmation dialog with the warning that it overwrites the current compose files with the snapshot version, a Redeploy stack after restore checkbox, and Cancel and Restore buttons -Sencho writes the snapshot's files back to the target node: -- **Local nodes** have files written directly, with the current files backed up first (creating a rollback point via the atomic deployment system) -- **Remote nodes** receive files via the Distributed API proxy +Sencho writes the snapshot's files back to the target node. Restoring over an existing stack captures a recovery generation of the current authored project before any snapshot file is written. If that capture fails, the restore stops and the live files are left unchanged. A stack that does not yet exist is created from the snapshot files. Local and remote restores run that capture-then-write sequence on the target node. If a later file write fails after capture succeeded, live files may already have changed; the captured generation remains so you can use Rollback to return to the pre-restore project. - Restoring overwrites the current compose and environment files on the target node. If atomic deployments are enabled, the current files are backed up before restoration. + Restoring overwrites the current compose and environment files on the target node. For an existing stack, Sencho captures a recovery generation first so you can use Rollback if the restore stops after that capture. ### Restoring an entire snapshot diff --git a/docs/features/git-sources.mdx b/docs/features/git-sources.mdx index 12efa58c..d443aa26 100644 --- a/docs/features/git-sources.mdx +++ b/docs/features/git-sources.mdx @@ -248,5 +248,5 @@ Pulls, applies, and create-from-git operations on the same stack are serialized - **Unsupported inputs are refused, not guessed.** Inputs that cannot be safely reproduced fail the pull with an actionable message: URL includes, Git LFS pointers, submodule contents, symbolic links, build contexts that exceed the size bounds, and include or extends declarations that point outside the repository or use dynamic `\${VAR}` paths (their contents cannot be enumerated). Nothing is applied until the declaration is fixed. Absolute host paths, host bind mounts, external resources, and dynamic `\${VAR}` data paths are never claimed as covered: they resolve at deploy time from the environment or the node, and the manifest records them as unmanaged. - **Materialization bounds.** The materialized project is bounded by file count, total bytes, per-file size, path depth, and build-context size, each adjustable with a `GITSOURCE_*` variable (see configuration). Crossing a bound refuses the pull with the counts so far rather than producing a partial project. - **Detach and export.** Removing a Git source renders the effective compose model into a single `compose.yaml`, keeps the remaining materialized files, removes auto-discovered override files so the exported model is final, and removes Git tracking. Resolved environment values are baked into the exported file. If removal is interrupted before it completes, Sencho restores the original files automatically. -- **Rollback scope.** Rollback of a Git-managed stack restores compose files and `.env`. Other materialized inputs are not reverted by rollback; re-apply the previous revision from Git to restore them. +- **Rollback scope.** Rollback of a Git-managed stack restores the managed authored inventory captured in the recovery generation (the same contract as deploy and update: ordered compose files, overrides, include/extends, and discoverable env/label/config/secret inputs, plus held prior image IDs where available). Named volumes and bind-mounted application data are not restored. Git apply captures a generation before promote. If promote fails mid-flight, Sencho restores the prior files. If apply-with-deploy fails after promote, the applied files stay on disk and the generation remains available for manual rollback (partial success); Sencho does not auto-compensate the deploy failure after apply. - **Some read-only views read the primary file.** The dependency graph, drift snapshot, and networking inspector summarize the primary compose file, so a service declared only in an override may not appear in those views. Deploy, update, image-update checks, and mesh attachment use the full merged set. diff --git a/docs/features/health-gated-updates.mdx b/docs/features/health-gated-updates.mdx index 008c319c..bf2dd0d9 100644 --- a/docs/features/health-gated-updates.mdx +++ b/docs/features/health-gated-updates.mdx @@ -71,7 +71,7 @@ The live verifying and recovery view is part of the deploy progress panel. If yo Deploy progress modal with the Health gate failed headline and the banner identifying the unhealthy container, with the hint that rollback options are available on the stack -When the gate fails, the stack page surfaces the same [recovery actions](/features/deploy-progress#recovery-actions) as a failed update: retry, restart, roll back when a backup exists, refresh the container state, or copy diagnostics. Rolling back is always your call; the gate never rolls anything back on its own. +When the gate fails, the stack page surfaces the same [recovery actions](/features/deploy-progress#recovery-actions) as a failed update: retry, restart, roll back when a recovery generation exists, refresh the container state, or copy diagnostics. Rolling back is always your call; the gate never rolls anything back on its own. The gate also runs for updates you did not click: scheduled image updates, webhook-triggered deploys and pulls, bulk updates, and Git source applies all record gate verdicts on the stack timeline. Rollbacks, App Store installs, and Sencho's own automation loops are deliberately not gated. @@ -90,24 +90,24 @@ Open **Settings > Infrastructure > Stacks > Deploy Guardrails** on the node you The Stack Dossier carries a **Rollback readiness** section that answers one question honestly: if this update goes wrong, what can a rollback actually restore? It reports an overall state of Ready, Partial, or Not ready, built from: -- **Previous compose file**: whether a backup slot exists and how old it is. -- **Previous env file**: whether the backup contains the stack's env file. Sencho lists how many variable names are covered; the values themselves are restored with the file and never displayed. -- **Previous image tag**: the known rollback target from the update preview. When the compose file uses a moving tag, restoring files alone does not revert the image, so the report names the exact tag you would pin to be precise. -- **Last successful deploy**: whether the backup reflects a configuration that actually deployed successfully. +- **Recovery generation**: whether a current generation exists for exact restore. Atomic deploy, full-stack update, Git apply, and on-demand or scheduled stack backup capture one. +- **Managed inputs**: whether the managed authored inventory (ordered compose files, overrides, include/extends, and discoverable env/label/config/secret inputs) is covered exactly. +- **Previous compose file** / **Previous env file**: compose and env coverage disclosures (driven by the current generation when one exists). +- **Previous image tag**: the known rollback target. When a generation holds prior image IDs, restore can retarget moving tags and supported local builds to those exact images; when no prior ID was captured, restoring files alone may leave a moving tag on a newer digest. +- **Last successful deploy**: whether recent activity shows a configuration that deployed successfully. - **Healthchecks**: whether a rollback can be verified beyond run state. -- **Application data**: always reported as not covered. Named volumes and bind-mounted data are not included in file backups; a rollback restores compose and env files only, and your application data keeps its current state. This row exists so the limit is stated where you decide, not discovered during an incident. - - - Rollback readiness section in the Stack Dossier showing the overall state chip and the six rows: Previous compose file, Previous env file, Previous image tag, Last successful deploy, Healthchecks, and the Application data row marked not covered - +- **Rollback eligibility**: whether policy allows restore for the current generation (integrity and held-image posture). +- **Application data**: always reported as not covered. Named volumes and bind-mounted data are not included; a rollback restores the managed authored inventory (and held images where available), and your application data keeps its current state. This row exists so the limit is stated where you decide, not discovered during an incident. ## Automatic rollback images -Before a full-stack update runs, Sencho captures the running image of every service as an opaque, uniquely named copy so it can automatically restore the prior state if the update or its health gate fails. These copies exist in Docker as `sencho-rb//:hold`, but they are Sencho-internal recovery state, not part of your image inventory: they are kept out of **Resources → Images** and listed instead in **Resources → Rollback**. If a captured image still carries its original registry tag alongside the hold tag (a compose file pinned to an immutable tag, for example), it stays visible in the Images tab too, badged **Rollback protected** instead of the usual unused label, since it is held on purpose rather than left behind by accident. +Before an atomic deploy, a full-stack update, or a Git apply that will promote (including apply-with-deploy), Sencho captures the running image of every service as an opaque, uniquely named copy so it can restore the prior runtime when a failure after handoff requires it. These copies exist in Docker as `sencho-rb//:hold`, but they are Sencho-internal recovery state, not part of your image inventory: they are kept out of **Resources → Images** and listed instead in **Resources → Rollback**. If a captured image still carries its original registry tag alongside the hold tag (a compose file pinned to an immutable tag, for example), it stays visible in the Images tab too, badged **Rollback protected** instead of the usual unused label, since it is held on purpose rather than left behind by accident. -Each capture is one **rollback generation**. The generation currently backing a stack's live deployment is retained for as long as it is current; once a newer update supersedes it, it is retained for a configurable window before Sencho cleans it up automatically. A stack updated repeatedly in a short span can have more than one superseded generation in that window at once. +Each capture is one **rollback generation**. The generation currently backing a stack's live deployment is retained for as long as it is current; once a newer protected run supersedes it, it is retained for a configurable window before Sencho cleans it up automatically. A stack updated repeatedly in a short span can have more than one superseded generation in that window at once. -**Resources → Rollback** lists every generation on the node: the stack it belongs to, a short generation id, whether it is the current protection or a superseded one awaiting cleanup, and roughly when it clears. Search by stack name or generation id, filter to Current or Superseded, and click a column header to sort. An admin can release a generation's protection early from that list, including the current one, which immediately frees its image but means Sencho cannot automatically roll that stack back until its next successful full-stack update; the confirmation dialog says so before you proceed. A generation that is mid-recovery or still being observed by a health gate cannot be released until that finishes. +The health gate remains observational: it records a verdict and surfaces recovery actions, but it never rolls anything back on its own. Automatic restore after handoff belongs to the atomic deploy/update path when a crash is detected; manual **Rollback** prefers the current recovery generation. On-demand and scheduled stack backup will not replace that generation while a health gate is still observing after a deploy or update. + +**Resources → Rollback** lists every generation on the node: the stack it belongs to, a short generation id, whether it is the current protection or a superseded one awaiting cleanup, and roughly when it clears. Search by stack name or generation id, filter to Current or Superseded, and click a column header to sort. An admin can release a generation's protection early from that list, including the current one, which immediately frees its image but means Sencho cannot automatically roll that stack back until its next successful atomic deploy, full-stack update, Git apply, or stack backup capture; the confirmation dialog says so before you proceed. A generation that is mid-recovery or still being observed by a health gate cannot be released until that finishes. Resources Rollback tab showing search, All Current and Superseded filter buttons, an info icon for help, and sortable Stack, Generation, State, and Retention column headers above the generations table @@ -122,7 +122,7 @@ Two settings under **Settings > Infrastructure > Stacks > Deploy Guardrails** co ## Classified failures -When a deploy or update fails, Sencho classifies the failure from the compose output and shows the cause with a suggested next step in the recovery panel: an image pull failure, a missing environment variable, a host port conflict, a missing bind-mount path, a permission problem, a crashed container, a failed healthcheck, an unavailable dependency, an unreachable node or Docker daemon, or an invalid compose file. The classification also lands in **Copy details**, so a bug report carries the cause, not just the raw output. +When a deploy or update fails, Sencho classifies the failure from the compose output and shows the cause with a suggested next step in the recovery panel: an image pull failure, a missing environment variable, a host port conflict, a missing bind-mount path, a permission problem, a crashed container, a failed healthcheck, an unavailable dependency, an unreachable node or Docker daemon, an invalid compose file, mixed replica images that cannot be captured exactly, or a project whose include/extends paths cannot be captured for exact rollback. The classification also lands in **Copy details**, so a bug report carries the cause, not just the raw output. ## Troubleshooting @@ -140,15 +140,15 @@ When a deploy or update fails, Sencho classifies the failure from the compose ou The modal holds its auto-close while the gate observes so the verdict is not lost. You can close it at any time; the observation continues server-side and the verdict lands on the stack timeline. If the gate result repeatedly cannot be retrieved, the modal gives up with an unknown verdict instead of waiting forever. - That is by design and true for every stack: file backups cover compose and env files, never named volumes or bind-mounted data. For point-in-time copies of stack files across the fleet, use [fleet snapshots](/features/fleet-backups); for application data, use a backup tool appropriate to the workload (database dumps, volume backups) before risky updates. + That is by design and true for every stack: recovery generations cover the managed authored inventory and held prior images, never named volumes or bind-mounted data. For point-in-time copies of stack files across the fleet, use [fleet snapshots](/features/fleet-backups); for application data, use a backup tool appropriate to the workload (database dumps, volume backups) before risky updates. The sidebar's per-stack **Update** action runs the same path as the editor toolbar, so it shows the same readiness dialog and deploy progress. One click on **Update now** proceeds. On nodes that do not advertise the capability, updates run directly without the dialog. - Those are automatic rollback images: an opaque copy of a service's prior image, held so Sencho can restore it if a full-stack update fails. They are not leftovers. Sencho keeps them out of **Resources → Images** and **Security** on purpose (they are recovery state, not image inventory or scan targets) and lists them in **Resources → Rollback** instead, showing which stack and generation each one belongs to and how soon it clears on its own. If one still carries a normal registry tag too, it also stays visible in the Images tab with a **Rollback protected** badge, and Security continues to scan that registry tag. + Those are automatic rollback images: an opaque copy of a service's prior image, held so Sencho can restore it if an atomic deploy, full-stack update, or Git apply-with-deploy path needs the prior runtime after handoff. They are not leftovers. Sencho keeps them out of **Resources → Images** and **Security** on purpose (they are recovery state, not image inventory or scan targets) and lists them in **Resources → Rollback** instead, showing which stack and generation each one belongs to and how soon it clears on its own. If one still carries a normal registry tag too, it also stays visible in the Images tab with a **Rollback protected** badge, and Security continues to scan that registry tag. - That failure is intentional: the image is protected by an active or recently superseded rollback generation. Open **Resources → Rollback**, find the matching generation, and use **Release** there if you are sure you do not need it. Releasing the current generation means Sencho cannot automatically roll that stack back until its next successful full-stack update. + That failure is intentional: the image is protected by an active or recently superseded rollback generation. Open **Resources → Rollback**, find the matching generation, and use **Release** there if you are sure you do not need it. Releasing the current generation means Sencho cannot automatically roll that stack back until its next successful atomic deploy, full-stack update, or Git apply capture. diff --git a/docs/features/stack-dossier.mdx b/docs/features/stack-dossier.mdx index e0035b9d..74ccbb13 100644 --- a/docs/features/stack-dossier.mdx +++ b/docs/features/stack-dossier.mdx @@ -108,7 +108,7 @@ Events from the Drift tab (drift detected, drift resolved) appear in the stack's ### Rollback readiness -At the bottom of the Dossier tab, on nodes that advertise the `update-guard` capability, a **Rollback readiness** section reports an overall **Ready**, **Partial**, or **Not ready** state for the stack, built from six signals: whether a compose backup exists, whether that backup covers the stack's env file, whether the image tag is pinned or moving, whether the last deploy succeeded, whether any service defines a healthcheck, and an explicit note that application data in named volumes and bind mounts is never restored by a rollback. It shares the same capability-plus-fetch-failure behavior described above for the Networking and Storage export summaries. See [Health-Gated Updates](/features/health-gated-updates#rollback-readiness) for what each signal means and how to act on it. +At the bottom of the Dossier tab, on nodes that advertise the `update-guard` capability, a **Rollback readiness** section reports an overall **Ready**, **Partial**, or **Not ready** state for the stack. It covers managed-input coverage, whether a current recovery generation exists, rollback policy eligibility, plus the existing disclosures for compose/env coverage, previous image identity, last successful deploy, and healthchecks. Application data in named volumes and bind mounts remains explicitly not covered. It shares the same capability-plus-fetch-failure behavior described above for the Networking and Storage export summaries. See [Health-Gated Updates](/features/health-gated-updates#rollback-readiness) for what each signal means and how to act on it. ## Permissions and per-node scoping diff --git a/docs/features/stack-management.mdx b/docs/features/stack-management.mdx index 48c9ddbb..4727ccf3 100644 --- a/docs/features/stack-management.mdx +++ b/docs/features/stack-management.mdx @@ -300,7 +300,7 @@ If an image update is available, or the stack declares one or more services with For actionable registry updates, the detail line also reads `patch · safe to apply` (green), `minor · review recommended` (amber), or `major · breaking changes possible` (rose). Build-only stacks show **Rebuild available** instead of a version bump, with a **Rebuild & Update** button. A stack that mixes registry images and local builds still shows a single banner, with the rebuild note folded into the same detail line. Major bumps use the rose styling and are worth reviewing before applying. -Rebuilds can take longer than a registry pull and depend on the local Dockerfile context, network access, and base-image availability. Atomic rollback restores compose and env files only; previously built image layers are not rolled back automatically. +Rebuilds can take longer than a registry pull and depend on the local Dockerfile context, network access, and base-image availability. When a recovery generation captured the prior image ID, supported local-build recovery restores that exact image identity. Without a held prior ID, restore falls back to the authored files alone and does not reconstruct previously built layers. A separate amber **scan** banner appears here if the most recent post-deploy vulnerability scan failed or was skipped for one or more images; deploys are never blocked by a scan failure. See [Vulnerability Scanning](/features/vulnerability-scanning). @@ -383,7 +383,7 @@ The stack header groups actions by frequency of use. The most common action is t | Secondary | **Stop** | `docker compose stop` | Stops containers without removing them. State is preserved. | | Secondary | **Take down** | `docker compose down` | Removes containers and compose-created networks. The stack definition stays on disk so you can deploy again later. Optional volume removal is available in the confirmation dialog. | | Secondary | **Update** | `docker compose pull` + `up -d` (or build-aware rebuild when services declare `build:`) | Pulls registry images and recreates containers. When one or more services use `build:`, Update rebuilds those images from source (`compose build --pull`), pulls any remaining registry images, then recreates containers. | -| Overflow | **Rollback** | Restores backup | Reverts compose and env files to the pre-deploy snapshot and redeploys. Only shown when a backup exists. | +| Overflow | **Rollback** | Restores recovery generation | Prefers the current recovery generation: restores the managed authored inventory and held prior image IDs where available, then redeploys. Only shown when a generation or fallback backup exists. | | Overflow | **Scan config** | Trivy config scan | Scans the compose file for misconfigurations (admin role). | | Overflow | **Monitor** | Opens Monitor sheet | Opens the stack **Monitor** sheet on the Alerts tab (alert rules and Auto-heal). Same sheet as sidebar **Alerts** / **Auto-Heal**. | | Overflow | **Mute** | Creates a mute rule | Quick presets to mute notifications, deploy-success noise, or monitor alerts for this stack, plus a link to manage its mute rules in full. See [Alerts and Notifications](/features/alerts-notifications). | diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts index 86e70be7..cdafa685 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts @@ -1206,6 +1206,23 @@ describe('useStackActions recovery records', () => { await act(async () => { await result.current.rollbackStack(); }); expect(stackListState.recordActionSuccess).toHaveBeenCalledWith('web.yml'); expect(stackListState.recordActionFailure).not.toHaveBeenCalled(); + expect(toast.success).toHaveBeenCalledWith('Stack rolled back from recovery generation.'); + }); + + it('toasts the backend generation rollback message', async () => { + vi.mocked(apiFetch).mockImplementation((url: string) => { + const u = String(url); + if (u.includes('/rollback')) { + return Promise.resolve(new Response( + JSON.stringify({ message: 'Restored generation gen-1.', recoveryId: 'gen-1' }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + )); + } + return Promise.resolve(new Response('[]', { status: 200 })); + }); + const { result } = setup(); + await act(async () => { await result.current.rollbackStack(); }); + expect(toast.success).toHaveBeenCalledWith('Restored generation gen-1.'); }); }); diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.ts index 19feaf6c..daed68e4 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.ts @@ -1661,7 +1661,9 @@ export function useStackActions(options: UseStackActionsOptions) { throw parseStackActionError(rawBody, 'Rollback failed', res.status); } overlayState.setPolicyBlock(null); - toast.success('Stack rolled back: compose and env files restored.'); + const body: unknown = await res.json().catch(() => null); + const message = isRecord(body) && typeof body.message === 'string' ? body.message.trim() : ''; + toast.success(message || 'Stack rolled back from recovery generation.'); stackListState.recordActionSuccess(stackFile); // The rollback already succeeded; a failure of the cosmetic refetches below // (containers redeployed by the rollback, restored compose content, backup diff --git a/frontend/src/components/resources/RollbackGenerationsTab.tsx b/frontend/src/components/resources/RollbackGenerationsTab.tsx index 3b37491f..cb51fa7d 100644 --- a/frontend/src/components/resources/RollbackGenerationsTab.tsx +++ b/frontend/src/components/resources/RollbackGenerationsTab.tsx @@ -14,6 +14,13 @@ import { toast } from '@/components/ui/toast-store'; import { SENCHO_OPEN_STACK_EVENT, type SenchoOpenStackDetail } from '@/lib/events'; import { TableSkeleton } from './TableSkeleton'; +export type RollbackOperationKind = + | 'update' + | 'deployment' + | 'git_apply' + | 'manual_backup' + | 'unknown'; + export interface RollbackGeneration { id: string; shortId: string; @@ -23,6 +30,8 @@ export interface RollbackGeneration { phase: string; createdAt: number; artifactExpiresAt: number | null; + createdBy: string | null; + operationKind: RollbackOperationKind | null; /** Best-effort UI hint only; the server revalidates eligibility on release. */ releasable: boolean; } @@ -280,22 +289,23 @@ export function RollbackGenerationsTab({ generations, isLoading, isAdmin, nodeId + Trigger Actions - {isLoading ? : ( + {isLoading ? : ( {generations.length === 0 ? ( - + No rollback-protected generations on this node. ) : sorted.length === 0 ? ( - + No generations match this filter. @@ -318,6 +328,9 @@ export function RollbackGenerationsTab({ generations, isLoading, isAdmin, nodeId {gen.shortId} + + {[gen.operationKind, gen.createdBy].filter(Boolean).join(' · ') || 'unknown'} + {formatExpiry(gen)} diff --git a/frontend/src/components/resources/__tests__/RollbackGenerationsTab.test.tsx b/frontend/src/components/resources/__tests__/RollbackGenerationsTab.test.tsx index bd2e4481..7a7e73ac 100644 --- a/frontend/src/components/resources/__tests__/RollbackGenerationsTab.test.tsx +++ b/frontend/src/components/resources/__tests__/RollbackGenerationsTab.test.tsx @@ -26,6 +26,8 @@ function generation(overrides: Partial = {}): RollbackGenera phase: 'immediate_verified', createdAt: Date.now(), artifactExpiresAt: Date.now() + 3 * 24 * 60 * 60 * 1000, + createdBy: null, + operationKind: null, releasable: true, ...overrides, }; @@ -48,7 +50,8 @@ function shortIdsInOrder(): string[] { function stateLabelsInOrder(): string[] { return screen.getAllByRole('row').slice(1).map((row) => { const cells = within(row).getAllByRole('cell'); - return cells[2]?.textContent ?? ''; + // Stack, Generation, Trigger, State, Retention, Actions + return cells[3]?.textContent ?? ''; }); } diff --git a/frontend/src/components/stack/RollbackReadinessSection.tsx b/frontend/src/components/stack/RollbackReadinessSection.tsx index ac389d58..6eb783d5 100644 --- a/frontend/src/components/stack/RollbackReadinessSection.tsx +++ b/frontend/src/components/stack/RollbackReadinessSection.tsx @@ -1,11 +1,11 @@ import { useEffect, useState } from 'react'; -import { Check, CircleHelp, Database, X, type LucideIcon } from 'lucide-react'; +import { AlertTriangle, Ban, Check, CircleHelp, Database, X, type LucideIcon } from 'lucide-react'; import { apiFetch } from '@/lib/api'; import { cn } from '@/lib/utils'; import { useNodes } from '@/context/NodeContext'; // Mirrors the backend payload shape (the frontend never imports backend). -type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered'; +type RollbackItemState = 'ready' | 'missing' | 'unknown' | 'not_covered' | 'blocked' | 'warning'; type RollbackOverall = 'ready' | 'partial' | 'not_ready'; interface RollbackReadinessItem { @@ -37,6 +37,8 @@ const STATE_META: Record missing: { icon: X, tone: 'text-destructive' }, unknown: { icon: CircleHelp, tone: 'text-stat-subtitle' }, not_covered: { icon: Database, tone: 'text-warning' }, + blocked: { icon: Ban, tone: 'text-destructive' }, + warning: { icon: AlertTriangle, tone: 'text-warning' }, }; /** diff --git a/frontend/src/components/stack/__tests__/RollbackReadinessSection.test.tsx b/frontend/src/components/stack/__tests__/RollbackReadinessSection.test.tsx index 329d6201..ce057b30 100644 --- a/frontend/src/components/stack/__tests__/RollbackReadinessSection.test.tsx +++ b/frontend/src/components/stack/__tests__/RollbackReadinessSection.test.tsx @@ -22,7 +22,7 @@ const report = (overall: Overall) => ({ overall, items: [ { id: 'compose_source', state: 'ready', label: 'Previous compose file', detail: 'A backup is available to restore.' }, - { id: 'volume_data', state: 'not_covered', label: 'Application data', detail: 'Named volumes and bind-mounted data are not included in file backups.' }, + { id: 'volume_data', state: 'not_covered', label: 'Application data', detail: 'Named volumes and bind-mounted data are not included in recovery generations.' }, ], }); @@ -42,7 +42,7 @@ describe('RollbackReadinessSection', () => { vi.mocked(apiFetch).mockResolvedValue(new Response(JSON.stringify(report('ready')), { status: 200 })); render(); await waitFor(() => expect(screen.getByText('Application data')).toBeInTheDocument()); - expect(screen.getByText(/not included in file backups/)).toBeInTheDocument(); + expect(screen.getByText(/not included in recovery generations/)).toBeInTheDocument(); }); it('renders nothing without the update-guard capability and never fetches', () => { @@ -58,4 +58,36 @@ describe('RollbackReadinessSection', () => { await waitFor(() => expect(apiFetch).toHaveBeenCalled()); expect(container).toBeEmptyDOMElement(); }); + it('renders blocked item state with Ban semantics', async () => { + vi.mocked(apiFetch).mockResolvedValue(new Response(JSON.stringify({ + stack: 'web', + computedAt: Date.now(), + overall: 'not_ready', + items: [ + { id: 'policy', state: 'blocked', label: 'Security policy', detail: 'Rollback target is prohibited by current policy.' }, + { id: 'volume_data', state: 'not_covered', label: 'Application data', detail: 'Named volumes and bind-mounted data are not included in recovery generations.' }, + ], + }), { status: 200 })); + render(); + await waitFor(() => expect(screen.getByText('Security policy')).toBeInTheDocument()); + expect(screen.getByText(/prohibited by current policy/)).toBeInTheDocument(); + expect(screen.getByTestId('rollback-overall')).toHaveAttribute('data-overall', 'not_ready'); + }); + + it('renders warning item state', async () => { + vi.mocked(apiFetch).mockResolvedValue(new Response(JSON.stringify({ + stack: 'web', + computedAt: Date.now(), + overall: 'partial', + items: [ + { id: 'held_images', state: 'warning', label: 'Held images', detail: 'Some held images could not be verified.' }, + { id: 'volume_data', state: 'not_covered', label: 'Application data', detail: 'Named volumes and bind-mounted data are not included in recovery generations.' }, + ], + }), { status: 200 })); + render(); + await waitFor(() => expect(screen.getByText('Held images')).toBeInTheDocument()); + expect(screen.getByText(/could not be verified/)).toBeInTheDocument(); + expect(screen.getByTestId('rollback-overall')).toHaveAttribute('data-overall', 'partial'); + }); + });