From 3f1f15a6f433eb137206833550c2eb0a0134fbf0 Mon Sep 17 00:00:00 2001 From: Anso Date: Tue, 21 Jul 2026 12:18:01 -0400 Subject: [PATCH] fix: keep running containers until stack pull/build succeeds (#1657) * fix: keep running containers until stack pull/build succeeds Acquire images before reconcile, capture a recovery generation for compensation, and only remove classified orphans after handoff. * fix: address recovery audit blockers for safe stack updates Retire abandoned and expired recovery artifacts, probe compensated runtimes before reporting rollback success, preserve local Docker when deleting a node, validate the exact Compose invocation before capture, and repair updateStack return-contract fixtures. * fix: resolve ESLint errors blocking CI on this branch Unused-import and unused-variable errors left over from the stack deletion refactor: MeshService in stacks.ts (its opt-out cascade moved into DeployedStackDeletionService), a redundant pruneVolumes destructure in deleteDeployedStack (the real one is re-derived from the same input object inside runDeletionBody), and an unused beforeAll import in a Docker-integration test stub. Also scopes the webhook pull-action case body in a block to satisfy no-case-declarations; purely syntactic, no behavior change. * fix: harden recovery probe, cleanup retry, and failed-pull Docker test Reject absent or unhealthy expected replicas before reporting rollback success, keep cleanup records until artifacts are actually removed, fail closed when a mesh override cannot be generated, and assert a real failed pull leaves the original container running. * fix: verify recovery probe image identity and stack-scoped override paths Reject recovered runtimes that use the wrong image or leave scale-zero services running, and confine tombstone override deletion to the intent stack directory so forged cross-stack paths cannot be swept. * test: batch notification cap fixtures in a SQLite transaction Unbatched 1200-row inserts were timing out at the default 30s under CI load even though the same assertions pass in under 2s when green. --- .github/workflows/ci.yml | 4 + backend/package.json | 1 + backend/src/__tests__/compose-service.test.ts | 146 +++- .../src/__tests__/database-metrics.test.ts | 78 +- .../deployed-stack-deletion-service.test.ts | 156 ++++ .../src/__tests__/docker-controller.test.ts | 20 + .../failed-pull-keeps-stack-running.test.ts | 112 +++ .../__tests__/image-updates-routes.test.ts | 4 +- .../src/__tests__/scheduler-service.test.ts | 2 +- .../src/__tests__/stack-bulk-routes.test.ts | 4 +- ...stack-delete-cascades-mesh-opt-out.test.ts | 4 +- .../stack-self-protected-routes.test.ts | 4 +- .../stack-update-orchestrator.test.ts | 5 +- .../stack-update-recovery-service.test.ts | 365 +++++++++ .../stacks-failure-notifications.test.ts | 6 +- .../__tests__/update-guard-rollback.test.ts | 9 +- .../__tests__/update-guard-service.test.ts | 7 +- .../src/__tests__/webhooks-trigger.test.ts | 2 +- backend/src/bootstrap/startup.ts | 12 + backend/src/routes/imageUpdates.ts | 8 +- backend/src/routes/nodes.ts | 10 +- backend/src/routes/stacks.ts | 115 +-- backend/src/services/BlueprintService.ts | 38 +- backend/src/services/ComposeService.ts | 279 +++++-- backend/src/services/DatabaseService.ts | 374 ++++++++- .../services/DeployedStackDeletionService.ts | 521 ++++++++++++ backend/src/services/DockerController.ts | 52 ++ backend/src/services/FileSystemService.ts | 5 + backend/src/services/SchedulerService.ts | 8 +- .../services/ServiceUpdateRecoveryService.ts | 12 +- backend/src/services/StackOpLockService.ts | 28 +- .../src/services/StackUpdateOrchestrator.ts | 27 +- .../services/StackUpdateRecoveryService.ts | 767 ++++++++++++++++++ backend/src/services/WebhookService.ts | 12 +- backend/src/services/composeProjectContext.ts | 94 +++ backend/src/services/updateGuard/readiness.ts | 4 +- backend/vitest.config.ts | 2 +- backend/vitest.docker-integration.config.ts | 16 + .../hooks/useStackActions.test.ts | 13 + .../EditorLayout/hooks/useStackActions.ts | 3 +- .../EditorLayout/operation-phase.ts | 2 +- 41 files changed, 3087 insertions(+), 244 deletions(-) create mode 100644 backend/src/__tests__/deployed-stack-deletion-service.test.ts create mode 100644 backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts create mode 100644 backend/src/__tests__/stack-update-recovery-service.test.ts create mode 100644 backend/src/services/DeployedStackDeletionService.ts create mode 100644 backend/src/services/StackUpdateRecoveryService.ts create mode 100644 backend/src/services/composeProjectContext.ts create mode 100644 backend/vitest.docker-integration.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e9f1a00..138d73c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,10 @@ jobs: working-directory: ./backend run: npm test + - name: Docker integration tests (dedicated Vitest config) + working-directory: ./backend + run: npm run test:docker-integration + - name: Lint (ESLint) working-directory: ./backend run: npm run lint diff --git a/backend/package.json b/backend/package.json index bdb61960..3c54811b 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,6 +10,7 @@ "start": "node dist/index.js", "predev": "node scripts/generate-version.js", "dev": "nodemon --watch src --ext ts,json --exec ts-node src/index.ts", + "test:docker-integration": "vitest run --config vitest.docker-integration.config.ts", "test": "vitest run", "lint": "eslint src", "reset-mfa": "node dist/cli/resetMfa.js", diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts index 8ca97de8..61d271cc 100644 --- a/backend/src/__tests__/compose-service.test.ts +++ b/backend/src/__tests__/compose-service.test.ts @@ -24,6 +24,17 @@ const { mockResolveMissingExternalNetworks, mockCreateNetwork, mockAddNotificationHistory, + mockIsMeshStackEnabled, + mockClassifyLegacyOrphansForUpdate, + mockCaptureCandidate, + mockMarkAcquired, + mockHandoff, + mockMarkReconciling, + mockMarkImmediateVerified, + mockAbandon, + mockCompensateWithCandidate, + mockBuildUnifiedHeldImagePredicate, + mockGetRecovery, } = vi.hoisted(() => ({ mockSpawn: vi.fn(), mockGetContainersByStack: vi.fn().mockResolvedValue([]), @@ -58,6 +69,24 @@ const { }), mockCreateNetwork: vi.fn().mockResolvedValue({ id: 'net-1' }), mockAddNotificationHistory: vi.fn(), + mockIsMeshStackEnabled: vi.fn().mockReturnValue(false), + mockClassifyLegacyOrphansForUpdate: vi.fn().mockResolvedValue({ status: 'none' }), + mockCaptureCandidate: vi.fn().mockResolvedValue({ + id: 'recovery-1', + node_id: 1, + stack_name: 'my-stack', + status: 'candidate', + phase: 'captured', + override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml', + }), + mockMarkAcquired: vi.fn().mockReturnValue(true), + mockHandoff: vi.fn().mockReturnValue(true), + mockMarkReconciling: vi.fn().mockReturnValue(true), + mockMarkImmediateVerified: vi.fn().mockReturnValue(true), + mockAbandon: vi.fn().mockResolvedValue(true), + mockCompensateWithCandidate: vi.fn().mockResolvedValue(true), + mockBuildUnifiedHeldImagePredicate: vi.fn().mockReturnValue(() => false), + mockGetRecovery: vi.fn().mockReturnValue(undefined), })); vi.mock('child_process', () => ({ spawn: mockSpawn, execFile: vi.fn() })); @@ -89,6 +118,7 @@ vi.mock('../services/DockerController', () => ({ getInstance: () => ({ getContainersByStack: mockGetContainersByStack, getLegacyOrphanContainersByStack: mockGetLegacyOrphanContainersByStack, + classifyLegacyOrphansForUpdate: mockClassifyLegacyOrphansForUpdate, removeContainers: mockRemoveContainers, pruneDanglingImages: mockPruneDanglingImages, createNetwork: mockCreateNetwork, @@ -111,6 +141,7 @@ vi.mock('../services/DatabaseService', () => ({ getGitSource: () => undefined, getStackProjectEnvFiles: () => [], addNotificationHistory: mockAddNotificationHistory, + isMeshStackEnabled: (...args: unknown[]) => mockIsMeshStackEnabled(...args), }), }, })); @@ -150,6 +181,24 @@ vi.mock('../services/MeshService', () => ({ }, })); +vi.mock('../services/StackUpdateRecoveryService', () => ({ + StackUpdateRecoveryService: { + getInstance: () => ({ + captureCandidate: mockCaptureCandidate, + markAcquired: mockMarkAcquired, + handoff: mockHandoff, + markReconciling: mockMarkReconciling, + markImmediateVerified: mockMarkImmediateVerified, + abandon: mockAbandon, + compensateWithCandidate: mockCompensateWithCandidate, + buildUnifiedHeldImagePredicate: mockBuildUnifiedHeldImagePredicate, + get: mockGetRecovery, + linkGateOrRetain: vi.fn(), + }), + }, +})); + + vi.mock('../services/SelfIdentityService', () => ({ default: { getInstance: () => ({ getBindMounts: mockGetBindMounts }), @@ -247,6 +296,23 @@ beforeEach(() => { declaredExternalCount: 0, }); mockCreateNetwork.mockResolvedValue({ id: 'net-1' }); + mockClassifyLegacyOrphansForUpdate.mockResolvedValue({ status: 'none' }); + mockCaptureCandidate.mockResolvedValue({ + id: 'recovery-1', + node_id: 1, + stack_name: 'my-stack', + status: 'candidate', + phase: 'captured', + override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml', + }); + mockMarkAcquired.mockReturnValue(true); + mockHandoff.mockReturnValue(true); + mockMarkReconciling.mockReturnValue(true); + mockMarkImmediateVerified.mockReturnValue(true); + mockAbandon.mockResolvedValue(true); + mockIsMeshStackEnabled.mockReturnValue(false); + mockCompensateWithCandidate.mockResolvedValue(true); + mockBuildUnifiedHeldImagePredicate.mockReturnValue(() => false); delete process.env.SENCHO_MODE; vi.useFakeTimers({ shouldAdvanceTime: true }); }); @@ -586,6 +652,16 @@ describe('ComposeService - authoredComposeArgs mesh override', () => { // ── deployStack ──────────────────────────────────────────────────────── + + it('fails closed when a mesh-enabled stack cannot generate its override', async () => { + mockIsMeshStackEnabled.mockReturnValue(true); + mockEnsureStackOverride.mockRejectedValue(new Error('mesh override write failed')); + + const svc = ComposeService.getInstance(1); + await expect(svc.validateExactComposeInvocation('my-stack')).rejects.toThrow(/mesh override write failed/i); + expect(mockSpawn).not.toHaveBeenCalled(); + }); + describe('ComposeService - deployStack', () => { it('blocks a pilot deploy with relative binds before replacing containers when path mapping differs', async () => { process.env.SENCHO_MODE = 'pilot'; @@ -987,7 +1063,7 @@ describe('ComposeService - updateStack build-aware', () => { expect(spawnArgs.some(args => args.includes('pull') && !args.includes('--ignore-buildable'))).toBe(true); }); - it('rolls back compose files when a build step fails during atomic update', async () => { + it('abandons recovery and leaves runtime untouched when acquire fails before handoff', async () => { mockLoadStackBuildServices.mockResolvedValueOnce(['app']); let spawnCount = 0; mockSpawn.mockImplementation(() => { @@ -1004,8 +1080,12 @@ describe('ComposeService - updateStack build-aware', () => { const error = await result; expect(error).not.toBeNull(); - expect(mockRestoreStackFiles).toHaveBeenCalled(); - expect(getComposeRollbackInfo(error)?.attempted).toBe(true); + expect(mockAbandon).toHaveBeenCalledWith('recovery-1'); + expect(mockHandoff).not.toHaveBeenCalled(); + expect(mockRemoveContainers).not.toHaveBeenCalled(); + expect(mockCompensateWithCandidate).not.toHaveBeenCalled(); + const spawnArgs = mockSpawn.mock.calls.map(c => c[1] as string[]); + expect(spawnArgs.some(args => args.includes('up'))).toBe(false); }); }); @@ -1106,7 +1186,7 @@ describe('ComposeService - updateStack prune-on-update', () => { // The update already succeeded before the prune ran, so a prune failure // must neither reject nor trigger the atomic restore. - await expect(promise).resolves.toBeUndefined(); + await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' }); expect(mockRestoreStackFiles).not.toHaveBeenCalled(); }); @@ -1120,7 +1200,7 @@ describe('ComposeService - updateStack prune-on-update', () => { const promise = svc.updateStack('my-stack'); await vi.advanceTimersByTimeAsync(3100); - await expect(promise).resolves.toBeUndefined(); + await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' }); }); }); @@ -1428,19 +1508,10 @@ describe('ComposeService - idle-output stall backstop', () => { await promise; }); - it('preserves STACK_STALLED_OUTPUT through the atomic rollback wrapper', async () => { + it('surfaces STACK_STALLED_OUTPUT on acquire stall without runtime compensation', async () => { process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000'; const pullProc = createMockProcess(); - let call = 0; - // First spawn is the stalling pull; later spawns (the rollback restore's - // `up`) close cleanly, proving the restore is not idle-timeout armed. - mockSpawn.mockImplementation(() => { - call += 1; - if (call === 1) return pullProc; - const p = createMockProcess(); - Promise.resolve().then(() => p.emit('close', 0)); - return p; - }); + mockSpawn.mockImplementation(() => pullProc); const svc = ComposeService.getInstance(1); const result = svc.updateStack('my-stack', undefined, true).then(() => null, (e: Error) => e); @@ -1451,7 +1522,10 @@ describe('ComposeService - idle-output stall backstop', () => { const error = await result; expect(error).not.toBeNull(); expect(error!.message).toContain('STACK_STALLED_OUTPUT'); - expect(getComposeRollbackInfo(error)).toMatchObject({ attempted: true }); + // Acquire failure is pre-handoff: abandon candidate, do not wrap as ComposeRollbackError. + expect(getComposeRollbackInfo(error)).toBeNull(); + expect(mockAbandon).toHaveBeenCalledWith('recovery-1'); + expect(mockCompensateWithCandidate).not.toHaveBeenCalled(); }); }); @@ -1580,3 +1654,41 @@ describe('ComposeService - streamLogs', () => { ); }); }); + + +describe('ComposeService - updateStack safe ordering', () => { + it('never broad-removes Compose-managed containers before up', async () => { + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([]); + mockGetContainersByStack.mockResolvedValue([{ Id: 'should-not-remove' }]); + + const svc = ComposeService.getInstance(1); + const promise = svc.updateStack('my-stack'); + await vi.advanceTimersByTimeAsync(3100); + await promise; + + expect(mockGetContainersByStack).not.toHaveBeenCalled(); + expect(mockRemoveContainers).not.toHaveBeenCalled(); + expect(mockCaptureCandidate).toHaveBeenCalled(); + expect(mockClassifyLegacyOrphansForUpdate).toHaveBeenCalledWith('my-stack'); + expect(mockHandoff).toHaveBeenCalled(); + }); + + it('removes only classified orphans after handoff', async () => { + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([]); + mockClassifyLegacyOrphansForUpdate.mockResolvedValueOnce({ + status: 'orphans', + ids: ['orphan-1'], + }); + mockRemoveContainers.mockResolvedValueOnce([{ id: 'orphan-1', success: true }]); + + const svc = ComposeService.getInstance(1); + const promise = svc.updateStack('my-stack'); + await vi.advanceTimersByTimeAsync(3100); + await promise; + + expect(mockHandoff).toHaveBeenCalled(); + expect(mockRemoveContainers).toHaveBeenCalledWith(['orphan-1']); + }); +}); diff --git a/backend/src/__tests__/database-metrics.test.ts b/backend/src/__tests__/database-metrics.test.ts index 64459f3f..522e0d87 100644 --- a/backend/src/__tests__/database-metrics.test.ts +++ b/backend/src/__tests__/database-metrics.test.ts @@ -225,23 +225,25 @@ describe('DatabaseService - notification history cap (periodic)', () => { // A chatty stack writes 600 events. const base = Date.now(); - for (let i = 0; i < 600; i++) { - db.addNotificationHistory(0, { - level: 'info', - message: `chatty-${i}`, - timestamp: base + i, - stack_name: 'chatty', - }); - } - // A quiet stack writes 3 events long before the chatty burst. - for (let i = 0; i < 3; i++) { - db.addNotificationHistory(0, { - level: 'info', - message: `quiet-${i}`, - timestamp: base - 10_000 + i, - stack_name: 'quiet', - }); - } + db.transaction(() => { + for (let i = 0; i < 600; i++) { + db.addNotificationHistory(0, { + level: 'info', + message: `chatty-${i}`, + timestamp: base + i, + stack_name: 'chatty', + }); + } + // A quiet stack writes 3 events long before the chatty burst. + for (let i = 0; i < 3; i++) { + db.addNotificationHistory(0, { + level: 'info', + message: `quiet-${i}`, + timestamp: base - 10_000 + i, + stack_name: 'quiet', + }); + } + }); // No per-insert prune: every row is present. const beforeCleanup = db.getNotificationHistory(0, 2000); @@ -261,13 +263,15 @@ describe('DatabaseService - notification history cap (periodic)', () => { db.deleteAllNotifications(0); const base = Date.now(); - for (let i = 0; i < 1200; i++) { - db.addNotificationHistory(0, { - level: 'info', - message: `system-${i}`, - timestamp: base + i, - }); - } + db.transaction(() => { + for (let i = 0; i < 1200; i++) { + db.addNotificationHistory(0, { + level: 'info', + message: `system-${i}`, + timestamp: base + i, + }); + } + }); db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 }); @@ -279,14 +283,16 @@ describe('DatabaseService - notification history cap (periodic)', () => { it('keeps the newest entries per (node, stack) after periodic cap', () => { db.deleteAllNotifications(0); const base = Date.now(); - for (let i = 0; i < 600; i++) { - db.addNotificationHistory(0, { - level: 'info', - message: `ordered-${i}`, - timestamp: base + i * 10, - stack_name: 'ordered', - }); - } + db.transaction(() => { + for (let i = 0; i < 600; i++) { + db.addNotificationHistory(0, { + level: 'info', + message: `ordered-${i}`, + timestamp: base + i * 10, + stack_name: 'ordered', + }); + } + }); db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 }); @@ -301,9 +307,11 @@ describe('DatabaseService - notification history cap (periodic)', () => { it('uses safe defaults when called with only the retention argument', () => { db.deleteAllNotifications(0); const base = Date.now(); - for (let i = 0; i < 600; i++) { - db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' }); - } + db.transaction(() => { + for (let i = 0; i < 600; i++) { + db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' }); + } + }); // Production caller (MonitorService) only passes daysToKeep; the cap defaults must enforce the per-stack 500 limit. const summary = db.cleanupOldNotifications(30); const after = db.getNotificationHistory(0, 2000).filter((n: any) => n.stack_name === 'default'); diff --git a/backend/src/__tests__/deployed-stack-deletion-service.test.ts b/backend/src/__tests__/deployed-stack-deletion-service.test.ts new file mode 100644 index 00000000..27052fd8 --- /dev/null +++ b/backend/src/__tests__/deployed-stack-deletion-service.test.ts @@ -0,0 +1,156 @@ +/** + * Deployed-stack deletion: ready transaction retires both recovery models; + * blocking intents gate same-name create. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import { randomUUID } from 'crypto'; +import path from 'path'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import type { + StackUpdateRecoveryGenerationRow, + ServiceUpdateRecoveryRow, + StackUpdateCleanupPendingRow, +} from '../services/DatabaseService'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let DeployedStackDeletionService: typeof import('../services/DeployedStackDeletionService').DeployedStackDeletionService; +let overrideDeletionContainmentBase: typeof import('../services/DeployedStackDeletionService').overrideDeletionContainmentBase; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ DeployedStackDeletionService, overrideDeletionContainmentBase } = await import('../services/DeployedStackDeletionService')); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +function db() { + return DatabaseService.getInstance(); +} + +beforeEach(() => { + const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db; + raw.prepare('DELETE FROM stack_update_recovery_generations').run(); + raw.prepare('DELETE FROM service_update_recovery').run(); + raw.prepare('DELETE FROM stack_update_cleanup_pending').run(); +}); + +const NODE = 1; + +describe('DeployedStackDeletionService ready transaction', () => { + it('commitStackDeletionReadyTransaction deletes full-stack and service recovery rows', () => { + const now = Date.now(); + const stackName = 'del-stack'; + const gen: StackUpdateRecoveryGenerationRow = { + id: randomUUID(), + node_id: NODE, + stack_name: stackName, + status: 'active', + phase: 'immediate_verified', + is_current: 1, + backup_slot_id: null, + override_path: null, + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: now, + updated_at: now, + created_by: null, + artifacts_retired: 0, + }; + db().insertStackUpdateRecoveryGeneration(gen); + const svc: ServiceUpdateRecoveryRow = { + id: randomUUID(), + node_id: NODE, + stack_name: stackName, + service_name: 'web', + replicas_json: '[]', + majority_image_id: 'sha256:abc', + declared_image_ref: 'nginx:latest', + weak_floating_tag: 0, + health_gate_id: null, + status: 'active', + expires_at: now + 60_000, + claim_expires_at: null, + created_at: now, + created_by: null, + }; + db().insertServiceUpdateRecovery(svc); + const intentId = randomUUID(); + const intent: StackUpdateCleanupPendingRow = { + id: intentId, + node_id: NODE, + stack_name: stackName, + status: 'prepared', + target_kind: 'local_socket', + rollback_tags_json: '[]', + override_paths_json: '[]', + prune_volumes_requested: 0, + created_at: now, + updated_at: now, + }; + db().insertCleanupPending(intent); + + expect(db().commitStackDeletionReadyTransaction(intentId, NODE, stackName)).toBe(true); + expect(db().listStackUpdateRecoveryForStack(NODE, stackName)).toHaveLength(0); + expect(db().listActiveServiceUpdateRecoveries(NODE, stackName, 'web')).toHaveLength(0); + expect(db().getCleanupPending(intentId)?.status).toBe('ready'); + }); + + it('hasBlockingDeletionIntent is true for prepared intents', () => { + const now = Date.now(); + db().insertCleanupPending({ + id: randomUUID(), + node_id: NODE, + stack_name: 'blocked', + status: 'prepared', + target_kind: 'local_socket', + rollback_tags_json: '[]', + override_paths_json: '[]', + prune_volumes_requested: 0, + created_at: now, + updated_at: now, + }); + expect(db().hasBlockingDeletionIntent(NODE, 'blocked')).toBe(true); + expect(db().hasBlockingDeletionIntent(NODE, 'other')).toBe(false); + }); + + it('assertNoBlockingDeletionIntent throws for prepared stacks', () => { + const now = Date.now(); + db().insertCleanupPending({ + id: randomUUID(), + node_id: NODE, + stack_name: 'prep', + status: 'prepared', + target_kind: 'local_socket', + rollback_tags_json: '[]', + override_paths_json: '[]', + prune_volumes_requested: 0, + created_at: now, + updated_at: now, + }); + expect(() => { + DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(NODE, 'prep'); + }).toThrow(/deletion in progress/i); + }); +}); + +describe('overrideDeletionContainmentBase', () => { + it('confines stack-scoped intents to the stack directory', () => { + expect(overrideDeletionContainmentBase('/app/compose', 'my-stack')).toBe( + path.resolve('/app/compose', 'my-stack'), + ); + expect(overrideDeletionContainmentBase('/app/compose', null)).toBe( + path.resolve('/app/compose'), + ); + }); + + it('rejects invalid stack names that could traverse', () => { + expect(overrideDeletionContainmentBase('/app/compose', '../other')).toBeNull(); + expect(overrideDeletionContainmentBase('/app/compose', 'bad/name')).toBeNull(); + }); +}); + diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 5afd827d..30cc4e99 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -1714,3 +1714,23 @@ describe('DockerController - getLegacyOrphanContainersByStack', () => { fallbackSpy.mockRestore(); }); }); + +describe('DockerController - classifyLegacyOrphansForUpdate', () => { + it('returns none when compose ps already manages containers', async () => { + // Reuse the same mocks as getLegacyOrphanContainersByStack tests in this file. + const { default: DockerController } = await import('../services/DockerController'); + const dc = DockerController.getInstance(1); + vi.spyOn(dc as any, 'fetchComposePsContainers').mockResolvedValue([{ ID: 'c1' }]); + await expect(dc.classifyLegacyOrphansForUpdate('my-stack')).resolves.toEqual({ status: 'none' }); + }); + + it('returns classification_failed when compose ps and fallback both fail', async () => { + const { default: DockerController } = await import('../services/DockerController'); + const dc = DockerController.getInstance(1); + vi.spyOn(dc as any, 'fetchComposePsContainers').mockRejectedValue(new Error('compose boom')); + vi.spyOn(dc as any, 'smartFallback').mockRejectedValue(new Error('fallback boom')); + const result = await dc.classifyLegacyOrphansForUpdate('my-stack'); + expect(result.status).toBe('classification_failed'); + }); +}); + diff --git a/backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts b/backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts new file mode 100644 index 00000000..696db72c --- /dev/null +++ b/backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts @@ -0,0 +1,112 @@ +/** + * Docker-backed integration: a failed pull must leave the running stack untouched. + * Skipped automatically when Docker is unavailable. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { execFileSync } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb'; + +function dockerAvailable(): boolean { + try { + execFileSync('docker', ['info'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +const hasDocker = dockerAvailable(); +const STACK = 'fpkeep'; + +function compose(args: string[], cwd: string): string { + return execFileSync('docker', ['compose', '-p', STACK, ...args], { + cwd, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +describe.skipIf(!hasDocker)('failed pull keeps stack running', () => { + let tmpDir: string; + let composeDir: string; + let stackDir: string; + let nodeId: number; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + composeDir = process.env.COMPOSE_DIR!; + stackDir = path.join(composeDir, STACK); + fs.mkdirSync(stackDir, { recursive: true }); + + const { DatabaseService } = await import('../../services/DatabaseService'); + const db = DatabaseService.getInstance(); + const local = db.getDefaultNode(); + if (!local?.id) throw new Error('Test DB has no default local node'); + nodeId = local.id; + db.updateGlobalSetting('prune_on_update', '0'); + + fs.writeFileSync( + path.join(stackDir, 'compose.yaml'), + [ + 'services:', + ' web:', + ' image: busybox:1.36.1', + ' command: ["sleep", "3600"]', + '', + ].join('\n'), + 'utf8', + ); + + compose(['up', '-d', '--pull', 'always'], stackDir); + }, 180_000); + + afterAll(async () => { + try { + if (stackDir && fs.existsSync(stackDir)) { + compose(['down', '--remove-orphans'], stackDir); + } + } catch { + // Best-effort cleanup. + } + if (tmpDir) cleanupTestDb(tmpDir); + }, 120_000); + + it('leaves the original container running when ComposeService updateStack pull fails', async () => { + const beforeId = compose(['ps', '-q'], stackDir).trim(); + expect(beforeId.length).toBeGreaterThan(0); + + const beforeInspect = JSON.parse( + execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }), + ) as Array<{ State: { Running: boolean; Status: string } }>; + expect(beforeInspect[0].State.Running).toBe(true); + + fs.writeFileSync( + path.join(stackDir, 'compose.yaml'), + [ + 'services:', + ' web:', + ' image: busybox:sencho-does-not-exist-fpkeep-xyz', + ' command: ["sleep", "3600"]', + '', + ].join('\n'), + 'utf8', + ); + + const { StackUpdateRecoveryService } = await import('../../services/StackUpdateRecoveryService'); + StackUpdateRecoveryService.resetForTests(); + const { ComposeService } = await import('../../services/ComposeService'); + await expect( + ComposeService.getInstance(nodeId).updateStack(STACK, undefined, true), + ).rejects.toThrow(); + + const afterId = compose(['ps', '-q'], stackDir).trim(); + expect(afterId).toBe(beforeId); + + const afterInspect = JSON.parse( + execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }), + ) as Array<{ State: { Running: boolean } }>; + expect(afterInspect[0].State.Running).toBe(true); + }, 180_000); +}); diff --git a/backend/src/__tests__/image-updates-routes.test.ts b/backend/src/__tests__/image-updates-routes.test.ts index 40157d54..3d07a7b0 100644 --- a/backend/src/__tests__/image-updates-routes.test.ts +++ b/backend/src/__tests__/image-updates-routes.test.ts @@ -375,7 +375,7 @@ describe('POST /api/auto-update/execute', () => { .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') .mockResolvedValue({ hasUpdate: true } as never); - const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue(); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); const gateSpy = vi.spyOn(PolicyEnforcement, 'enforcePolicyPreDeploy').mockResolvedValue({ ok: false, bypassed: false, @@ -421,7 +421,7 @@ describe('POST /api/auto-update/execute', () => { .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') .mockResolvedValue({ hasUpdate: true } as never); - const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue(); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au'); try { const res = await request(app) diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index 3b07ed5f..18f8ae4f 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -54,7 +54,7 @@ const { mockStartContainer: vi.fn().mockResolvedValue(undefined), mockStopContainer: vi.fn().mockResolvedValue(undefined), mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }), - mockUpdateStack: vi.fn().mockResolvedValue(undefined), + mockUpdateStack: vi.fn().mockResolvedValue({ recoveryId: null }), mockGetStacks: vi.fn().mockResolvedValue([]), mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockResolvedValue(''), diff --git a/backend/src/__tests__/stack-bulk-routes.test.ts b/backend/src/__tests__/stack-bulk-routes.test.ts index 80d9a3e3..3d9fe675 100644 --- a/backend/src/__tests__/stack-bulk-routes.test.ts +++ b/backend/src/__tests__/stack-bulk-routes.test.ts @@ -260,7 +260,7 @@ describe('POST /api/stacks/bulk execution', () => { }); it('handles update action (paid tier) by calling ComposeService.updateStack', async () => { - mockUpdateStack.mockResolvedValue(undefined); + mockUpdateStack.mockResolvedValue({ recoveryId: null }); const { LicenseService } = await import('../services/LicenseService'); const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); try { @@ -287,7 +287,7 @@ describe('POST /api/stacks/bulk execution', () => { policy: { id: 1, name: 'block-criticals', node_id: null, node_identity: '', stack_pattern: null, max_severity: 'HIGH', block_on_deploy: 1, block_on_severity: 1, block_on_kev: 0, block_on_fixable: 0, enabled: 1, replicated_from_control: 0, created_at: Date.now(), updated_at: Date.now() }, violations: [{ imageRef: 'nginx:latest', severity: 'CRITICAL', criticalCount: 3, highCount: 0, kevCount: 0, fixableCount: 0, reasons: ['severity'], scanId: 1 }], }); - mockUpdateStack.mockResolvedValue(undefined); + mockUpdateStack.mockResolvedValue({ recoveryId: null }); try { const res = await request(app) .post('/api/stacks/bulk') diff --git a/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts b/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts index 2b92c17c..037d6ed5 100644 --- a/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts +++ b/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts @@ -124,7 +124,9 @@ describe('DELETE /api/stacks/:stackName mesh opt-out cascade (F-1 / F-14)', () = expect(res.status).toBe(200); const sawCascadeWarning = warnSpy.mock.calls.some(args => - typeof args[0] === 'string' && args[0].includes('Mesh opt-out cascade failed') + typeof args[0] === 'string' + && (args[0].includes('Mesh opt-out cascade failed') + || args[0].includes('Mesh opt-out failed')) ); expect(sawCascadeWarning).toBe(true); }); diff --git a/backend/src/__tests__/stack-self-protected-routes.test.ts b/backend/src/__tests__/stack-self-protected-routes.test.ts index 30210dfb..315a33e8 100644 --- a/backend/src/__tests__/stack-self-protected-routes.test.ts +++ b/backend/src/__tests__/stack-self-protected-routes.test.ts @@ -164,7 +164,7 @@ describe('self stack lifecycle refusal', () => { }); it('allows update on a non-self stack', async () => { - mockUpdateStack.mockResolvedValue(undefined); + mockUpdateStack.mockResolvedValue({ recoveryId: null }); const res = await request(app) .post('/api/stacks/web/update') .set('Cookie', authCookie); @@ -177,7 +177,7 @@ describe('self stack lifecycle refusal', () => { describe('POST /api/stacks/bulk self stack skip', () => { beforeEach(() => { stubSelfProject('sencho'); - mockUpdateStack.mockResolvedValue(undefined); + mockUpdateStack.mockResolvedValue({ recoveryId: null }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]); }); diff --git a/backend/src/__tests__/stack-update-orchestrator.test.ts b/backend/src/__tests__/stack-update-orchestrator.test.ts index 103ad017..7e58775f 100644 --- a/backend/src/__tests__/stack-update-orchestrator.test.ts +++ b/backend/src/__tests__/stack-update-orchestrator.test.ts @@ -35,7 +35,7 @@ function spec(name: string): EffectiveServiceSpec { beforeEach(() => { state.updateStack.mockReset(); - state.updateStack.mockResolvedValue(undefined); + state.updateStack.mockResolvedValue({ recoveryId: null }); state.model = null; }); @@ -85,8 +85,9 @@ describe('StackUpdateOrchestrator stack branch', () => { { nodeId: 0, stackName: 'web', target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' }, { atomic: true, terminalWs: null }, ); - expect(result).toEqual({ kind: 'stack_compose_done' }); + expect(result).toEqual({ kind: 'stack_compose_done', recoveryId: null }); expect(state.updateStack).toHaveBeenCalledWith('web', undefined, true); + // recoveryId is forwarded from ComposeService.updateStack }); }); diff --git a/backend/src/__tests__/stack-update-recovery-service.test.ts b/backend/src/__tests__/stack-update-recovery-service.test.ts new file mode 100644 index 00000000..b7e784bc --- /dev/null +++ b/backend/src/__tests__/stack-update-recovery-service.test.ts @@ -0,0 +1,365 @@ +/** + * Direct tests for StackUpdateRecoveryService artifact lifecycle and compensation probe. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockTag = vi.fn().mockResolvedValue(undefined); +const mockRemove = vi.fn().mockResolvedValue(undefined); +const mockListContainers = vi.fn().mockResolvedValue([]); +const mockInspectContainer = vi.fn(); +const mockGetContainer = vi.fn(() => ({ inspect: mockInspectContainer })); +const mockGetImage = vi.fn((ref: string) => ({ + tag: mockTag, + remove: mockRemove, + inspect: vi.fn().mockResolvedValue({ RepoDigests: [] }), + _ref: ref, +})); + +vi.mock('../services/DockerController', () => ({ + default: { + getInstance: () => ({ + getDocker: () => ({ + listContainers: mockListContainers, + getContainer: mockGetContainer, + getImage: mockGetImage, + }), + }), + }, +})); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBaseDir: () => '/test/compose', + backupStackFiles: vi.fn().mockResolvedValue(undefined), + restoreStackFiles: vi.fn().mockResolvedValue(undefined), + }), + }, +})); + +vi.mock('../services/composeProjectContext', () => ({ + classifyReferenceKind: () => 'moving_tag', + resolveComposeProjectContext: vi.fn().mockResolvedValue({ + validateForMutation: vi.fn().mockResolvedValue(undefined), + backupFromContext: vi.fn().mockResolvedValue('backup-1'), + restoreFromContext: vi.fn().mockResolvedValue(undefined), + }), +})); + +vi.mock('../services/effectiveServiceModel', () => ({ + buildEffectiveServiceModel: vi.fn().mockResolvedValue({ + renderable: true, + services: [{ name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1 }], + }), +})); + +const mockValidateExact = vi.fn().mockResolvedValue(undefined); +vi.mock('../services/ComposeService', async () => { + const actual = await vi.importActual('../services/ComposeService'); + return { + ...actual, + ComposeService: { + getInstance: () => ({ + validateExactComposeInvocation: mockValidateExact, + buildAuthoredComposeArgs: vi.fn(), + }), + }, + getComposeCommandTimeoutMs: () => 60_000, + }; +}); + +const mockUnlink = vi.fn().mockResolvedValue(undefined); +const mockWriteFile = vi.fn().mockResolvedValue(undefined); +const mockRealpath = vi.fn(async (p: string) => p); +vi.mock('fs/promises', () => ({ + default: { + unlink: (p: string) => mockUnlink(p), + writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc), + realpath: (p: string) => mockRealpath(p), + }, + unlink: (p: string) => mockUnlink(p), + writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc), + realpath: (p: string) => mockRealpath(p), +})); + +import { DatabaseService } from '../services/DatabaseService'; +import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService'; + +describe('StackUpdateRecoveryService', () => { + beforeEach(() => { + vi.clearAllMocks(); + StackUpdateRecoveryService.resetForTests(); + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'running', Labels: {} }, + ]); + mockInspectContainer.mockResolvedValue({ + State: { Status: 'running', ExitCode: 0 }, + Image: 'sha256:abc', + }); + mockValidateExact.mockResolvedValue(undefined); + mockUnlink.mockResolvedValue(undefined); + mockWriteFile.mockResolvedValue(undefined); + mockRealpath.mockImplementation(async (p: string) => p); + mockRemove.mockResolvedValue(undefined); + mockTag.mockResolvedValue(undefined); + }); + + it('validates exact invocation before tagging images', async () => { + const order: string[] = []; + mockValidateExact.mockImplementation(async () => { order.push('validate'); }); + mockTag.mockImplementation(async () => { order.push('tag'); }); + mockWriteFile.mockImplementation(async () => { order.push('write'); }); + + const spyInsert = vi.spyOn(DatabaseService.prototype, 'insertStackUpdateRecoveryGeneration') + .mockImplementation(() => { order.push('insert'); }); + vi.spyOn(DatabaseService.prototype, 'getGlobalSettings').mockReturnValue({}); + + await StackUpdateRecoveryService.getInstance().captureCandidate({ + nodeId: 1, + stackName: 'my-stack', + createdBy: 'test', + }); + + expect(order.indexOf('validate')).toBeLessThan(order.indexOf('tag')); + expect(order.indexOf('tag')).toBeLessThan(order.indexOf('write')); + expect(order.indexOf('write')).toBeLessThan(order.indexOf('insert')); + spyInsert.mockRestore(); + }); + + it('retires opaque tags and override on abandon', async () => { + const row = { + id: 'gen-1', + node_id: 1, + stack_name: 'my-stack', + status: 'candidate' as const, + phase: 'captured' as const, + is_current: 0, + backup_slot_id: null, + override_path: '/test/compose/my-stack/.sencho-recovery-aaaaaaaaaaaa.yml', + services_json: JSON.stringify([{ + serviceName: 'web', + scale: 1, + hasBuild: false, + declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', + replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold' }], + }]), + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + }; + + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row); + vi.spyOn(DatabaseService.prototype, 'abandonStackUpdateRecoveryGeneration').mockReturnValue(true); + const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired') + .mockReturnValue(true); + + const ok = await StackUpdateRecoveryService.getInstance().abandon('gen-1'); + expect(ok).toBe(true); + expect(mockRemove).toHaveBeenCalled(); + expect(mockUnlink).toHaveBeenCalledWith(row.override_path); + expect(markRetired).toHaveBeenCalledWith('gen-1'); + }); + + it('does not mark restored_current when recovery probe finds crashed containers', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'web', + scale: 1, + hasBuild: false, + declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', + replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/x/web:hold' }], + }]); + const row = { + id: 'gen-2', + node_id: 1, + stack_name: 'my-stack', + status: 'active' as const, + phase: 'reconciling' as const, + is_current: 1, + backup_slot_id: 'b1', + override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml', + services_json: servicesJson, + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + }; + vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row); + const update = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration') + .mockImplementation(() => undefined); + + mockListContainers.mockResolvedValue([{ Id: 'c1', State: 'exited', Labels: { 'com.docker.compose.service': 'web' } }]); + mockInspectContainer.mockResolvedValue({ State: { ExitCode: 0 } }); + + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().compensateWithCandidate( + 'gen-2', + async () => undefined, + ); + await vi.advanceTimersByTimeAsync(3100); + const ok = await promise; + vi.useRealTimers(); + + expect(ok).toBe(false); + expect(update).toHaveBeenCalledWith('gen-2', expect.objectContaining({ status: 'recovery_required' })); + expect(update).not.toHaveBeenCalledWith( + 'gen-2', + expect.objectContaining({ status: 'restored_current' }), + ); + }); + + const capturedWebReplica = { + containerId: 'c1', + imageId: 'sha256:oldimg', + repoDigest: null, + state: 'running' as const, + rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold', + }; + + it('probeRecoveredStack rejects empty runtime when expected replicas were running', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', replicas: [capturedWebReplica], + }]); + mockListContainers.mockResolvedValue([]); + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(false); + vi.useRealTimers(); + }); + + it('probeRecoveredStack rejects restarting and unhealthy expected containers', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', replicas: [capturedWebReplica], + }]); + + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'restarting', Labels: { 'com.docker.compose.service': 'web' } }, + ]); + vi.useFakeTimers(); + let promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(false); + vi.useRealTimers(); + + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + ]); + mockInspectContainer.mockResolvedValue({ + Image: 'sha256:oldimg', + State: { Status: 'running', Health: { Status: 'unhealthy' } }, + }); + vi.useFakeTimers(); + promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(false); + vi.useRealTimers(); + }); + + it('probeRecoveredStack rejects healthy containers running a mismatched image', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', replicas: [capturedWebReplica], + }]); + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + ]); + mockInspectContainer.mockResolvedValue({ + Image: 'sha256:newfailed', + Config: { Image: 'nginx:alpine' }, + State: { Status: 'running' }, + }); + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(false); + vi.useRealTimers(); + }); + + it('probeRecoveredStack rejects a running replica for a service captured at scale zero', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'worker', scale: 0, hasBuild: false, declaredImageRef: 'busybox:latest', + referenceKind: 'moving_tag', + replicas: [{ + containerId: 'c0', imageId: 'sha256:worker', repoDigest: null, state: 'stopped', + rollbackTag: 'sencho-rb/aaaaaaaaaaaa/worker:hold', + }], + }]); + mockListContainers.mockResolvedValue([ + { Id: 'c0', State: 'running', Labels: { 'com.docker.compose.service': 'worker' } }, + ]); + mockInspectContainer.mockResolvedValue({ + Image: 'sha256:worker', + State: { Status: 'running' }, + }); + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(false); + vi.useRealTimers(); + }); + + it('probeRecoveredStack accepts healthy running replicas matching capture scale and image', async () => { + const servicesJson = JSON.stringify([{ + serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', replicas: [capturedWebReplica], + }]); + mockListContainers.mockResolvedValue([ + { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } }, + ]); + mockInspectContainer.mockResolvedValue({ + Image: 'sha256:oldimg', + Config: { Image: 'sencho-rb/aaaaaaaaaaaa/web:hold' }, + State: { Status: 'running' }, + }); + vi.useFakeTimers(); + const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson); + await vi.advanceTimersByTimeAsync(3100); + await expect(promise).resolves.toBe(true); + vi.useRealTimers(); + }); + + it('does not mark artifacts retired when tag removal fails', async () => { + const row = { + id: 'gen-3', + node_id: 1, + stack_name: 'my-stack', + status: 'abandoned' as const, + phase: 'captured' as const, + is_current: 0, + backup_slot_id: null, + override_path: null, + services_json: JSON.stringify([{ + serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest', + referenceKind: 'moving_tag', + replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/cccccccccccc/web:hold' }], + }]), + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: Date.now() - 1, + operation_lease_expires_at: null, + created_at: Date.now(), + updated_at: Date.now(), + created_by: null, + artifacts_retired: 0, + }; + mockRemove.mockRejectedValueOnce(Object.assign(new Error('docker busy'), { statusCode: 500 })); + const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired') + .mockReturnValue(true); + const ok = await StackUpdateRecoveryService.getInstance().retireGenerationArtifacts(row); + expect(ok).toBe(false); + expect(markRetired).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/__tests__/stacks-failure-notifications.test.ts b/backend/src/__tests__/stacks-failure-notifications.test.ts index 8adaae5f..168f0877 100644 --- a/backend/src/__tests__/stacks-failure-notifications.test.ts +++ b/backend/src/__tests__/stacks-failure-notifications.test.ts @@ -270,7 +270,7 @@ describe('health gate begin call sites', () => { }); it('begins a gate after a manual update and returns its id', async () => { - mockUpdateStack.mockResolvedValue(undefined); + mockUpdateStack.mockResolvedValue({ recoveryId: null }); const res = await request(app) .post('/api/stacks/myapp/update') .set('Cookie', authCookie) @@ -281,7 +281,7 @@ describe('health gate begin call sites', () => { }); it('begins a gate per stack in a bulk update and carries ids in the results', async () => { - mockUpdateStack.mockResolvedValue(undefined); + mockUpdateStack.mockResolvedValue({ recoveryId: null }); const res = await request(app) .post('/api/stacks/bulk') .set('Cookie', authCookie) @@ -490,7 +490,7 @@ describe('deploy_failure notification on /update error', () => { }); it('uses trusted proxy tier headers for remote atomic updates', async () => { - mockUpdateStack.mockResolvedValue(undefined); + mockUpdateStack.mockResolvedValue({ recoveryId: null }); const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); const res = await request(app) diff --git a/backend/src/__tests__/update-guard-rollback.test.ts b/backend/src/__tests__/update-guard-rollback.test.ts index 41e026a2..cb6a9db7 100644 --- a/backend/src/__tests__/update-guard-rollback.test.ts +++ b/backend/src/__tests__/update-guard-rollback.test.ts @@ -96,11 +96,12 @@ describe('buildRollbackItems', () => { expect(known.detail).toContain('nginx:1.27.1'); }); - it('downgrades the previous image to not_covered for a moving tag', () => { + it('treats a moving tag as ready because full-stack updates retain the prior image ID', () => { const item = itemById(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), 'previous_images'); - expect(item.state).toBe('not_covered'); + expect(item.state).toBe('ready'); expect(item.detail).toContain('moving image tag'); expect(item.detail).toContain('nginx:latest'); + expect(item.detail).toContain('recovery window'); }); it('does not mistake an image literally named error for a failed preview', () => { @@ -139,9 +140,9 @@ describe('aggregateRollbackOverall', () => { expect(aggregateRollbackOverall(items)).toBe('partial'); }); - it('is partial when the rollback target is a moving tag', () => { + it('is ready when the rollback target is a moving tag (image ID capture covers it)', () => { const items = buildRollbackItems(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), NOW); - expect(aggregateRollbackOverall(items)).toBe('partial'); + expect(aggregateRollbackOverall(items)).toBe('ready'); }); it('is partial when env coverage is missing', () => { diff --git a/backend/src/__tests__/update-guard-service.test.ts b/backend/src/__tests__/update-guard-service.test.ts index ad6c055a..df9896b2 100644 --- a/backend/src/__tests__/update-guard-service.test.ts +++ b/backend/src/__tests__/update-guard-service.test.ts @@ -295,12 +295,11 @@ describe('UpdateGuardService.computeRollbackReadiness moving-tag wiring', () => mockListContainers.mockResolvedValue([]); }); - it('marks previous_images not_covered (overall partial) when any image uses a moving tag', async () => { - // Primary pinned, sidecar on a moving tag: a file rollback cannot revert it. + it('marks previous_images ready when any image uses a moving tag (full-stack image ID capture)', async () => { mockGetPreview.mockResolvedValue(preview([{ current_tag: '1.2.3' }, { current_tag: 'latest' }])); const report = await UpdateGuardService.getInstance().computeRollbackReadiness(0, 'app'); - expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('not_covered'); - expect(report.overall).toBe('partial'); + expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('ready'); + expect(report.overall).toBe('ready'); }); it('marks previous_images ready (overall ready) when every image is pinned', async () => { diff --git a/backend/src/__tests__/webhooks-trigger.test.ts b/backend/src/__tests__/webhooks-trigger.test.ts index ad264d2f..982f96fe 100644 --- a/backend/src/__tests__/webhooks-trigger.test.ts +++ b/backend/src/__tests__/webhooks-trigger.test.ts @@ -481,7 +481,7 @@ describe('WebhookService.execute: health gate begin call sites', () => { const { HealthGateService } = await import('../services/HealthGateService'); vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined); vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]); - vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue(undefined); + vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook'); const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true); diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index e46401d7..f3415e6c 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -12,6 +12,8 @@ import { MonitorService } from '../services/MonitorService'; import { AutoHealService } from '../services/AutoHealService'; import { HealthGateService } from '../services/HealthGateService'; import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService'; +import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService'; +import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService'; import { FleetSyncRetryService } from '../services/FleetSyncRetryService'; import { DockerEventManager } from '../services/DockerEventManager'; import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService'; @@ -134,6 +136,16 @@ export async function startServer(server: Server): Promise { // Initialize the license service before any tier-gated code can run. LicenseService.getInstance().initialize(); + // Deletion-intent reconciliation must finish before mutation-capable + // background services or HTTP accept traffic that could recreate a stack + // name still covered by a prepared/ready tombstone. + try { + await DeployedStackDeletionService.getInstance().reconcileAtStartup(); + } catch (err) { + console.error('[Startup] Deployed stack deletion reconcile failed:', (err as Error).message); + } + StackUpdateRecoveryService.getInstance().start(); + // Synchronous starts: schedule background timers and continue. None of // these fire their first tick for at least a few seconds, so they // safely run alongside the async initializers below. diff --git a/backend/src/routes/imageUpdates.ts b/backend/src/routes/imageUpdates.ts index 213a8380..3b4700c7 100644 --- a/backend/src/routes/imageUpdates.ts +++ b/backend/src/routes/imageUpdates.ts @@ -426,7 +426,13 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp continue; } db.clearStackUpdateStatus(req.nodeId, stackName); - HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`); + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`); + const orchResult = lock.result; + const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; + if (recoveryId) { + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId); + } NotificationService.getInstance().broadcastEvent({ type: 'state-invalidate', diff --git a/backend/src/routes/nodes.ts b/backend/src/routes/nodes.ts index b479afb1..61c36f5e 100644 --- a/backend/src/routes/nodes.ts +++ b/backend/src/routes/nodes.ts @@ -7,6 +7,7 @@ import { requirePermission } from '../middleware/permissions'; import { rejectApiTokenScope } from '../middleware/apiTokenScope'; import { requireAdmin, requirePaid } from '../middleware/tierGates'; import { enrollmentLimiter } from '../middleware/rateLimiters'; +import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService'; import { DatabaseService } from '../services/DatabaseService'; import { NodeRegistry } from '../services/NodeRegistry'; import { CacheService } from '../services/CacheService'; @@ -401,7 +402,14 @@ nodesRouter.delete('/:id', async (req: Request, res: Response) => { // local node, or one with no active connection). Mirrors the re-enroll path. MeshProxyTunnelDialer.getInstance().closeBridge(id, 'node deleted'); PilotTunnelManager.getInstance().closeTunnel(id, PilotCloseCode.NormalClosure, 'node deleted'); - DatabaseService.getInstance().deleteNode(id); + // Local-socket nodes: ready tombstone + recovery-row retirement in the same + // transaction as the node delete, then sweep tags/paths. Remote hub records + // create no Docker cleanup tombstone. + if (existing.type === 'local') { + await DeployedStackDeletionService.getInstance().deleteLocalNode(id); + } else { + DatabaseService.getInstance().deleteNode(id); + } NodeRegistry.getInstance().evictConnection(id); NodeRegistry.getInstance().notifyNodeRemoved(id); CacheService.getInstance().invalidate(`${REMOTE_META_NAMESPACE}:${id}`); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 0d753c4c..4fe318de 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -16,7 +16,6 @@ import { ComposeService, getComposeRollbackInfo } from '../services/ComposeServi import { StackUpdateOrchestrator, shortImageId, type OrchestratorResult } from '../services/StackUpdateOrchestrator'; import DockerController, { type BulkStackInfo } from '../services/DockerController'; import { DatabaseService, type StackDossierFields } from '../services/DatabaseService'; -import { MeshService } from '../services/MeshService'; import { CacheService, type CacheFetchOutcome } from '../services/CacheService'; import { UpdatePreviewService } from '../services/UpdatePreviewService'; import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService'; @@ -38,6 +37,8 @@ import { labelInventoryOptionsFromRequest, requireRevealAdmin } from '../helpers import { EXPOSURE_INTENTS, type ExposureIntent } from '../services/network/types'; import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService'; import { HealthGateService } from '../services/HealthGateService'; +import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService'; +import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService'; import { classifyFailure } from '../services/updateGuard/failureClassifier'; import { requirePermission, checkPermission } from '../middleware/permissions'; import { NotificationService, type NotificationCategory } from '../services/NotificationService'; @@ -99,8 +100,14 @@ const STACK_OP_PRESENT_PARTICIPLE: Record = { update: 'updating', rollback: 'rolling back', backup: 'backing up', + delete: 'deleting', }; +function linkStackUpdateRecoveryGate(recoveryId: string | null | undefined, healthGateId: string | null): void { + if (!recoveryId) return; + StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId); +} + function tryAcquireStackOpLock( req: Request, res: Response, @@ -474,7 +481,7 @@ async function runStackBulkOp( }; } const atomic = true; - await StackUpdateOrchestrator.getInstance().execute( + const orchResult = await StackUpdateOrchestrator.getInstance().execute( { nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'bulk', actor: user }, { atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) }, ); @@ -492,6 +499,8 @@ async function runStackBulkOp( console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err), ); const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); + const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; + linkStackUpdateRecoveryGate(recoveryId, healthGateId); return { stackName, ok: true, healthGateId }; } else { const outcome = await containerActionForStack(req.nodeId, stackName, action); @@ -929,6 +938,12 @@ stacksRouter.post('/', async (req: Request, res: Response) => { if (!isValidStackName(stackName)) { return res.status(400).json({ error: 'Stack name can only contain alphanumeric characters, hyphens, and underscores' }); } + try { + DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(req.nodeId, stackName); + } catch (guardError) { + res.status(409).json({ error: getErrorMessage(guardError, 'Stack deletion in progress'), code: 'stack_deletion_in_progress' }); + return; + } await FileSystemService.getInstance(req.nodeId).createStack(stackName); invalidateNodeCaches(req.nodeId); dlog(`[Stacks] Stack created: ${sanitizeForLog(stackName)}`); @@ -1109,82 +1124,32 @@ stacksRouter.delete('/:stackName', async (req: Request, res: Response) => { const sanitizedName = sanitizeForLog(stackName); if (debug) console.debug(`[Stacks:debug] Delete starting`, { stackName: sanitizedName, pruneVolumes, nodeId: req.nodeId }); - // Step 1: compose down. Best-effort: a stack with corrupt compose files or - // a temporarily-unreachable daemon should still be removable. Logged so the - // operator can investigate orphaned containers separately. - try { - await ComposeService.getInstance(req.nodeId).downStack(stackName); - if (debug) console.debug(`[Stacks:debug] Delete: down OK`, { stackName: sanitizedName }); - } catch (downErr) { - console.warn('[Stacks] Compose down failed or no-op for %s:', sanitizeForLog(stackName), downErr); - } + const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ + nodeId: req.nodeId, + stackName, + pruneVolumes, + actor: req.user?.username ?? 'system', + }); - // Step 2: volume prune (only if requested). Best-effort. - if (pruneVolumes) { - try { - const result = await DockerController.getInstance().pruneManagedOnly('volumes', [stackName]); - dlog(`[Stacks] Pruned volumes for ${sanitizeForLog(stackName)}: ${result.reclaimedBytes} bytes reclaimed`); - } catch (pruneErr) { - console.warn('[Stacks] Volume prune failed for %s, continuing delete:', sanitizeForLog(stackName), pruneErr); + if (!result.ok) { + if (result.code === 'lock_conflict') { + res.status(409).json({ + error: result.error, + code: 'stack_op_in_progress', + action: result.existingAction ?? 'delete', + }); + return; } - } - - // Step 3: filesystem delete. If this fails the on-disk compose files are - // still present, so we abort BEFORE touching the database — keeping DB and - // FS in sync so the operator can retry. Otherwise a half-deleted stack - // (rows gone, files remain) becomes invisible to the UI. - try { - await FileSystemService.getInstance(req.nodeId).deleteStack(stackName); - if (debug) console.debug(`[Stacks:debug] Delete: fs OK`, { stackName: sanitizedName }); - } catch (fsErr) { - console.error('[Stacks] File deletion failed for %s; database state untouched:', sanitizeForLog(stackName), fsErr); - const message = getErrorMessage(fsErr, 'Failed to remove stack files'); - res.status(500).json({ - error: `${message}. Stack containers may have been stopped but on-disk files remain. Retry the delete or clean the files manually.`, - }); + if (result.code === 'fs_failed') { + res.status(500).json({ + error: `${result.error}. Stack containers may have been stopped but on-disk files remain. Retry the delete or clean the files manually.`, + }); + return; + } + res.status(500).json({ error: result.error }); return; } - // Step 4: database cleanup. Per-call idempotent; safe to run sequentially. - try { - DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName); - DatabaseService.getInstance().clearStackScanAttempts(req.nodeId, stackName); - DatabaseService.getInstance().deleteServiceUpdateRecoveries(req.nodeId, stackName); - DatabaseService.getInstance().deleteRoleAssignmentsByResource('stack', stackName); - DatabaseService.getInstance().deleteGitSource(stackName); - DatabaseService.getInstance().deleteStackDossier(req.nodeId, stackName); - DatabaseService.getInstance().deleteStackDriftFindings(req.nodeId, stackName); - DatabaseService.getInstance().deleteStackExposureIntents(req.nodeId, stackName); - DatabaseService.getInstance().deleteStackExposure(req.nodeId, stackName); - DatabaseService.getInstance().deleteStackProjectEnvFiles(req.nodeId, stackName); - DatabaseService.getInstance().deleteStackScans(req.nodeId, stackName); - if (debug) console.debug(`[Stacks:debug] Delete: db OK`, { stackName: sanitizedName }); - } catch (dbErr) { - console.error('[Stacks] Database cleanup failed for %s; files already removed:', sanitizeForLog(stackName), dbErr); - const message = getErrorMessage(dbErr, 'Failed to clear stack database state'); - res.status(500).json({ - error: `${message}. Stack files have been removed; some database rows for this stack may remain. Recreating the stack with the same name will reuse those rows.`, - }); - return; - } - - // Step 5: mesh opt-out cascade. Idempotent; best-effort cleanup of derived - // aliases / override files. A mesh failure here must not flip the delete - // result, since the stack itself is already gone. - try { - await MeshService.getInstance().optOutStack( - req.nodeId, - stackName, - req.user?.username ?? 'system', - ); - } catch (meshErr) { - console.warn( - '[Stacks] Mesh opt-out cascade failed for %s, continuing delete:', - sanitizeForLog(stackName), - meshErr, - ); - } - invalidateNodeCaches(req.nodeId); dlog(`[Stacks] Stack deleted: ${sanitizeForLog(stackName)}`); res.json({ success: true }); @@ -2267,7 +2232,7 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { const debug = isDebugEnabled(); const atomic = true; if (debug) console.debug('[Stacks:debug] Update starting', { stackName, atomic, nodeId: req.nodeId }); - await StackUpdateOrchestrator.getInstance().execute( + const orchResult = await StackUpdateOrchestrator.getInstance().execute( { nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null }, { atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) }, ); @@ -2285,6 +2250,8 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`); ok = true; const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); + const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; + linkStackUpdateRecoveryGate(recoveryId, healthGateId); res.json({ status: 'Update completed', healthGateId }); notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system'); if (!skipScan) { diff --git a/backend/src/services/BlueprintService.ts b/backend/src/services/BlueprintService.ts index 80a1459b..e97299c4 100644 --- a/backend/src/services/BlueprintService.ts +++ b/backend/src/services/BlueprintService.ts @@ -10,6 +10,7 @@ import { } from './DatabaseService'; import { ComposeService } from './ComposeService'; import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService'; +import { DeployedStackDeletionService } from './DeployedStackDeletionService'; import { FileSystemService } from './FileSystemService'; import { NodeRegistry } from './NodeRegistry'; import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers'; @@ -463,32 +464,17 @@ export class BlueprintService { } private async withdrawLocal(blueprint: Blueprint, node: Node): Promise { - // Hold the per-stack lock across both the compose down and the directory - // delete so a withdraw cannot race a manual operation, nor tear the - // files out from under one that starts mid-withdraw. - const lock = await StackOpLockService.getInstance().runExclusive( - node.id, blueprint.name, 'down', 'system', - async () => { - try { - await ComposeService.getInstance(node.id).downStack(blueprint.name); - } catch (err) { - // best-effort: continue to delete the directory even if down fails - console.warn(`[BlueprintService] downStack failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`); - } - if (await this.stackDirExists(node.id, blueprint.name)) { - await FileSystemService.getInstance(node.id).deleteStack(blueprint.name); - } - // Remove the exposure descriptor so a withdrawn blueprint - // stack does not leave a stale row that escalates posture. - try { - DatabaseService.getInstance().deleteStackExposure(node.id, blueprint.name); - } catch (e) { - console.warn(`[BlueprintService] deleteStackExposure failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(e)}`); - } - }, - ); - if (!lock.ran) { - throw new Error(stackOpSkipMessage(blueprint.name, lock.existing.action)); + const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ + nodeId: node.id, + stackName: blueprint.name, + pruneVolumes: false, + actor: 'system:blueprint', + }); + if (!result.ok) { + if (result.code === 'lock_conflict') { + throw new Error(result.error); + } + throw new Error(result.error); } } diff --git a/backend/src/services/ComposeService.ts b/backend/src/services/ComposeService.ts index f6aaf99d..4648b1d3 100644 --- a/backend/src/services/ComposeService.ts +++ b/backend/src/services/ComposeService.ts @@ -143,6 +143,33 @@ export class ComposeService { * same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh * override, rendering the user's authored model without mesh injection. */ + /** Public wrapper for dual-arg assembly and recovery Compose invocations. */ + public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise { + return this.authoredComposeArgs(stackName, action); + } + + public async validateStackForMutation(stackName: string): Promise { + await this.assertRequiredEnvPresent(stackName); + await this.assertSafePilotBindMapping(stackName); + } + + /** + * Render/validate the exact Compose invocation used by mutating operations + * (authored files, env pins, and generated Mesh override) before capture. + */ + public async validateExactComposeInvocation(stackName: string): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + const baseResolved = path.resolve(this.baseDir); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']); + await this.execute('docker', args, stackDir, undefined, true); + } + private async authoredComposeArgs(stackName: string, action: string[]): Promise { const args: string[] = ['compose']; const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); @@ -151,12 +178,23 @@ export class ComposeService { // directory, so deploy/update resolve the same effective config the validator did. args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId)); + const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName); let overridePath: string | null = null; try { overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); } catch (err) { + if (meshEnabled) { + throw err instanceof Error + ? err + : new Error(`Mesh override generation failed: ${String(err)}`); + } console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message)); } + if (meshEnabled && !overridePath) { + throw new Error( + `Mesh override is required for stack "${stackName}" but could not be generated`, + ); + } if (overridePath) { if (filePrefix.length === 0) { // Single-file stack: passing any -f disables compose's auto-discovery, so name @@ -799,7 +837,50 @@ export class ComposeService { startStream(); } - async updateStack(stackName: string, ws?: WebSocket, atomic?: boolean): Promise { + + /** + * Authored (+ mesh) compose args with an optional recovery override layered LAST. + * Used for pinned lifecycle / compensation ups (`--pull never --no-build`). + */ + public async buildComposeArgsWithRecoveryOverride( + stackName: string, + action: string[], + recoveryOverridePath: string | null, + ): Promise { + const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']); + const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__'); + const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel; + const out = [...prefix]; + if (recoveryOverridePath) { + if (!out.includes('-f')) { + const fsSvc = FileSystemService.getInstance(this.nodeId); + const baseFilename = await fsSvc.getComposeFilename(stackName); + out.push('-f', baseFilename); + try { + const userOverride = await fsSvc.getOverrideFilename(stackName); + if (userOverride) out.push('-f', userOverride); + } catch (err) { + const code = (err as { code?: string }).code; + if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') { + throw err; + } + console.warn( + '[ComposeService] could not resolve user compose override for recovery args:', + sanitizeForLog((err as Error).message), + ); + } + } + out.push('-f', recoveryOverridePath); + } + out.push(...action); + return out; + } + + async updateStack( + stackName: string, + ws?: WebSocket, + atomic?: boolean, + ): Promise<{ recoveryId: string | null }> { await this.assertRequiredEnvPresent(stackName); await this.assertSafePilotBindMapping(stackName); const stackDir = path.join(this.baseDir, stackName); @@ -810,48 +891,106 @@ export class ComposeService { if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); }; - if (atomic) { - await this.createAtomicBackup(stackName, 'update', sendOutput); - } + // Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs). + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + const recoverySvc = StackUpdateRecoveryService.getInstance(); + let recoveryId: string | null = null; + let handedOff = false; try { - try { - const dockerController = DockerController.getInstance(this.nodeId); - const legacyContainers = await dockerController.getContainersByStack(stackName); - if (legacyContainers && legacyContainers.length > 0) { - sendOutput(`=== Cleaning up existing containers for clean update ===\n`); - await dockerController.removeContainers(legacyContainers.map((c: any) => c.Id)); - } - } catch (e) { - console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e); - } + sendOutput('=== Validating stack for update ===\n'); + sendOutput('=== Capturing rollback generation ===\n'); + const candidate = await recoverySvc.captureCandidate({ + nodeId: this.nodeId, + stackName, + createdBy: null, + }); + recoveryId = candidate.id; const buildServices = await loadStackBuildServices(this.nodeId, stackName); const buildAware = buildServices.length > 0; - await this.withRegistryAuth(async (env) => { - if (buildAware) { - sendOutput('=== Building images ===\n'); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull']), stackDir, ws, true, env, getComposeStallTimeoutMs()); + try { + await this.withRegistryAuth(async (env) => { + if (buildAware) { + sendOutput('=== Building images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['build', '--pull']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + sendOutput('=== Pulling registry images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + } else { + sendOutput('=== Pulling latest images ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['pull']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + } + }, sendOutput); + } catch (acquireError) { + // Acquisition failure: abandon candidate; leave runtime untouched. + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw acquireError; + } - sendOutput('=== Pulling registry images ===\n'); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), stackDir, ws, true, env, getComposeStallTimeoutMs()); - } else { - sendOutput('=== Pulling latest images ===\n'); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull']), stackDir, ws, true, env, getComposeStallTimeoutMs()); - } - - sendOutput('=== Recreating containers ===\n'); - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); - }, sendOutput); - - // Post-Update Health Probe - await new Promise(resolve => setTimeout(resolve, 3000)); + if (!recoverySvc.markAcquired(candidate.id)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to mark recovery generation as acquired'); + } const dockerController = DockerController.getInstance(this.nodeId); + sendOutput('=== Classifying legacy orphans ===\n'); + const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName); + if (classified.status === 'classification_failed') { + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw new Error(`Legacy orphan classification failed: ${classified.error}`); + } + + if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) { + await recoverySvc.abandon(candidate.id); + recoveryId = null; + throw new Error('Failed to hand off recovery generation'); + } + handedOff = true; + if (!recoverySvc.markReconciling(candidate.id)) { + throw new Error('Failed to mark recovery generation as reconciling after handoff'); + } + + if (classified.status === 'orphans') { + sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`); + const results = await dockerController.removeContainers(classified.ids); + const failed = results.filter((r) => !r.success); + if (failed.length > 0) { + throw new Error( + `Failed to remove ${failed.length} legacy orphan container(s) after handoff`, + ); + } + } + + await this.withRegistryAuth(async (env) => { + sendOutput('=== Recreating containers ===\n'); + await this.execute( + 'docker', + await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), + stackDir, ws, true, env, getComposeStallTimeoutMs(), + ); + }, sendOutput); + + // Immediate verification probe + await new Promise((resolve) => setTimeout(resolve, 3000)); + const containers = await dockerController.getDocker().listContainers({ all: true, - filters: { label: [`com.docker.compose.project=${stackName}`] } + filters: { label: [`com.docker.compose.project=${stackName}`] }, }); for (const containerInfo of containers) { @@ -859,56 +998,90 @@ export class ComposeService { const container = dockerController.getDocker().getContainer(containerInfo.Id); const inspectData = await container.inspect(); const exitCode = inspectData.State.ExitCode; - if (exitCode !== 0) { throw this.createContainerCrashError(exitCode); } } } + if (!recoverySvc.markImmediateVerified(candidate.id)) { + console.warn( + '[ComposeService] Could not CAS immediate_verified for recovery %s', + sanitizeForLog(candidate.id), + ); + } + sendOutput('=== Stack updated successfully ===\n'); - // Opt-out (default ON): after a clean update, prune the node's dangling - // (untagged) image layers, including the one this pull just orphaned. Read - // fresh each run so a remote node honors its own setting. Wrapped so a - // prune failure can never reach the atomic-rollback catch below. + + // Defer prune until gate retention / gate link: only prune when no active holds + // would be violated. Still honor prune_on_update, but use unified holds so + // candidate/current rollback images are retained. try { const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'; if (pruneOnUpdate) { - // Dynamic import: ServiceUpdateRecoveryService imports getComposeCommandTimeoutMs - // from this module, so a static import here would be circular. - const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService'); - const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(this.nodeId); + const isImageHeld = recoverySvc.buildUnifiedHeldImagePredicate(this.nodeId); const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld); - // The Docker prune API does not report SpaceReclaimed on the containerd - // image store, so only show the figure when the daemon actually returns one. const reclaimed = result.reclaimedBytes > 0 ? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB` : ''; sendOutput(`=== Pruned dangling images${reclaimed} ===\n`); } } catch (pruneError) { - console.warn('Failed to prune dangling images after update for %s:', sanitizeForLog(stackName), pruneError); + console.warn( + 'Failed to prune dangling images after update for %s:', + sanitizeForLog(stackName), + pruneError, + ); + } + + if (debug) { + console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName }); } - if (debug) console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName }); } catch (updateError) { - if (atomic) { - sendOutput('\n=== Update failed - restoring previous compose and env files ===\n'); - const rolledBack = await this.restoreAtomicBackup(stackName, stackDir, ws, sendOutput); + if (!handedOff && recoveryId) { + await recoverySvc.abandon(recoveryId); + recoveryId = null; + } + if (handedOff && recoveryId) { + sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n'); + const rolledBack = await recoverySvc.compensateWithCandidate( + recoveryId, + async (overridePath) => { + await this.withRegistryAuth(async (env) => { + await this.execute( + 'docker', + await this.buildComposeArgsWithRecoveryOverride( + stackName, + ['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'], + overridePath, + ), + stackDir, + ws, + true, + env, + getComposeStallTimeoutMs(), + ); + }, sendOutput); + }, + ); throw new ComposeRollbackError(updateError, true, rolledBack); } + // Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched. throw updateError; } - // Reached only on a successful update; re-baseline so temporal drift compares - // against what is now deployed (see deployStack for why this lives here), then - // reconcile the ledger against the updated runtime. + await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName); await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName); try { await this.refreshExposureCache(stackName); } catch (err) { - console.warn('[ComposeService] Exposure refresh failed after update for %s:', - sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown'))); + console.warn( + '[ComposeService] Exposure refresh failed after update for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(err, 'unknown')), + ); } + return { recoveryId }; } /** diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 5f7cce45..406bf374 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -213,6 +213,42 @@ export interface HealthGateRunRow { } /** Pre-update image snapshot enabling a manual per-service restore after a service-scoped update. */ + +/** Full-stack update recovery generation. Separate from service_update_recovery. */ +export interface StackUpdateRecoveryGenerationRow { + id: string; + node_id: number; + stack_name: string; + status: 'candidate' | 'active' | 'restored_current' | 'superseded' | 'abandoned' | 'recovery_required'; + phase: 'captured' | 'acquired' | 'handoff_committed' | 'reconciling' | 'immediate_verified'; + is_current: number; + backup_slot_id: string | null; + override_path: string | null; + services_json: string; + health_gate_id: string | null; + gate_retain_until: number | null; + artifact_expires_at: number | null; + operation_lease_expires_at: number | null; + created_at: number; + updated_at: number; + created_by: string | null; + artifacts_retired: number; +} + +/** Durable cleanup tombstone for stack/node deletion artifact sweep. */ +export interface StackUpdateCleanupPendingRow { + id: string; + node_id: number | null; + stack_name: string | null; + status: 'prepared' | 'ready' | 'cancelled'; + target_kind: 'local_socket'; + rollback_tags_json: string; + override_paths_json: string; + prune_volumes_requested: number; + created_at: number; + updated_at: number; +} + export interface ServiceUpdateRecoveryRow { id: string; node_id: number; @@ -1647,6 +1683,50 @@ export class DatabaseService { CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_claim_expires ON service_update_recovery(status, claim_expires_at); + + CREATE TABLE IF NOT EXISTS stack_update_recovery_generations ( + id TEXT PRIMARY KEY, + node_id INTEGER NOT NULL, + stack_name TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('candidate','active','restored_current','superseded','abandoned','recovery_required')), + phase TEXT NOT NULL CHECK (phase IN ('captured','acquired','handoff_committed','reconciling','immediate_verified')), + is_current INTEGER NOT NULL DEFAULT 0, + backup_slot_id TEXT, + override_path TEXT, + services_json TEXT NOT NULL, + health_gate_id TEXT, + gate_retain_until INTEGER, + artifact_expires_at INTEGER, + operation_lease_expires_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + created_by TEXT, + artifacts_retired INTEGER NOT NULL DEFAULT 0 + ); + CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_node_stack_current + ON stack_update_recovery_generations(node_id, stack_name, is_current); + CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_status_expires + ON stack_update_recovery_generations(status, artifact_expires_at); + CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_phase_lease + ON stack_update_recovery_generations(phase, operation_lease_expires_at); + + CREATE TABLE IF NOT EXISTS stack_update_cleanup_pending ( + id TEXT PRIMARY KEY, + node_id INTEGER, + stack_name TEXT, + status TEXT NOT NULL CHECK (status IN ('prepared','ready','cancelled')), + target_kind TEXT NOT NULL CHECK (target_kind IN ('local_socket')), + rollback_tags_json TEXT NOT NULL, + override_paths_json TEXT NOT NULL, + prune_volumes_requested INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_status + ON stack_update_cleanup_pending(status); + CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_node_stack + ON stack_update_cleanup_pending(node_id, stack_name); + CREATE TABLE IF NOT EXISTS secrets ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE, @@ -1699,6 +1779,8 @@ export class DatabaseService { try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); } catch (e) { /* ignore */ } }; + maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0'); + // Distributed API model columns maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''"); maybeAddCol('nodes', 'api_token', "TEXT DEFAULT ''"); @@ -3510,6 +3592,268 @@ export class DatabaseService { this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName); } + public insertStackUpdateRecoveryGeneration(row: StackUpdateRecoveryGenerationRow): void { + this.db.prepare( + `INSERT INTO stack_update_recovery_generations ( + id, node_id, stack_name, status, phase, is_current, backup_slot_id, override_path, + services_json, health_gate_id, gate_retain_until, artifact_expires_at, + operation_lease_expires_at, created_at, updated_at, created_by, artifacts_retired + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current, + row.backup_slot_id, row.override_path, row.services_json, row.health_gate_id, + row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at, + row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0, + ); + } + + public getStackUpdateRecoveryGeneration(id: string): StackUpdateRecoveryGenerationRow | undefined { + return this.db.prepare('SELECT * FROM stack_update_recovery_generations WHERE id = ?').get(id) as StackUpdateRecoveryGenerationRow | undefined; + } + + public getCurrentStackUpdateRecovery(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined { + return this.db.prepare( + 'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? AND is_current = 1 LIMIT 1' + ).get(nodeId, stackName) as StackUpdateRecoveryGenerationRow | undefined; + } + + public listStackUpdateRecoveryForStack(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow[] { + return this.db.prepare( + 'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? ORDER BY created_at DESC' + ).all(nodeId, stackName) as StackUpdateRecoveryGenerationRow[]; + } + + public listStackUpdateRecoveryGenerationsForNode(nodeId: number): StackUpdateRecoveryGenerationRow[] { + return this.db.prepare( + 'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? ORDER BY created_at DESC' + ).all(nodeId) as StackUpdateRecoveryGenerationRow[]; + } + + public updateStackUpdateRecoveryGeneration( + id: string, + patch: Partial>, + ): void { + const keys = Object.keys(patch) as Array; + if (keys.length === 0) return; + const sets = keys.map((k) => `${String(k)} = ?`).join(', '); + const values = keys.map((k) => patch[k]); + this.db.prepare( + `UPDATE stack_update_recovery_generations SET ${sets}, updated_at = ? WHERE id = ?` + ).run(...values, Date.now(), id); + } + + public casStackUpdateRecoveryPhase( + id: string, + expectedPhase: StackUpdateRecoveryGenerationRow['phase'], + nextPhase: StackUpdateRecoveryGenerationRow['phase'], + ): boolean { + const result = this.db.prepare( + 'UPDATE stack_update_recovery_generations SET phase = ?, updated_at = ? WHERE id = ? AND phase = ?' + ).run(nextPhase, Date.now(), id, expectedPhase); + return result.changes === 1; + } + + public casHandoffGeneration(candidateId: string, nodeId: number, stackName: string): boolean { + const handoff = this.db.transaction(() => { + const candidate = this.getStackUpdateRecoveryGeneration(candidateId); + if (!candidate || candidate.node_id !== nodeId || candidate.stack_name !== stackName) return false; + if (candidate.status !== 'candidate' || candidate.phase !== 'acquired') return false; + this.db.prepare( + `UPDATE stack_update_recovery_generations + SET status = 'superseded', is_current = 0, artifact_expires_at = ?, updated_at = ? + WHERE node_id = ? AND stack_name = ? AND is_current = 1 AND id != ?` + ).run(Date.now() + 7 * 24 * 60 * 60 * 1000, Date.now(), nodeId, stackName, candidateId); + const result = this.db.prepare( + `UPDATE stack_update_recovery_generations + SET status = 'active', is_current = 1, phase = 'handoff_committed', updated_at = ? + WHERE id = ? AND status = 'candidate' AND phase = 'acquired'` + ).run(Date.now(), candidateId); + return result.changes === 1; + }); + return handoff(); + } + + + /** Generations whose Docker/FS artifacts can be retired (not actively held). */ + public listStackUpdateRecoveryGenerationsForArtifactRetirement(now: number): StackUpdateRecoveryGenerationRow[] { + return this.db.prepare( + `SELECT * FROM stack_update_recovery_generations + WHERE artifacts_retired = 0 + AND is_current = 0 + AND status IN ('abandoned', 'superseded') + AND (artifact_expires_at IS NULL OR artifact_expires_at <= ?) + AND (gate_retain_until IS NULL OR gate_retain_until <= ?)` + ).all(now, now) as StackUpdateRecoveryGenerationRow[]; + } + + public markStackUpdateRecoveryArtifactsRetired(id: string): boolean { + const result = this.db.prepare( + `UPDATE stack_update_recovery_generations + SET artifacts_retired = 1, override_path = NULL, updated_at = ? + WHERE id = ? AND artifacts_retired = 0` + ).run(Date.now(), id); + return result.changes === 1; + } + + public abandonStackUpdateRecoveryGeneration(id: string): boolean { + const now = Date.now(); + const result = this.db.prepare( + `UPDATE stack_update_recovery_generations + SET status = 'abandoned', is_current = 0, artifact_expires_at = ?, + operation_lease_expires_at = NULL, updated_at = ? + WHERE id = ? AND status = 'candidate'` + ).run(now, now, id); + return result.changes === 1; + } + + /** Pre-handoff candidates whose operation lease has expired. */ + public listStaleStackUpdateRecoveryCandidates(now: number): StackUpdateRecoveryGenerationRow[] { + return this.db.prepare( + `SELECT * FROM stack_update_recovery_generations + WHERE status = 'candidate' + AND operation_lease_expires_at IS NOT NULL + AND operation_lease_expires_at <= ?` + ).all(now) as StackUpdateRecoveryGenerationRow[]; + } + + /** Post-handoff generations stuck before immediate_verified with an expired lease. */ + public listStuckStackUpdateRecoveryGenerations(now: number): StackUpdateRecoveryGenerationRow[] { + return this.db.prepare( + `SELECT * FROM stack_update_recovery_generations + WHERE status IN ('active', 'restored_current') + AND phase IN ('handoff_committed', 'reconciling') + AND operation_lease_expires_at IS NOT NULL + AND operation_lease_expires_at <= ?` + ).all(now) as StackUpdateRecoveryGenerationRow[]; + } + + public linkStackUpdateRecoveryHealthGate(id: string, healthGateId: string): void { + this.db.prepare( + 'UPDATE stack_update_recovery_generations SET health_gate_id = ?, updated_at = ? WHERE id = ?' + ).run(healthGateId, Date.now(), id); + } + + public setStackUpdateRecoveryGateRetainUntil(id: string, until: number): void { + this.db.prepare( + 'UPDATE stack_update_recovery_generations SET gate_retain_until = ?, updated_at = ? WHERE id = ?' + ).run(until, Date.now(), id); + } + + public listHeldStackUpdateRecoveryImageIds(nodeId: number, now: number): string[] { + const rows = this.db.prepare( + `SELECT services_json FROM stack_update_recovery_generations + WHERE node_id = ? + AND status IN ('candidate','active','restored_current','recovery_required') + AND (artifact_expires_at IS NULL OR artifact_expires_at > ? OR gate_retain_until > ? OR is_current = 1)` + ).all(nodeId, now, now) as Array<{ services_json: string }>; + const ids = new Set(); + for (const row of rows) { + try { + const parsed: unknown = JSON.parse(row.services_json); + if (!Array.isArray(parsed)) continue; + for (const item of parsed) { + if (!item || typeof item !== 'object') continue; + const replicas = (item as { replicas?: unknown }).replicas; + if (Array.isArray(replicas)) { + for (const replica of replicas) { + if (replica && typeof replica === 'object' + && typeof (replica as { imageId?: unknown }).imageId === 'string' + && (replica as { imageId: string }).imageId.trim()) { + ids.add((replica as { imageId: string }).imageId); + } + } + } else if (typeof (item as { imageId?: unknown }).imageId === 'string') { + ids.add((item as { imageId: string }).imageId); + } + } + } catch { + // Corrupt JSON: skip. + } + } + return [...ids]; + } + + public deleteStackUpdateRecoveryGenerations(nodeId: number, stackName: string): void { + this.db.prepare( + 'DELETE FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ?' + ).run(nodeId, stackName); + } + + public insertCleanupPending(row: StackUpdateCleanupPendingRow): void { + this.db.prepare( + `INSERT INTO stack_update_cleanup_pending ( + id, node_id, stack_name, status, target_kind, rollback_tags_json, + override_paths_json, prune_volumes_requested, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + row.id, row.node_id, row.stack_name, row.status, row.target_kind, + row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested, + row.created_at, row.updated_at, + ); + } + + public getCleanupPending(id: string): StackUpdateCleanupPendingRow | undefined { + return this.db.prepare('SELECT * FROM stack_update_cleanup_pending WHERE id = ?').get(id) as StackUpdateCleanupPendingRow | undefined; + } + + public listReadyCleanupPending(): StackUpdateCleanupPendingRow[] { + return this.db.prepare( + "SELECT * FROM stack_update_cleanup_pending WHERE status = 'ready' ORDER BY created_at ASC" + ).all() as StackUpdateCleanupPendingRow[]; + } + + public listPreparedCleanupPending(): StackUpdateCleanupPendingRow[] { + return this.db.prepare( + "SELECT * FROM stack_update_cleanup_pending WHERE status = 'prepared' ORDER BY created_at ASC" + ).all() as StackUpdateCleanupPendingRow[]; + } + + public updateCleanupPendingStatus(id: string, status: StackUpdateCleanupPendingRow['status']): void { + this.db.prepare( + 'UPDATE stack_update_cleanup_pending SET status = ?, updated_at = ? WHERE id = ?' + ).run(status, Date.now(), id); + } + + public deleteCleanupPending(id: string): void { + this.db.prepare('DELETE FROM stack_update_cleanup_pending WHERE id = ?').run(id); + } + + public hasBlockingDeletionIntent(nodeId: number, stackName: string): boolean { + const row = this.db.prepare( + `SELECT id FROM stack_update_cleanup_pending + WHERE node_id = ? AND stack_name = ? AND status IN ('prepared','ready') LIMIT 1` + ).get(nodeId, stackName); + return !!row; + } + + public getPreparedDeletionIntent(nodeId: number, stackName: string): StackUpdateCleanupPendingRow | undefined { + return this.db.prepare( + `SELECT * FROM stack_update_cleanup_pending + WHERE node_id = ? AND stack_name = ? AND status = 'prepared' LIMIT 1` + ).get(nodeId, stackName) as StackUpdateCleanupPendingRow | undefined; + } + + public getDeletionIntentById(id: string): StackUpdateCleanupPendingRow | undefined { + return this.getCleanupPending(id); + } + + public commitStackDeletionReadyTransaction(intentId: string, nodeId: number, stackName: string): boolean { + const run = this.db.transaction(() => { + const intent = this.getCleanupPending(intentId); + if (!intent || intent.status !== 'prepared') return false; + if (intent.node_id !== nodeId || intent.stack_name !== stackName) return false; + this.db.prepare( + "UPDATE stack_update_cleanup_pending SET status = 'ready', updated_at = ? WHERE id = ? AND status = 'prepared'" + ).run(Date.now(), intentId); + this.deleteStackUpdateRecoveryGenerations(nodeId, stackName); + this.deleteServiceUpdateRecoveries(nodeId, stackName); + return true; + }); + return run(); + } + // --- Notification History --- private mapNotificationRow(row: any): NotificationHistory { @@ -3878,7 +4222,16 @@ export class DatabaseService { this.db.prepare(`UPDATE nodes SET ${fields.join(', ')} WHERE id = ?`).run(...values); } - public deleteNode(id: number): void { + /** + * Delete a node row and related per-node state. + * For a non-default local-socket node, pass localCleanup so a ready tombstone + * is inserted in the same transaction (caller enumerates tags/paths first). + * Remote hub node records must omit localCleanup (no Docker tombstone). + */ + public deleteNode( + id: number, + localCleanup?: { tombstoneId: string; tags: string[]; overridePaths: string[] }, + ): void { const node = this.getNode(id); // Protect the last local node regardless of is_default flag. if (node && node.type === 'local' && this.getLocalNodeCount() <= 1) { @@ -3887,7 +4240,25 @@ export class DatabaseService { if (node?.is_default) { throw new Error('Cannot delete the default node'); } + if (localCleanup && node?.type !== 'local') { + throw new Error('Local cleanup tombstones are only valid for local-socket nodes'); + } this.db.transaction(() => { + if (localCleanup && node?.type === 'local') { + const now = Date.now(); + this.insertCleanupPending({ + id: localCleanup.tombstoneId, + node_id: id, + stack_name: null, + status: 'ready', + target_kind: 'local_socket', + rollback_tags_json: JSON.stringify(localCleanup.tags), + override_paths_json: JSON.stringify(localCleanup.overridePaths), + prune_volumes_requested: 0, + created_at: now, + updated_at: now, + }); + } this.db.prepare('DELETE FROM scheduled_task_runs WHERE task_id IN (SELECT id FROM scheduled_tasks WHERE node_id = ?)').run(id); this.db.prepare('DELETE FROM scheduled_tasks WHERE node_id = ?').run(id); this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ?').run(id); @@ -3901,6 +4272,7 @@ export class DatabaseService { this.db.prepare('DELETE FROM preflight_acknowledgements WHERE node_id = ?').run(id); this.db.prepare('DELETE FROM stack_exposure WHERE node_id = ?').run(id); this.db.prepare('DELETE FROM health_gate_runs WHERE node_id = ?').run(id); + this.db.prepare('DELETE FROM stack_update_recovery_generations WHERE node_id = ?').run(id); this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ?').run(id); this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id); this.deleteRoleAssignmentsByResource('node', String(id)); diff --git a/backend/src/services/DeployedStackDeletionService.ts b/backend/src/services/DeployedStackDeletionService.ts new file mode 100644 index 00000000..3bddbd1a --- /dev/null +++ b/backend/src/services/DeployedStackDeletionService.ts @@ -0,0 +1,521 @@ +/** + * Shared deployed-stack deletion lifecycle (manual DELETE + Blueprint withdrawLocal). + * + * prepared -> (down, optional volume prune, FS delete) -> ready transaction that + * retires full-stack recovery generations and service_update_recovery rows, then + * sweeper removes ready tombstone tags/overrides. + */ +import { randomUUID } from 'crypto'; +import Docker from 'dockerode'; +import fs from 'fs/promises'; +import path from 'path'; +import { + DatabaseService, + type StackUpdateCleanupPendingRow, +} from './DatabaseService'; +import { ComposeService } from './ComposeService'; +import DockerController from './DockerController'; +import { FileSystemService } from './FileSystemService'; +import { NodeRegistry } from './NodeRegistry'; +import { MeshService } from './MeshService'; +import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService'; +import { getErrorMessage } from '../utils/errors'; +import { sanitizeForLog } from '../utils/safeLog'; +import { isPathWithinBase, isValidStackName } from '../utils/validation'; + +/** + * Directory that may contain recovery override files for a tombstone sweep. + * Stack-scoped intents are limited to `//`; + * node-wide intents (null stack) use the whole compose root. + */ +export function overrideDeletionContainmentBase( + composeDir: string, + stackName: string | null, +): string | null { + const resolvedCompose = path.resolve(composeDir); + if (!stackName) return resolvedCompose; + if (!isValidStackName(stackName)) return null; + const stackDir = path.resolve(resolvedCompose, stackName); + if (!isPathWithinBase(stackDir, resolvedCompose)) return null; + return stackDir; +} + +export interface DeleteDeployedStackInput { + nodeId: number; + stackName: string; + pruneVolumes: boolean; + actor: string; + /** When true, skip acquiring a new lock (caller already holds delete via continuation). */ + continuationIntentId?: string; +} + +export type DeleteDeployedStackResult = + | { ok: true } + | { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string }; + +function collectArtifactsFromGenerations( + generations: Array<{ override_path: string | null; services_json: string }>, +): { tags: string[]; overridePaths: string[] } { + const tags = new Set(); + const overridePaths = new Set(); + + for (const gen of generations) { + if (gen.override_path) overridePaths.add(gen.override_path); + try { + const parsed: unknown = JSON.parse(gen.services_json); + if (!Array.isArray(parsed)) continue; + for (const svc of parsed) { + if (!svc || typeof svc !== 'object') continue; + const replicas = (svc as { replicas?: unknown }).replicas; + if (!Array.isArray(replicas)) continue; + for (const replica of replicas) { + const tag = replica && typeof replica === 'object' + ? (replica as { rollbackTag?: unknown }).rollbackTag + : null; + if (typeof tag === 'string' && tag.trim()) tags.add(tag); + } + } + } catch { + // Corrupt capture JSON: skip tags for this generation. + } + } + + return { tags: [...tags], overridePaths: [...overridePaths] }; +} + +function collectArtifacts(nodeId: number, stackName: string): { tags: string[]; overridePaths: string[] } { + return collectArtifactsFromGenerations( + DatabaseService.getInstance().listStackUpdateRecoveryForStack(nodeId, stackName), + ); +} + +function parseJsonStringArray(raw: string): string[] { + try { + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) return []; + return parsed.filter((item): item is string => typeof item === 'string'); + } catch { + return []; + } +} + +export class DeployedStackDeletionService { + private static instance: DeployedStackDeletionService; + + public static getInstance(): DeployedStackDeletionService { + if (!DeployedStackDeletionService.instance) { + DeployedStackDeletionService.instance = new DeployedStackDeletionService(); + } + return DeployedStackDeletionService.instance; + } + + public assertNoBlockingDeletionIntent(nodeId: number, stackName: string): void { + if (DatabaseService.getInstance().hasBlockingDeletionIntent(nodeId, stackName)) { + throw new Error( + `Stack "${stackName}" has a deletion in progress and cannot be created or mutated until cleanup finishes.`, + ); + } + } + + public async deleteDeployedStack(input: DeleteDeployedStackInput): Promise { + const { nodeId, stackName, actor } = input; + const locks = StackOpLockService.getInstance(); + + if (input.continuationIntentId) { + const cont = locks.tryAcquireDeletionContinuation({ + intentId: input.continuationIntentId, + nodeId, + stackName, + }); + if (!cont.acquired) { + return { + ok: false, + code: 'lock_conflict', + error: stackOpSkipMessage(stackName, cont.existing.action), + existingAction: cont.existing.action, + }; + } + try { + return await this.runDeletionBody(input, input.continuationIntentId); + } finally { + locks.release(nodeId, stackName); + } + } + + const exclusive = await locks.runExclusive(nodeId, stackName, 'delete', actor, async () => { + return this.runDeletionBody(input); + }); + if (!exclusive.ran) { + return { + ok: false, + code: 'lock_conflict', + error: stackOpSkipMessage(stackName, exclusive.existing.action), + existingAction: exclusive.existing.action, + }; + } + return exclusive.result; + } + + private async runDeletionBody( + input: DeleteDeployedStackInput, + existingIntentId?: string, + ): Promise { + const { nodeId, stackName, pruneVolumes } = input; + const db = DatabaseService.getInstance(); + + let intentId = existingIntentId; + if (!intentId) { + const { tags, overridePaths } = collectArtifacts(nodeId, stackName); + const now = Date.now(); + const row: StackUpdateCleanupPendingRow = { + id: randomUUID(), + node_id: nodeId, + stack_name: stackName, + status: 'prepared', + target_kind: 'local_socket', + rollback_tags_json: JSON.stringify(tags), + override_paths_json: JSON.stringify(overridePaths), + prune_volumes_requested: pruneVolumes ? 1 : 0, + created_at: now, + updated_at: now, + }; + try { + db.insertCleanupPending(row); + } catch (error) { + return { + ok: false, + code: 'tombstone_failed', + error: getErrorMessage(error, 'Failed to prepare stack deletion'), + }; + } + intentId = row.id; + } + + const intent = db.getDeletionIntentById(intentId); + if (!intent || intent.status !== 'prepared') { + return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' }; + } + + try { + await ComposeService.getInstance(nodeId).downStack(stackName); + } catch (downErr) { + console.warn( + '[DeployedStackDeletion] Compose down failed or no-op for %s:', + sanitizeForLog(stackName), + downErr, + ); + } + + if (intent.prune_volumes_requested === 1) { + try { + await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]); + } catch (pruneErr) { + console.warn( + '[DeployedStackDeletion] Volume prune failed for %s, continuing delete:', + sanitizeForLog(stackName), + pruneErr, + ); + } + } + + try { + await FileSystemService.getInstance(nodeId).deleteStack(stackName); + } catch (fsErr) { + db.updateCleanupPendingStatus(intentId, 'cancelled'); + return { + ok: false, + code: 'fs_failed', + error: getErrorMessage(fsErr, 'Failed to remove stack files'), + }; + } + + if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) { + return { + ok: false, + code: 'db_failed', + error: 'Failed to commit stack deletion ready transaction', + }; + } + + try { + db.clearStackUpdateStatus(nodeId, stackName); + db.clearStackScanAttempts(nodeId, stackName); + db.deleteRoleAssignmentsByResource('stack', stackName); + db.deleteGitSource(stackName); + db.deleteStackDossier(nodeId, stackName); + db.deleteStackDriftFindings(nodeId, stackName); + db.deleteStackExposureIntents(nodeId, stackName); + db.deleteStackExposure(nodeId, stackName); + db.deleteStackProjectEnvFiles(nodeId, stackName); + db.deleteStackScans(nodeId, stackName); + } catch (dbErr) { + console.error( + '[DeployedStackDeletion] Secondary DB cleanup failed for %s; recovery rows already retired:', + sanitizeForLog(stackName), + dbErr, + ); + return { + ok: false, + code: 'db_failed', + error: getErrorMessage(dbErr, 'Failed to clear stack database state'), + }; + } + + try { + await MeshService.getInstance().optOutStack(nodeId, stackName, input.actor); + } catch (meshErr) { + console.warn( + '[DeployedStackDeletion] Mesh opt-out failed for %s:', + sanitizeForLog(stackName), + meshErr, + ); + } + + await this.sweepReadyIntent(intentId); + return { ok: true }; + } + + /** + * Remove rollback tags + override paths for a ready tombstone, then drop the row. + * When the node row is already gone (local-node delete), pass a preserved local + * Docker handle and composeDir so we never fall back to a remote default node. + */ + public async sweepReadyIntent( + intentId: string, + opts?: { docker?: Docker; composeDir?: string }, + ): Promise { + const db = DatabaseService.getInstance(); + const intent = db.getCleanupPending(intentId); + if (!intent || intent.status !== 'ready') return; + if (intent.node_id == null) { + db.deleteCleanupPending(intentId); + return; + } + + const tags = parseJsonStringArray(intent.rollback_tags_json); + const overridePaths = parseJsonStringArray(intent.override_paths_json); + + let docker: Docker; + if (opts?.docker) { + docker = opts.docker; + } else if (db.getNode(intent.node_id)) { + docker = DockerController.getInstance(intent.node_id).getDocker(); + } else if (intent.target_kind === 'local_socket') { + // Deleted local node: local_socket tombstones always target the host Docker socket. + docker = new Docker(); + } else { + console.warn( + '[DeployedStackDeletion] Cannot sweep tombstone %s: node gone and target is not local_socket', + sanitizeForLog(intentId), + ); + return; + } + + let incomplete = false; + + for (const tag of tags) { + try { + await docker.getImage(tag).remove({ force: true }); + } catch (error) { + const status = (error as { statusCode?: number }).statusCode; + const message = getErrorMessage(error, 'unknown').toLowerCase(); + if (status === 404 || message.includes('no such image') || message.includes('not found')) { + continue; + } + incomplete = true; + console.warn( + '[DeployedStackDeletion] Failed to remove rollback tag %s: %s', + sanitizeForLog(tag), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + } + + const baseDir = opts?.composeDir + ?? (db.getNode(intent.node_id) + ? FileSystemService.getInstance(intent.node_id).getBaseDir() + : null); + + const containmentBase = baseDir + ? overrideDeletionContainmentBase(baseDir, intent.stack_name) + : null; + if (baseDir && !containmentBase) { + incomplete = true; + console.warn( + '[DeployedStackDeletion] Refusing override sweep: invalid stack containment for tombstone %s', + sanitizeForLog(intentId), + ); + } + + for (const overridePath of overridePaths) { + try { + const resolved = path.resolve(overridePath); + const basename = path.basename(resolved); + if (!/^\.sencho-recovery-[a-f0-9]+\.yml$/i.test(basename)) { + incomplete = true; + console.warn( + '[DeployedStackDeletion] Refusing to delete non-recovery override: %s', + sanitizeForLog(overridePath), + ); + continue; + } + if (containmentBase && !isPathWithinBase(resolved, containmentBase)) { + incomplete = true; + console.warn( + '[DeployedStackDeletion] Refusing to delete override outside stack containment: %s', + sanitizeForLog(overridePath), + ); + continue; + } + if (!baseDir && !path.isAbsolute(resolved)) { + incomplete = true; + console.warn( + '[DeployedStackDeletion] Refusing relative override without compose dir: %s', + sanitizeForLog(overridePath), + ); + continue; + } + if (!containmentBase && baseDir) { + incomplete = true; + continue; + } + await fs.unlink(resolved); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + incomplete = true; + console.warn( + '[DeployedStackDeletion] Failed to delete override %s: %s', + sanitizeForLog(overridePath), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + } + } + + // Keep the ready tombstone when cleanup is incomplete so startup can retry. + if (!incomplete) { + db.deleteCleanupPending(intentId); + } + } + + /** Enumerate rollback tags and override paths for every stack recovery on a node. */ + public collectNodeArtifacts(nodeId: number): { tags: string[]; overridePaths: string[] } { + return collectArtifactsFromGenerations( + DatabaseService.getInstance().listStackUpdateRecoveryGenerationsForNode(nodeId), + ); + } + + /** + * Delete a local-socket node with an atomic ready tombstone, then sweep. + * Remote node records call DatabaseService.deleteNode without cleanup. + */ + public async deleteLocalNode(nodeId: number): Promise { + const db = DatabaseService.getInstance(); + const node = db.getNode(nodeId); + if (!node) throw new Error('Node not found'); + if (node.type !== 'local') { + db.deleteNode(nodeId); + return; + } + // Preserve Docker + compose dir before the row disappears so sweep never + // targets a remote default node. + const docker = DockerController.getInstance(nodeId).getDocker(); + const composeDir = FileSystemService.getInstance(nodeId).getBaseDir(); + const { tags, overridePaths } = this.collectNodeArtifacts(nodeId); + const tombstoneId = randomUUID(); + db.deleteNode(nodeId, { tombstoneId, tags, overridePaths }); + NodeRegistry.getInstance().evictConnection(nodeId); + try { + await this.sweepReadyIntent(tombstoneId, { docker, composeDir }); + } catch (error) { + // Node row is already gone; leave the ready tombstone for startup resume. + console.error( + '[DeployedStackDeletion] Sweep after local-node delete deferred for tombstone %s: %s', + sanitizeForLog(tombstoneId), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + } + + /** + * Startup reconciliation: resume prepared intents, complete ready transitions + * when the directory is already gone, and sweep ready tombstones. + */ + public async reconcileAtStartup(): Promise { + const db = DatabaseService.getInstance(); + for (const intent of db.listPreparedCleanupPending()) { + if (intent.node_id == null || !intent.stack_name) continue; + const nodeId = intent.node_id; + const stackName = intent.stack_name; + const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName); + let dirExists = true; + try { + await fs.access(stackDir); + } catch (accessError) { + const code = (accessError as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + dirExists = false; + } else { + console.warn( + '[DeployedStackDeletion] Startup access error for %s (not treating as absent): %s', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(accessError, 'unknown')), + ); + continue; + } + } + + if (!dirExists) { + if (!db.commitStackDeletionReadyTransaction(intent.id, nodeId, stackName)) { + console.warn( + '[DeployedStackDeletion] Startup ready commit failed for %s/%s', + nodeId, + sanitizeForLog(stackName), + ); + continue; + } + try { + await MeshService.getInstance().optOutStack(nodeId, stackName, 'system:startup'); + } catch (meshErr) { + console.warn( + '[DeployedStackDeletion] Startup mesh opt-out failed for %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(meshErr, 'unknown')), + ); + } + await this.sweepReadyIntent(intent.id); + continue; + } + + const result = await this.deleteDeployedStack({ + nodeId, + stackName, + pruneVolumes: intent.prune_volumes_requested === 1, + actor: 'system:startup', + continuationIntentId: intent.id, + }); + if (!result.ok) { + console.warn( + '[DeployedStackDeletion] Startup resume failed for %s: %s', + sanitizeForLog(stackName), + result.error, + ); + } + } + + for (const intent of db.listReadyCleanupPending()) { + if (intent.node_id != null && intent.stack_name) { + try { + await MeshService.getInstance().optOutStack(intent.node_id, intent.stack_name, 'system:startup'); + } catch (meshErr) { + console.warn( + '[DeployedStackDeletion] Ready-resume mesh opt-out failed for %s: %s', + sanitizeForLog(intent.stack_name), + sanitizeForLog(getErrorMessage(meshErr, 'unknown')), + ); + } + } + await this.sweepReadyIntent(intent.id); + } + } +} diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index cf1effc3..8ee42120 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -24,6 +24,7 @@ import { import type { NetworkingNetworkBase } from './network/networkingTypes'; import { isPathWithinBase } from '../utils/validation'; import { isDebugEnabled } from '../utils/debug'; +import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; import { describeSpawnError } from '../utils/spawnErrors'; import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs'; @@ -2258,6 +2259,57 @@ class DockerController { } } + /** + * Strict Result API for full-stack updates. Never converts Compose-ps + fallback + * failure into empty success. Compose-managed containers are never orphan IDs. + */ + public async classifyLegacyOrphansForUpdate( + stackName: string, + ): Promise< + | { status: 'none' } + | { status: 'orphans'; ids: string[] } + | { status: 'classification_failed'; error: string } + > { + const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName); + const toIds = (list: Array<{ Id?: string }>) => + list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0) + .map((c) => c.Id); + + const fallbackOrphans = async (): Promise< + | { status: 'none' } + | { status: 'orphans'; ids: string[] } + | { status: 'classification_failed'; error: string } + > => { + try { + const ids = toIds(await this.smartFallback(stackName, stackDir)); + return ids.length === 0 ? { status: 'none' } : { status: 'orphans', ids }; + } catch (fallbackError) { + return { + status: 'classification_failed', + error: getErrorMessage(fallbackError, 'Legacy orphan classification failed'), + }; + } + }; + + try { + const composeContainers = await this.fetchComposePsContainers(stackName, stackDir); + // Compose already manages this stack: no legacy orphan cleanup (same as deploy). + if (composeContainers.length > 0) return { status: 'none' }; + return await fallbackOrphans(); + } catch (error) { + const execError = error as NodeJS.ErrnoException & { stderr?: string }; + const mapped = describeSpawnError(execError, { command: 'docker compose ps' }); + const detail = execError.stderr || mapped.message || getErrorMessage(error, 'docker compose ps failed'); + console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail)); + // Unlike getLegacyOrphanContainersByStack, never convert dual failure into empty success. + const fallback = await fallbackOrphans(); + if (fallback.status === 'classification_failed') { + return { status: 'classification_failed', error: String(detail) }; + } + return fallback; + } + } + public async getContainersByStack(stackName: string) { // Resolve the compose dir and the authored prefix for THIS controller's node, // not the process default, so a non-default local node sees its own stack dir diff --git a/backend/src/services/FileSystemService.ts b/backend/src/services/FileSystemService.ts index 96420647..de356b09 100644 --- a/backend/src/services/FileSystemService.ts +++ b/backend/src/services/FileSystemService.ts @@ -545,6 +545,11 @@ export class FileSystemService { } async createStack(stackName: string): Promise { + if (DatabaseService.getInstance().hasBlockingDeletionIntent(this.nodeId, stackName)) { + throw new Error( + `Stack "${stackName}" has a deletion in progress and cannot be created until cleanup finishes.`, + ); + } const stackDir = this.resolveStackDir(stackName); await this.assertRealWithinBase(stackDir); diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index d1f40710..98fe7992 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -1104,7 +1104,13 @@ export class SchedulerService { ); if (!lock.ran) return skipMessage(stackName, lock.existing.action); db.clearStackUpdateStatus(nodeId, stackName); - HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler'); + const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler'); + const orchResult = lock.result; + const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; + if (recoveryId) { + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId); + } this.safeDispatch( 'info', diff --git a/backend/src/services/ServiceUpdateRecoveryService.ts b/backend/src/services/ServiceUpdateRecoveryService.ts index 17779eff..d51c0e63 100644 --- a/backend/src/services/ServiceUpdateRecoveryService.ts +++ b/backend/src/services/ServiceUpdateRecoveryService.ts @@ -237,7 +237,17 @@ export class ServiceUpdateRecoveryService { return (imageId: string) => { const held = this.getHeldImageIds(nodeId); if (held === null) return true; - return held.has(imageId); + if (held.has(imageId)) return true; + try { + // Dynamic import avoids a static cycle with StackUpdateRecoveryService. + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { StackUpdateRecoveryService } = require('./StackUpdateRecoveryService') as typeof import('./StackUpdateRecoveryService'); + const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(nodeId); + if (stackHeld === null) return true; + return stackHeld.has(imageId); + } catch { + return true; + } }; } diff --git a/backend/src/services/StackOpLockService.ts b/backend/src/services/StackOpLockService.ts index ef3bb6c5..fcb49926 100644 --- a/backend/src/services/StackOpLockService.ts +++ b/backend/src/services/StackOpLockService.ts @@ -1,3 +1,4 @@ +import { DatabaseService } from './DatabaseService'; /** * Tracks in-flight stack lifecycle operations (deploy, down, restart, stop, * start, update, rollback, backup) per (nodeId, stackName). A second request to @@ -9,7 +10,7 @@ * which matches the lifecycle of any in-flight `docker compose` child process. */ -export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup'; +export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete'; /** * Note returned by a background path that skipped its operation because a manual @@ -70,6 +71,31 @@ export class StackOpLockService { this.locks.delete(this.key(nodeId, stackName)); } + /** + * Matching-intent continuation for crash-resume of a prepared deletion. + * Succeeds only when the persisted intent matches and no other in-memory lock holds. + */ + public tryAcquireDeletionContinuation(args: { + intentId: string; + nodeId: number; + stackName: string; + }): AcquireResult { + const intent = DatabaseService.getInstance().getDeletionIntentById(args.intentId); + if ( + !intent + || intent.status !== 'prepared' + || intent.node_id !== args.nodeId + || intent.stack_name !== args.stackName + ) { + const existing = this.locks.get(this.key(args.nodeId, args.stackName)); + return { + acquired: false, + existing: existing ?? { action: 'delete', startedAt: Date.now(), user: 'system' }, + }; + } + return this.tryAcquire(args.nodeId, args.stackName, 'delete', 'system:deletion-continuation'); + } + /** * Acquire the per-(nodeId, stackName) lock for the duration of `fn`, then * release it. Returns `{ ran: true, result }` when the lock was free, or diff --git a/backend/src/services/StackUpdateOrchestrator.ts b/backend/src/services/StackUpdateOrchestrator.ts index fb1379fd..737ff31b 100644 --- a/backend/src/services/StackUpdateOrchestrator.ts +++ b/backend/src/services/StackUpdateOrchestrator.ts @@ -47,7 +47,7 @@ export interface ServiceUpdateOptions { } export type OrchestratorResult = - | { kind: 'stack_compose_done' } + | { kind: 'stack_compose_done'; recoveryId: string | null } | { kind: 'service_done'; serviceName: string; @@ -182,10 +182,11 @@ export class StackUpdateOrchestrator { ctx: UpdateOperationContext, options: StackComposeOptions, ): Promise { - await ComposeService.getInstance(ctx.nodeId).updateStack( + const updateResult = await ComposeService.getInstance(ctx.nodeId).updateStack( ctx.stackName, options.terminalWs ?? undefined, options.atomic, ); - return { kind: 'stack_compose_done' }; + const recoveryId = updateResult?.recoveryId ?? null; + return { kind: 'stack_compose_done', recoveryId }; } private async executeServiceUpdate( @@ -195,6 +196,16 @@ export class StackUpdateOrchestrator { ): Promise { const { nodeId, stackName } = ctx; + { + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) { + return serviceFailed( + 'stack_recovery_pin_active', + 'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.', + ); + } + } + const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName); if (!loaded.ok) return loaded.result; const { spec, services } = loaded; @@ -308,6 +319,16 @@ export class StackUpdateOrchestrator { const { nodeId, stackName } = ctx; const recoveryId = options.recoveryId as string; + { + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) { + return serviceFailed( + 'stack_recovery_pin_active', + 'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.', + { recoveryId }, + ); + } + } const recovery = ServiceUpdateRecoveryService.getInstance().get(recoveryId); if ( !recovery || diff --git a/backend/src/services/StackUpdateRecoveryService.ts b/backend/src/services/StackUpdateRecoveryService.ts new file mode 100644 index 00000000..4a6fccd9 --- /dev/null +++ b/backend/src/services/StackUpdateRecoveryService.ts @@ -0,0 +1,767 @@ +/** + * Full-stack update recovery generations: capture, opaque rollback tags, + * stack-local recovery override, handoff, compensation, and prune holds. + * + * Separate from ServiceUpdateRecoveryService (service-scoped snapshots). + * Does not run Compose; ComposeService / orchestrator own Docker mutations. + * + * Recovery fidelity contract (supported): + * - Prior images are restored via opaque hold tags (`--pull never --no-build`). + * - Observed running replica count is restored via compose `scale`. + * - Services that were fully stopped at capture are kept at `scale: 0`. + * - Authored replica counts are not used when they diverge from observed state. + */ +import { randomUUID } from 'crypto'; +import fs from 'fs/promises'; +import path from 'path'; +import { + DatabaseService, + type StackUpdateRecoveryGenerationRow, +} from './DatabaseService'; +import DockerController from './DockerController'; +import { FileSystemService } from './FileSystemService'; +import { buildEffectiveServiceModel } from './effectiveServiceModel'; +import { + classifyReferenceKind, + type ImageReferenceKind, + resolveComposeProjectContext, +} from './composeProjectContext'; +import { getComposeCommandTimeoutMs } from './ComposeService'; +import { getErrorMessage } from '../utils/errors'; +import { sanitizeForLog } from '../utils/safeLog'; +import { isValidStackName } from '../utils/validation'; + +const SWEEP_INTERVAL_MS = 5 * 60_000; +const INITIAL_SWEEP_DELAY_MS = 30_000; +const MIN_RECOVERY_WINDOW_SECONDS = 90; +const RECOVERY_TTL_BUFFER_MS = 30 * 60_000; +const GATE_RETAIN_DEFAULT_MS = 2 * 60 * 60_000; +const RECOVERY_PROBE_DELAY_MS = 3_000; + +export interface StackRecoveryReplicaCapture { + containerId: string | null; + imageId: string | null; + repoDigest: string | null; + state: 'running' | 'stopped' | 'none'; + rollbackTag: string | null; +} + +export interface StackRecoveryServiceCapture { + serviceName: string; + /** Observed running replica count at capture (supported restore scale). */ + scale: number; + hasBuild: boolean; + declaredImageRef: string | null; + referenceKind: ImageReferenceKind; + replicas: StackRecoveryReplicaCapture[]; +} + +export interface CaptureStackUpdateInput { + nodeId: number; + stackName: string; + createdBy: string | null; +} + +function yamlQuote(value: string): string { + return JSON.stringify(value); +} + +function sanitizeServiceSlug(name: string): string { + return name.replace(/[^a-zA-Z0-9._-]/g, '-').toLowerCase() || 'svc'; +} + +function opaqueRollbackTag(generationId: string, serviceName: string): string { + const short = generationId.replace(/-/g, '').slice(0, 12); + return `sencho-rb/${short}/${sanitizeServiceSlug(serviceName)}:hold`; +} + +function parseServicesJson(raw: string): StackRecoveryServiceCapture[] { + try { + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) return []; + return parsed as StackRecoveryServiceCapture[]; + } catch { + return []; + } +} + +export function collectImageIdsFromServicesJson(servicesJson: string): string[] { + const ids = new Set(); + for (const svc of parseServicesJson(servicesJson)) { + for (const replica of svc.replicas ?? []) { + if (replica.imageId && replica.imageId.trim()) ids.add(replica.imageId); + } + } + return [...ids]; +} + +function collectRollbackTags(services: StackRecoveryServiceCapture[]): string[] { + const tags = new Set(); + for (const svc of services) { + for (const replica of svc.replicas ?? []) { + if (replica.rollbackTag) tags.add(replica.rollbackTag); + } + } + return [...tags]; +} + +export class StackUpdateRecoveryService { + private static instance: StackUpdateRecoveryService; + private started = false; + private intervalId: NodeJS.Timeout | null = null; + private initialTimer: NodeJS.Timeout | null = null; + + private constructor() {} + + public static getInstance(): StackUpdateRecoveryService { + if (!StackUpdateRecoveryService.instance) { + StackUpdateRecoveryService.instance = new StackUpdateRecoveryService(); + } + return StackUpdateRecoveryService.instance; + } + + public static resetForTests(): void { + if (StackUpdateRecoveryService.instance) { + StackUpdateRecoveryService.instance.stop(); + } + StackUpdateRecoveryService.instance = new StackUpdateRecoveryService(); + } + + public start(): void { + this.started = true; + if (this.initialTimer || this.intervalId) return; + this.initialTimer = setTimeout(() => { + void this.reconcileIncomplete(); + this.intervalId = setInterval(() => { + void this.reconcileIncomplete(); + }, SWEEP_INTERVAL_MS); + }, INITIAL_SWEEP_DELAY_MS); + } + + public stop(): void { + this.started = false; + if (this.initialTimer) { + clearTimeout(this.initialTimer); + this.initialTimer = null; + } + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + } + + /** + * Validate exact Compose invocation, backup files, snapshot runtime, create + * opaque tags + recovery override, insert candidate generation. + */ + public async captureCandidate(input: CaptureStackUpdateInput): Promise { + const { nodeId, stackName, createdBy } = input; + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack name'); + } + + const context = await resolveComposeProjectContext(nodeId, stackName); + await context.validateForMutation(); + // Exact mutating invocation (authored files + env + generated Mesh override) + // must validate before any backup, tag, or override write. + const { ComposeService } = await import('./ComposeService'); + await ComposeService.getInstance(nodeId).validateExactComposeInvocation(stackName); + + const backupSlotId = await context.backupFromContext('update'); + + const model = await buildEffectiveServiceModel(nodeId, stackName); + if (!model.renderable) { + throw new Error(model.error || 'Effective Compose model failed to render'); + } + + const generationId = randomUUID(); + const docker = DockerController.getInstance(nodeId).getDocker(); + const services: StackRecoveryServiceCapture[] = []; + const createdTags: string[] = []; + let overridePath: string | null = null; + + try { + for (const spec of model.services) { + const declaredImageRef = spec.declaredImage; + const referenceKind = classifyReferenceKind(declaredImageRef); + const listed = await docker.listContainers({ + all: true, + filters: { + label: [ + `com.docker.compose.project=${stackName}`, + `com.docker.compose.service=${spec.name}`, + ], + }, + }); + + const replicas: StackRecoveryReplicaCapture[] = []; + for (const info of listed) { + try { + const inspect = await docker.getContainer(info.Id).inspect(); + const status = inspect.State?.Status; + const state: StackRecoveryReplicaCapture['state'] = + status === 'running' ? 'running' : status ? 'stopped' : 'none'; + const imageId = typeof inspect.Image === 'string' && inspect.Image.length > 0 + ? inspect.Image + : null; + if ((state === 'running' || state === 'stopped') && !imageId) { + throw new Error( + `Service "${spec.name}" replica ${info.Id.slice(0, 12)} has no protectable image id`, + ); + } + let repoDigest: string | null = null; + if (imageId) { + try { + const image = await docker.getImage(imageId).inspect(); + const digests = (image.RepoDigests ?? []) as string[]; + repoDigest = digests.length > 0 ? digests[0] : null; + } catch { + repoDigest = null; + } + } + replicas.push({ + containerId: info.Id, + imageId, + repoDigest, + state, + rollbackTag: null, + }); + } catch (error) { + if ((error as { statusCode?: number })?.statusCode === 404) continue; + throw error; + } + } + + const runningCount = replicas.filter((r) => r.state === 'running').length; + services.push({ + serviceName: spec.name, + scale: runningCount, + hasBuild: spec.hasBuild, + declaredImageRef, + referenceKind, + replicas, + }); + } + + const taggedIds = new Set(); + for (const svc of services) { + const primary = svc.replicas.find((r) => r.imageId) ?? null; + if (!primary?.imageId) continue; + + const tag = opaqueRollbackTag(generationId, svc.serviceName); + const tagKey = `${primary.imageId}|${tag}`; + if (!taggedIds.has(tagKey)) { + const { repo, tagName } = splitOpaqueTag(tag); + await docker.getImage(primary.imageId).tag({ repo, tag: tagName }); + taggedIds.add(tagKey); + createdTags.push(tag); + } + for (const replica of svc.replicas) { + if (replica.imageId === primary.imageId) { + replica.rollbackTag = tag; + } + } + } + + overridePath = await this.writeRecoveryOverride(nodeId, stackName, generationId, services); + + const now = Date.now(); + const row: StackUpdateRecoveryGenerationRow = { + id: generationId, + node_id: nodeId, + stack_name: stackName, + status: 'candidate', + phase: 'captured', + is_current: 0, + backup_slot_id: backupSlotId, + override_path: overridePath, + services_json: JSON.stringify(services), + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: now + getComposeCommandTimeoutMs() + RECOVERY_TTL_BUFFER_MS, + created_at: now, + updated_at: now, + created_by: createdBy, + artifacts_retired: 0, + }; + DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row); + return row; + } catch (error) { + await this.bestEffortRemoveTags(nodeId, createdTags); + if (overridePath) { + try { + await fs.unlink(overridePath); + } catch { + // Best-effort mid-capture cleanup. + } + } + throw error; + } + } + + private async writeRecoveryOverride( + nodeId: number, + stackName: string, + generationId: string, + services: StackRecoveryServiceCapture[], + ): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack name'); + } + // Canonical inline path barrier (same pattern as ComposeService.renderConfig). + const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir()); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + let stackDirReal: string; + let baseReal: string; + try { + [stackDirReal, baseReal] = await Promise.all([ + fs.realpath(stackDir), + fs.realpath(baseResolved), + ]); + } catch { + throw new Error('Stack directory not found'); + } + if (stackDirReal !== baseReal && !stackDirReal.startsWith(baseReal + path.sep)) { + throw new Error('Stack directory escapes compose base'); + } + + const short = generationId.replace(/-/g, '').slice(0, 12); + if (!/^[a-f0-9]{12}$/i.test(short)) { + throw new Error('Invalid recovery generation id'); + } + const filename = `.sencho-recovery-${short}.yml`; + const abs = path.resolve(stackDirReal, filename); + if (!abs.startsWith(stackDirReal + path.sep)) { + throw new Error('Recovery override path escapes stack directory'); + } + + const lines: string[] = ['services:']; + let wroteAny = false; + for (const svc of services) { + const tag = svc.replicas.find((r) => r.rollbackTag)?.rollbackTag ?? null; + const stoppedOnly = + svc.replicas.length > 0 + && svc.replicas.every((r) => r.state === 'stopped' || r.state === 'none'); + if (!tag && svc.scale !== 0 && !stoppedOnly) continue; + const key = /^[a-zA-Z0-9._-]+$/.test(svc.serviceName) ? svc.serviceName : yamlQuote(svc.serviceName); + lines.push(` ${key}:`); + if (tag) { + lines.push(` image: ${yamlQuote(tag)}`); + } + // Observed running count; fully stopped services stay at scale 0. + if (svc.scale === 0 || stoppedOnly) { + lines.push(' scale: 0'); + } else if (svc.scale > 0) { + lines.push(` scale: ${svc.scale}`); + } + wroteAny = true; + } + const body = wroteAny ? `${lines.join('\n')}\n` : 'services: {}\n'; + await fs.writeFile(abs, body, 'utf8'); + return abs; + } + + public markAcquired(id: string): boolean { + return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'captured', 'acquired'); + } + + public handoff(candidateId: string, nodeId: number, stackName: string): boolean { + return DatabaseService.getInstance().casHandoffGeneration(candidateId, nodeId, stackName); + } + + public markReconciling(id: string): boolean { + return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'handoff_committed', 'reconciling'); + } + + public markImmediateVerified(id: string): boolean { + const ok = DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'reconciling', 'immediate_verified'); + if (ok) { + DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(id, { + artifact_expires_at: Date.now() + this.activeRecoveryTtlMs() + RECOVERY_TTL_BUFFER_MS, + operation_lease_expires_at: null, + }); + } + return ok; + } + + /** Mark candidate abandoned in DB and retire Docker/FS artifacts. */ + public async abandon(id: string): Promise { + const row = this.get(id); + const ok = DatabaseService.getInstance().abandonStackUpdateRecoveryGeneration(id); + if (ok && row) { + await this.retireGenerationArtifacts({ ...row, status: 'abandoned', artifacts_retired: 0 }); + } + return ok; + } + + public linkHealthGate(id: string, healthGateId: string): void { + DatabaseService.getInstance().linkStackUpdateRecoveryHealthGate(id, healthGateId); + } + + public setGateRetainUntil(id: string, until: number = Date.now() + GATE_RETAIN_DEFAULT_MS): void { + DatabaseService.getInstance().setStackUpdateRecoveryGateRetainUntil(id, until); + } + + /** Link a health gate when present; otherwise set bounded gate_retain_until. */ + public linkGateOrRetain(id: string, healthGateId: string | null): void { + try { + if (healthGateId) { + this.linkHealthGate(id, healthGateId); + } else { + this.setGateRetainUntil(id); + } + } catch (error) { + console.warn( + '[StackUpdateRecovery] linkGateOrRetain failed for %s; setting retain window:', + sanitizeForLog(id), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + try { + this.setGateRetainUntil(id); + } catch (retainError) { + console.warn( + '[StackUpdateRecovery] setGateRetainUntil failed for %s:', + sanitizeForLog(id), + sanitizeForLog(getErrorMessage(retainError, 'unknown')), + ); + } + } + } + + public get(id: string): StackUpdateRecoveryGenerationRow | undefined { + return DatabaseService.getInstance().getStackUpdateRecoveryGeneration(id); + } + + public getCurrent(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined { + return DatabaseService.getInstance().getCurrentStackUpdateRecovery(nodeId, stackName); + } + + public isRestoredCurrentPinActive(nodeId: number, stackName: string): boolean { + const current = this.getCurrent(nodeId, stackName); + return !!current && current.status === 'restored_current'; + } + + public getHeldImageIds(nodeId: number): Set | null { + try { + const now = Date.now(); + const fromStack = DatabaseService.getInstance().listHeldStackUpdateRecoveryImageIds(nodeId, now); + return new Set(fromStack); + } catch (error) { + console.warn( + '[StackUpdateRecovery] Failed to compute held image ids for node %d:', + nodeId, + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return null; + } + } + + /** + * Unified held-image predicate: service-scoped + full-stack holds. + * Fail closed (skip prune) when either lookup fails. + */ + public buildUnifiedHeldImagePredicate(nodeId: number): (imageId: string) => boolean { + // Dynamic require avoids a static cycle with ServiceUpdateRecoveryService. + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { ServiceUpdateRecoveryService } = require('./ServiceUpdateRecoveryService') as typeof import('./ServiceUpdateRecoveryService'); + const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(nodeId); + const stackHeld = this.getHeldImageIds(nodeId); + if (serviceHeld === null || stackHeld === null) { + return () => true; + } + return (imageId: string) => serviceHeld.has(imageId) || stackHeld.has(imageId); + } + + /** + * Post-handoff compensation: restore files + pinned up, then probe before + * reporting restored_current / immediate_verified. + */ + public async compensateWithCandidate( + generationId: string, + composeUp: (overridePath: string) => Promise, + ): Promise { + const row = this.get(generationId); + if (!row) return false; + try { + const context = await resolveComposeProjectContext(row.node_id, row.stack_name); + await context.restoreFromContext(); + if (!row.override_path) { + throw new Error('Recovery generation has no override path'); + } + await composeUp(row.override_path); + const probeOk = await this.probeRecoveredStack( + row.node_id, + row.stack_name, + row.services_json, + ); + if (!probeOk) { + DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, { + status: 'recovery_required', + }); + return false; + } + DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, { + status: 'restored_current', + phase: 'immediate_verified', + is_current: 1, + artifact_expires_at: null, + }); + return true; + } catch (error) { + console.error( + '[StackUpdateRecovery] Compensation failed for %s: %s', + sanitizeForLog(generationId), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, { + status: 'recovery_required', + }); + return false; + } + } + + /** + * Verify recovered runtime against the captured generation. + * Rejects absent, restarting, dead, exited, or unhealthy expected replicas, + * image-id mismatches vs capture, and any running replica of a scale-0 service. + */ + public async probeRecoveredStack( + nodeId: number, + stackName: string, + servicesJson: string, + ): Promise { + await new Promise((resolve) => setTimeout(resolve, RECOVERY_PROBE_DELAY_MS)); + try { + const expected = parseServicesJson(servicesJson); + const expectedRunning = new Map(); + const expectedImageIds = new Map>(); + const scaleZeroServices = new Set(); + + for (const svc of expected) { + const imageIds = new Set(); + for (const replica of svc.replicas ?? []) { + if (replica.imageId?.trim()) imageIds.add(replica.imageId); + } + if (svc.scale > 0) { + // Fail closed when we cannot verify image identity for expected runners. + if (imageIds.size === 0) return false; + expectedRunning.set(svc.serviceName, svc.scale); + expectedImageIds.set(svc.serviceName, imageIds); + } else { + scaleZeroServices.add(svc.serviceName); + } + } + + const docker = DockerController.getInstance(nodeId).getDocker(); + const containers = await docker.listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${stackName}`] }, + }); + + if (expectedRunning.size > 0 && containers.length === 0) { + return false; + } + + const runningByService = new Map(); + for (const containerInfo of containers) { + const labels = (containerInfo.Labels ?? {}) as Record; + const serviceName = labels['com.docker.compose.service']; + const state = (containerInfo.State || '').toLowerCase(); + + if (state === 'restarting' || state === 'dead') { + return false; + } + + if (state === 'exited' || state === 'created' || state === 'removing') { + if (serviceName && expectedRunning.has(serviceName)) { + return false; + } + continue; + } + + if (state !== 'running') { + if (serviceName && expectedRunning.has(serviceName)) { + return false; + } + continue; + } + + // Captured at scale 0 must stay stopped; any running replica fails the probe. + if (serviceName && scaleZeroServices.has(serviceName)) { + return false; + } + + const inspectData = await docker.getContainer(containerInfo.Id).inspect(); + const health = inspectData.State?.Health?.Status; + if (health === 'unhealthy') { + return false; + } + + if (serviceName && expectedImageIds.has(serviceName)) { + const allowedIds = expectedImageIds.get(serviceName)!; + const actualImageId = typeof inspectData.Image === 'string' ? inspectData.Image : ''; + if (!actualImageId || !allowedIds.has(actualImageId)) { + return false; + } + } + + if (serviceName) { + runningByService.set(serviceName, (runningByService.get(serviceName) ?? 0) + 1); + } + } + + for (const [serviceName, need] of expectedRunning) { + if ((runningByService.get(serviceName) ?? 0) < need) { + return false; + } + } + return true; + } catch (error) { + console.warn( + '[StackUpdateRecovery] Recovery probe failed for %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return false; + } + } + + /** + * Idempotent removal of opaque tags and override files for a generation. + * Marks artifacts_retired only when every artifact is removed or confirmed absent, + * so transient failures remain retryable via reconcileIncomplete. + */ + public async retireGenerationArtifacts(row: StackUpdateRecoveryGenerationRow): Promise { + if (row.artifacts_retired === 1) return true; + const services = parseServicesJson(row.services_json); + const tagsOk = await this.removeRollbackTags(row.node_id, collectRollbackTags(services)); + let overrideOk = true; + if (row.override_path) { + try { + await fs.unlink(row.override_path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + overrideOk = false; + console.warn( + '[StackUpdateRecovery] Failed to delete override %s: %s', + sanitizeForLog(row.override_path), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + } + } + if (!tagsOk || !overrideOk) return false; + DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id); + return true; + } + + /** + * Abandon lease-expired candidates, flag stuck post-handoff generations, + * and retire expired abandoned/superseded artifacts. + */ + public async reconcileIncomplete(): Promise { + if (!this.started) return; + try { + const db = DatabaseService.getInstance(); + const now = Date.now(); + let abandoned = 0; + for (const row of db.listStaleStackUpdateRecoveryCandidates(now)) { + if (await this.abandon(row.id)) abandoned += 1; + } + let flagged = 0; + for (const row of db.listStuckStackUpdateRecoveryGenerations(now)) { + db.updateStackUpdateRecoveryGeneration(row.id, { + status: 'recovery_required', + operation_lease_expires_at: null, + }); + flagged += 1; + } + let retired = 0; + for (const row of db.listStackUpdateRecoveryGenerationsForArtifactRetirement(now)) { + // Never retire an active/current or recovery_required hold target. + if (row.is_current === 1 || row.status === 'recovery_required') continue; + if (await this.retireGenerationArtifacts(row)) retired += 1; + } + if (abandoned > 0 || flagged > 0 || retired > 0) { + console.log( + `[StackUpdateRecovery] Reconciled ${abandoned} stale candidate(s), ` + + `${flagged} stuck generation(s), retired ${retired} artifact set(s)`, + ); + } + } catch (error) { + console.error( + '[StackUpdateRecovery] Reconcile failed: %s', + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + } + + /** Returns true when every tag is removed or already absent. */ + private async removeRollbackTags(nodeId: number, tags: string[]): Promise { + if (tags.length === 0) return true; + try { + const docker = DockerController.getInstance(nodeId).getDocker(); + let allOk = true; + for (const tag of tags) { + try { + await docker.getImage(tag).remove({ force: true }); + } catch (error) { + const status = (error as { statusCode?: number }).statusCode; + const message = getErrorMessage(error, 'unknown').toLowerCase(); + if (status === 404 || message.includes('no such image') || message.includes('not found')) { + continue; + } + allOk = false; + console.warn( + '[StackUpdateRecovery] Failed to remove rollback tag %s: %s', + sanitizeForLog(tag), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + } + } + return allOk; + } catch (error) { + console.warn( + '[StackUpdateRecovery] Docker unavailable while removing tags: %s', + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return false; + } + } + + private async bestEffortRemoveTags(nodeId: number, tags: string[]): Promise { + await this.removeRollbackTags(nodeId, tags); + } + + private activeRecoveryTtlMs(): number { + return Math.max(getComposeCommandTimeoutMs(), this.readRecoveryWindowSeconds() * 1000); + } + + private readRecoveryWindowSeconds(): number { + try { + const raw = parseInt( + DatabaseService.getInstance().getGlobalSettings()['health_gate_window_seconds'] ?? '', + 10, + ); + return Number.isFinite(raw) ? Math.max(raw, MIN_RECOVERY_WINDOW_SECONDS) : MIN_RECOVERY_WINDOW_SECONDS; + } catch (error) { + console.warn( + '[StackUpdateRecovery] Settings read failed; using default window: %s', + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return MIN_RECOVERY_WINDOW_SECONDS; + } + } +} + +function splitOpaqueTag(tag: string): { repo: string; tagName: string } { + const lastColon = tag.lastIndexOf(':'); + if (lastColon > 0) { + return { repo: tag.slice(0, lastColon), tagName: tag.slice(lastColon + 1) }; + } + return { repo: tag, tagName: 'hold' }; +} diff --git a/backend/src/services/WebhookService.ts b/backend/src/services/WebhookService.ts index 9417420e..22ac714a 100644 --- a/backend/src/services/WebhookService.ts +++ b/backend/src/services/WebhookService.ts @@ -174,18 +174,24 @@ export class WebhookService { case 'start': await compose.runCommand(stackName, 'start'); break; - case 'pull': + case 'pull': { await assertPolicyGateAllows( stackName, nodeId, buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }), ); - await StackUpdateOrchestrator.getInstance().execute( + const orchResult = await StackUpdateOrchestrator.getInstance().execute( { nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' }, { atomic: atomic ?? false, terminalWs: null }, ); - HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook'); + const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook'); + const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; + if (recoveryId) { + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId); + } break; + } } }, ); diff --git a/backend/src/services/composeProjectContext.ts b/backend/src/services/composeProjectContext.ts new file mode 100644 index 00000000..bfeafe07 --- /dev/null +++ b/backend/src/services/composeProjectContext.ts @@ -0,0 +1,94 @@ +/** + * Thin Compose project context for safe full-stack updates. + * + * Wraps the current authored compose argument path and atomic file backup/restore. + * When a richer shared Compose project context lands, migrate callers to that type; + * this module must not become a competing full-manifest resolver. + */ +import path from 'path'; +import { randomUUID } from 'crypto'; +import { ComposeService } from './ComposeService'; +import { FileSystemService } from './FileSystemService'; +import { buildEffectiveServiceModel } from './effectiveServiceModel'; +import { getErrorMessage } from '../utils/errors'; + +export type ImageReferenceKind = 'moving_tag' | 'digest_pinned' | 'none'; + +const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i; + +export function classifyReferenceKind(declaredImageRef: string | null): ImageReferenceKind { + if (!declaredImageRef) return 'none'; + if (DIGEST_PIN_PATTERN.test(declaredImageRef)) return 'digest_pinned'; + return 'moving_tag'; +} + +async function requireRenderableModel(nodeId: number, stackName: string) { + const model = await buildEffectiveServiceModel(nodeId, stackName); + if (!model.renderable) { + throw new Error(model.error || 'Effective Compose model failed to render'); + } + return model; +} + +export interface ComposeProjectContext { + readonly nodeId: number; + readonly stackName: string; + readonly stackDir: string; + backupSlotId: string | null; + toComposeArgs(action: string[]): Promise; + validateForMutation(): Promise; + backupFromContext(operation: 'update' | 'deployment'): Promise; + restoreFromContext(): Promise; + resolveServiceImageMap(): Promise>; +} + +class AuthoredComposeProjectContext implements ComposeProjectContext { + backupSlotId: string | null = null; + + constructor( + readonly nodeId: number, + readonly stackName: string, + readonly stackDir: string, + ) {} + + async toComposeArgs(action: string[]): Promise { + return ComposeService.getInstance(this.nodeId).buildAuthoredComposeArgs(this.stackName, action); + } + + async validateForMutation(): Promise { + await ComposeService.getInstance(this.nodeId).validateStackForMutation(this.stackName); + await requireRenderableModel(this.nodeId, this.stackName); + } + + async backupFromContext(_operation: 'update' | 'deployment'): Promise { + await FileSystemService.getInstance(this.nodeId).backupStackFiles(this.stackName); + const slotId = randomUUID(); + this.backupSlotId = slotId; + return slotId; + } + + async restoreFromContext(): Promise { + await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName); + } + + async resolveServiceImageMap(): Promise> { + const model = await requireRenderableModel(this.nodeId, this.stackName); + const map = new Map(); + for (const svc of model.services) { + map.set(svc.name, svc.declaredImage); + } + return map; + } +} + +export async function resolveComposeProjectContext( + nodeId: number, + stackName: string, +): Promise { + const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName); + return new AuthoredComposeProjectContext(nodeId, stackName, stackDir); +} + +export function describeContextError(error: unknown): string { + return getErrorMessage(error, 'Compose project context failed'); +} diff --git a/backend/src/services/updateGuard/readiness.ts b/backend/src/services/updateGuard/readiness.ts index 854f30d5..a6d824a5 100644 --- a/backend/src/services/updateGuard/readiness.ts +++ b/backend/src/services/updateGuard/readiness.ts @@ -304,11 +304,11 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac if (inputs.rollbackTarget === 'error') { items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The update preview is unavailable.' }); } else if (inputs.rollbackTarget.target && inputs.rollbackTarget.moving) { - items.push({ id: 'previous_images', state: 'not_covered', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag, so restoring the compose and env files does not revert the image: the local tag still resolves to the newer digest. Pin every image to an immutable version tag for a true image rollback.` }); + items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag. Full-stack updates capture the running image ID before pull/build and retain it through the recovery window so an immediate restore can retarget the exact prior image, not only the tag string.` }); } else if (inputs.rollbackTarget.target) { items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Known rollback target: ${inputs.rollbackTarget.target}. The compose file pins an immutable tag, so restoring files also restores the image.` }); } else { - items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. A rollback restores compose and env files; a moving tag may keep the newer image.' }); + items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. When no prior image is recorded, a restore may fall back to compose and env files alone; a moving tag can still resolve to a newer digest.' }); } if (inputs.lastDeployAt === 'error') { diff --git a/backend/vitest.config.ts b/backend/vitest.config.ts index 1b868de1..11b3ba50 100644 --- a/backend/vitest.config.ts +++ b/backend/vitest.config.ts @@ -5,7 +5,7 @@ export default defineConfig({ environment: 'node', // Only run TypeScript sources - exclude the compiled dist/ output. include: ['src/__tests__/**/*.test.ts'], - exclude: ['dist/**', 'node_modules/**'], + exclude: ['dist/**', 'node_modules/**', 'src/__tests__/docker-integration/**'], // Build the baseline DB (schema + migrations + admin seed) once; each // test file's setupTestDb copies it instead of re-running migrations. globalSetup: ['./src/__tests__/helpers/vitestGlobalSetup.ts'], diff --git a/backend/vitest.docker-integration.config.ts b/backend/vitest.docker-integration.config.ts new file mode 100644 index 00000000..1e28be21 --- /dev/null +++ b/backend/vitest.docker-integration.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'node', + include: ['src/__tests__/docker-integration/**/*.test.ts'], + exclude: ['dist/**', 'node_modules/**'], + globalSetup: ['./src/__tests__/helpers/vitestGlobalSetup.ts'], + pool: 'forks', + maxWorkers: 1, + minWorkers: 1, + testTimeout: 180_000, + hookTimeout: 180_000, + sequence: { concurrent: false }, + }, +}); diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts index 011ace82..f56a9465 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts @@ -19,6 +19,7 @@ vi.mock('@/components/ui/toast-store', () => ({ })); import { apiFetch } from '@/lib/api'; +import { toast } from '@/components/ui/toast-store'; type EditorState = ReturnType; type StackListState = ReturnType; @@ -314,6 +315,18 @@ describe('useStackActions policy-block dialog wiring', () => { expect(overlayState.setPolicyBlock).not.toHaveBeenCalled(); }); + it('toasts a deletion-in-progress message when the conflict action is delete', async () => { + const inProgress = JSON.stringify({ + code: 'stack_op_in_progress', + inProgress: { action: 'delete', startedAt: Date.now(), user: 'admin' }, + }); + vi.mocked(apiFetch).mockResolvedValueOnce(new Response(inProgress, { status: 409 })); + const { result, overlayState } = setup(); + await act(async () => { await result.current.updateStack(mouseEvent); }); + expect(overlayState.setPolicyBlock).not.toHaveBeenCalled(); + expect(toast.error).toHaveBeenCalledWith(expect.stringMatching(/already deleting/i)); + }); + it('opens the dialog with action "update" via the sidebar update entry point', async () => { vi.mocked(apiFetch).mockResolvedValueOnce(new Response(JSON.stringify(policyPayload), { status: 409 })); const { result, overlayState } = setup(); diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.ts index 74d90e83..db8458e0 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.ts @@ -107,7 +107,7 @@ const parseFailureClassification = (value: unknown): FailureClassification | und return undefined; }; -type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update'; +type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'delete'; interface StackOpInProgressInfo { action: StackOpAction; @@ -122,6 +122,7 @@ const STACK_OP_PRESENT_PARTICIPLE: Record = { stop: 'stopping', start: 'starting', update: 'updating', + delete: 'deleting', }; const VALID_STACK_OP_ACTIONS: ReadonlySet = new Set( diff --git a/frontend/src/components/EditorLayout/operation-phase.ts b/frontend/src/components/EditorLayout/operation-phase.ts index 53da7e09..2493ac7e 100644 --- a/frontend/src/components/EditorLayout/operation-phase.ts +++ b/frontend/src/components/EditorLayout/operation-phase.ts @@ -38,7 +38,7 @@ export function classifyOperationPhase(rows: ParsedLogRow[], action: ActionVerb) if (stage === 'BUILD') { return 'Building images'; } - if (message.includes('Backup created for atomic') || message.includes('Cleaning up existing containers')) { + if (message.includes('Backup created for atomic') || message.includes('Validating stack for update') || message.includes('Capturing rollback generation')) { return 'Preparing'; } }