fix: keep running containers until stack pull/build succeeds (#1657)

* fix: keep running containers until stack pull/build succeeds

Acquire images before reconcile, capture a recovery generation for
compensation, and only remove classified orphans after handoff.

* fix: address recovery audit blockers for safe stack updates

Retire abandoned and expired recovery artifacts, probe compensated
runtimes before reporting rollback success, preserve local Docker when
deleting a node, validate the exact Compose invocation before capture,
and repair updateStack return-contract fixtures.

* fix: resolve ESLint errors blocking CI on this branch

Unused-import and unused-variable errors left over from the stack
deletion refactor: MeshService in stacks.ts (its opt-out cascade moved
into DeployedStackDeletionService), a redundant pruneVolumes
destructure in deleteDeployedStack (the real one is re-derived from
the same input object inside runDeletionBody), and an unused beforeAll
import in a Docker-integration test stub. Also scopes the webhook
pull-action case body in a block to satisfy no-case-declarations;
purely syntactic, no behavior change.

* fix: harden recovery probe, cleanup retry, and failed-pull Docker test

Reject absent or unhealthy expected replicas before reporting rollback
success, keep cleanup records until artifacts are actually removed, fail
closed when a mesh override cannot be generated, and assert a real
failed pull leaves the original container running.

* fix: verify recovery probe image identity and stack-scoped override paths

Reject recovered runtimes that use the wrong image or leave scale-zero
services running, and confine tombstone override deletion to the intent
stack directory so forged cross-stack paths cannot be swept.

* test: batch notification cap fixtures in a SQLite transaction

Unbatched 1200-row inserts were timing out at the default 30s under
CI load even though the same assertions pass in under 2s when green.

.github/workflows/ci.yml

@@ -49,6 +49,10 @@ jobs:
         working-directory: ./backend
         run: npm test
 
+      - name: Docker integration tests (dedicated Vitest config)
+        working-directory: ./backend
+        run: npm run test:docker-integration
+
       - name: Lint (ESLint)
         working-directory: ./backend
         run: npm run lint

backend/package.json

@@ -10,6 +10,7 @@
     "start": "node dist/index.js",
     "predev": "node scripts/generate-version.js",
     "dev": "nodemon --watch src --ext ts,json --exec ts-node src/index.ts",
+    "test:docker-integration": "vitest run --config vitest.docker-integration.config.ts",
     "test": "vitest run",
     "lint": "eslint src",
     "reset-mfa": "node dist/cli/resetMfa.js",

backend/src/__tests__/compose-service.test.ts

@@ -24,6 +24,17 @@ const {
   mockResolveMissingExternalNetworks,
   mockCreateNetwork,
   mockAddNotificationHistory,
+  mockIsMeshStackEnabled,
+  mockClassifyLegacyOrphansForUpdate,
+  mockCaptureCandidate,
+  mockMarkAcquired,
+  mockHandoff,
+  mockMarkReconciling,
+  mockMarkImmediateVerified,
+  mockAbandon,
+  mockCompensateWithCandidate,
+  mockBuildUnifiedHeldImagePredicate,
+  mockGetRecovery,
 } = vi.hoisted(() => ({
   mockSpawn: vi.fn(),
   mockGetContainersByStack: vi.fn().mockResolvedValue([]),
@@ -58,6 +69,24 @@ const {
   }),
   mockCreateNetwork: vi.fn().mockResolvedValue({ id: 'net-1' }),
   mockAddNotificationHistory: vi.fn(),
+  mockIsMeshStackEnabled: vi.fn().mockReturnValue(false),
+  mockClassifyLegacyOrphansForUpdate: vi.fn().mockResolvedValue({ status: 'none' }),
+  mockCaptureCandidate: vi.fn().mockResolvedValue({
+    id: 'recovery-1',
+    node_id: 1,
+    stack_name: 'my-stack',
+    status: 'candidate',
+    phase: 'captured',
+    override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml',
+  }),
+  mockMarkAcquired: vi.fn().mockReturnValue(true),
+  mockHandoff: vi.fn().mockReturnValue(true),
+  mockMarkReconciling: vi.fn().mockReturnValue(true),
+  mockMarkImmediateVerified: vi.fn().mockReturnValue(true),
+  mockAbandon: vi.fn().mockResolvedValue(true),
+  mockCompensateWithCandidate: vi.fn().mockResolvedValue(true),
+  mockBuildUnifiedHeldImagePredicate: vi.fn().mockReturnValue(() => false),
+  mockGetRecovery: vi.fn().mockReturnValue(undefined),
 }));
 
 vi.mock('child_process', () => ({ spawn: mockSpawn, execFile: vi.fn() }));
@@ -89,6 +118,7 @@ vi.mock('../services/DockerController', () => ({
     getInstance: () => ({
       getContainersByStack: mockGetContainersByStack,
       getLegacyOrphanContainersByStack: mockGetLegacyOrphanContainersByStack,
+      classifyLegacyOrphansForUpdate: mockClassifyLegacyOrphansForUpdate,
       removeContainers: mockRemoveContainers,
       pruneDanglingImages: mockPruneDanglingImages,
       createNetwork: mockCreateNetwork,
@@ -111,6 +141,7 @@ vi.mock('../services/DatabaseService', () => ({
       getGitSource: () => undefined,
       getStackProjectEnvFiles: () => [],
       addNotificationHistory: mockAddNotificationHistory,
+      isMeshStackEnabled: (...args: unknown[]) => mockIsMeshStackEnabled(...args),
     }),
   },
 }));
@@ -150,6 +181,24 @@ vi.mock('../services/MeshService', () => ({
   },
 }));
 
+vi.mock('../services/StackUpdateRecoveryService', () => ({
+  StackUpdateRecoveryService: {
+    getInstance: () => ({
+      captureCandidate: mockCaptureCandidate,
+      markAcquired: mockMarkAcquired,
+      handoff: mockHandoff,
+      markReconciling: mockMarkReconciling,
+      markImmediateVerified: mockMarkImmediateVerified,
+      abandon: mockAbandon,
+      compensateWithCandidate: mockCompensateWithCandidate,
+      buildUnifiedHeldImagePredicate: mockBuildUnifiedHeldImagePredicate,
+      get: mockGetRecovery,
+      linkGateOrRetain: vi.fn(),
+    }),
+  },
+}));
+
+
 vi.mock('../services/SelfIdentityService', () => ({
   default: {
     getInstance: () => ({ getBindMounts: mockGetBindMounts }),
@@ -247,6 +296,23 @@ beforeEach(() => {
     declaredExternalCount: 0,
   });
   mockCreateNetwork.mockResolvedValue({ id: 'net-1' });
+  mockClassifyLegacyOrphansForUpdate.mockResolvedValue({ status: 'none' });
+  mockCaptureCandidate.mockResolvedValue({
+    id: 'recovery-1',
+    node_id: 1,
+    stack_name: 'my-stack',
+    status: 'candidate',
+    phase: 'captured',
+    override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml',
+  });
+  mockMarkAcquired.mockReturnValue(true);
+  mockHandoff.mockReturnValue(true);
+  mockMarkReconciling.mockReturnValue(true);
+  mockMarkImmediateVerified.mockReturnValue(true);
+  mockAbandon.mockResolvedValue(true);
+  mockIsMeshStackEnabled.mockReturnValue(false);
+  mockCompensateWithCandidate.mockResolvedValue(true);
+  mockBuildUnifiedHeldImagePredicate.mockReturnValue(() => false);
   delete process.env.SENCHO_MODE;
   vi.useFakeTimers({ shouldAdvanceTime: true });
 });
@@ -586,6 +652,16 @@ describe('ComposeService - authoredComposeArgs mesh override', () => {
 
 // ── deployStack ────────────────────────────────────────────────────────
 
+
+  it('fails closed when a mesh-enabled stack cannot generate its override', async () => {
+    mockIsMeshStackEnabled.mockReturnValue(true);
+    mockEnsureStackOverride.mockRejectedValue(new Error('mesh override write failed'));
+
+    const svc = ComposeService.getInstance(1);
+    await expect(svc.validateExactComposeInvocation('my-stack')).rejects.toThrow(/mesh override write failed/i);
+    expect(mockSpawn).not.toHaveBeenCalled();
+  });
+
 describe('ComposeService - deployStack', () => {
   it('blocks a pilot deploy with relative binds before replacing containers when path mapping differs', async () => {
     process.env.SENCHO_MODE = 'pilot';
@@ -987,7 +1063,7 @@ describe('ComposeService - updateStack build-aware', () => {
     expect(spawnArgs.some(args => args.includes('pull') && !args.includes('--ignore-buildable'))).toBe(true);
   });
 
-  it('rolls back compose files when a build step fails during atomic update', async () => {
+  it('abandons recovery and leaves runtime untouched when acquire fails before handoff', async () => {
     mockLoadStackBuildServices.mockResolvedValueOnce(['app']);
     let spawnCount = 0;
     mockSpawn.mockImplementation(() => {
@@ -1004,8 +1080,12 @@ describe('ComposeService - updateStack build-aware', () => {
     const error = await result;
 
     expect(error).not.toBeNull();
-    expect(mockRestoreStackFiles).toHaveBeenCalled();
+    expect(mockAbandon).toHaveBeenCalledWith('recovery-1');
-    expect(getComposeRollbackInfo(error)?.attempted).toBe(true);
+    expect(mockHandoff).not.toHaveBeenCalled();
+    expect(mockRemoveContainers).not.toHaveBeenCalled();
+    expect(mockCompensateWithCandidate).not.toHaveBeenCalled();
+    const spawnArgs = mockSpawn.mock.calls.map(c => c[1] as string[]);
+    expect(spawnArgs.some(args => args.includes('up'))).toBe(false);
   });
 });
 
@@ -1106,7 +1186,7 @@ describe('ComposeService - updateStack prune-on-update', () => {
 
     // The update already succeeded before the prune ran, so a prune failure
     // must neither reject nor trigger the atomic restore.
-    await expect(promise).resolves.toBeUndefined();
+    await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' });
     expect(mockRestoreStackFiles).not.toHaveBeenCalled();
   });
 
@@ -1120,7 +1200,7 @@ describe('ComposeService - updateStack prune-on-update', () => {
     const promise = svc.updateStack('my-stack');
     await vi.advanceTimersByTimeAsync(3100);
 
-    await expect(promise).resolves.toBeUndefined();
+    await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' });
   });
 });
 
@@ -1428,19 +1508,10 @@ describe('ComposeService - idle-output stall backstop', () => {
     await promise;
   });
 
-  it('preserves STACK_STALLED_OUTPUT through the atomic rollback wrapper', async () => {
+  it('surfaces STACK_STALLED_OUTPUT on acquire stall without runtime compensation', async () => {
     process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000';
     const pullProc = createMockProcess();
-    let call = 0;
+    mockSpawn.mockImplementation(() => pullProc);
-    // First spawn is the stalling pull; later spawns (the rollback restore's
-    // `up`) close cleanly, proving the restore is not idle-timeout armed.
-    mockSpawn.mockImplementation(() => {
-      call += 1;
-      if (call === 1) return pullProc;
-      const p = createMockProcess();
-      Promise.resolve().then(() => p.emit('close', 0));
-      return p;
-    });
 
     const svc = ComposeService.getInstance(1);
     const result = svc.updateStack('my-stack', undefined, true).then(() => null, (e: Error) => e);
@@ -1451,7 +1522,10 @@ describe('ComposeService - idle-output stall backstop', () => {
     const error = await result;
     expect(error).not.toBeNull();
     expect(error!.message).toContain('STACK_STALLED_OUTPUT');
-    expect(getComposeRollbackInfo(error)).toMatchObject({ attempted: true });
+    // Acquire failure is pre-handoff: abandon candidate, do not wrap as ComposeRollbackError.
+    expect(getComposeRollbackInfo(error)).toBeNull();
+    expect(mockAbandon).toHaveBeenCalledWith('recovery-1');
+    expect(mockCompensateWithCandidate).not.toHaveBeenCalled();
   });
 });
 
@@ -1580,3 +1654,41 @@ describe('ComposeService - streamLogs', () => {
     );
   });
 });
+
+
+describe('ComposeService - updateStack safe ordering', () => {
+  it('never broad-removes Compose-managed containers before up', async () => {
+    setupAutoCloseSpawn();
+    mockListContainers.mockResolvedValue([]);
+    mockGetContainersByStack.mockResolvedValue([{ Id: 'should-not-remove' }]);
+
+    const svc = ComposeService.getInstance(1);
+    const promise = svc.updateStack('my-stack');
+    await vi.advanceTimersByTimeAsync(3100);
+    await promise;
+
+    expect(mockGetContainersByStack).not.toHaveBeenCalled();
+    expect(mockRemoveContainers).not.toHaveBeenCalled();
+    expect(mockCaptureCandidate).toHaveBeenCalled();
+    expect(mockClassifyLegacyOrphansForUpdate).toHaveBeenCalledWith('my-stack');
+    expect(mockHandoff).toHaveBeenCalled();
+  });
+
+  it('removes only classified orphans after handoff', async () => {
+    setupAutoCloseSpawn();
+    mockListContainers.mockResolvedValue([]);
+    mockClassifyLegacyOrphansForUpdate.mockResolvedValueOnce({
+      status: 'orphans',
+      ids: ['orphan-1'],
+    });
+    mockRemoveContainers.mockResolvedValueOnce([{ id: 'orphan-1', success: true }]);
+
+    const svc = ComposeService.getInstance(1);
+    const promise = svc.updateStack('my-stack');
+    await vi.advanceTimersByTimeAsync(3100);
+    await promise;
+
+    expect(mockHandoff).toHaveBeenCalled();
+    expect(mockRemoveContainers).toHaveBeenCalledWith(['orphan-1']);
+  });
+});

backend/src/__tests__/database-metrics.test.ts

@@ -225,23 +225,25 @@ describe('DatabaseService - notification history cap (periodic)', () => {
 
     // A chatty stack writes 600 events.
     const base = Date.now();
-    for (let i = 0; i < 600; i++) {
+    db.transaction(() => {
-      db.addNotificationHistory(0, {
+      for (let i = 0; i < 600; i++) {
-        level: 'info',
+        db.addNotificationHistory(0, {
-        message: `chatty-${i}`,
+          level: 'info',
-        timestamp: base + i,
+          message: `chatty-${i}`,
-        stack_name: 'chatty',
+          timestamp: base + i,
-      });
+          stack_name: 'chatty',
-    }
+        });
-    // A quiet stack writes 3 events long before the chatty burst.
+      }
-    for (let i = 0; i < 3; i++) {
+      // A quiet stack writes 3 events long before the chatty burst.
-      db.addNotificationHistory(0, {
+      for (let i = 0; i < 3; i++) {
-        level: 'info',
+        db.addNotificationHistory(0, {
-        message: `quiet-${i}`,
+          level: 'info',
-        timestamp: base - 10_000 + i,
+          message: `quiet-${i}`,
-        stack_name: 'quiet',
+          timestamp: base - 10_000 + i,
-      });
+          stack_name: 'quiet',
-    }
+        });
+      }
+    });
 
     // No per-insert prune: every row is present.
     const beforeCleanup = db.getNotificationHistory(0, 2000);
@@ -261,13 +263,15 @@ describe('DatabaseService - notification history cap (periodic)', () => {
     db.deleteAllNotifications(0);
 
     const base = Date.now();
-    for (let i = 0; i < 1200; i++) {
+    db.transaction(() => {
-      db.addNotificationHistory(0, {
+      for (let i = 0; i < 1200; i++) {
-        level: 'info',
+        db.addNotificationHistory(0, {
-        message: `system-${i}`,
+          level: 'info',
-        timestamp: base + i,
+          message: `system-${i}`,
-      });
+          timestamp: base + i,
-    }
+        });
+      }
+    });
 
     db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
 
@@ -279,14 +283,16 @@ describe('DatabaseService - notification history cap (periodic)', () => {
   it('keeps the newest entries per (node, stack) after periodic cap', () => {
     db.deleteAllNotifications(0);
     const base = Date.now();
-    for (let i = 0; i < 600; i++) {
+    db.transaction(() => {
-      db.addNotificationHistory(0, {
+      for (let i = 0; i < 600; i++) {
-        level: 'info',
+        db.addNotificationHistory(0, {
-        message: `ordered-${i}`,
+          level: 'info',
-        timestamp: base + i * 10,
+          message: `ordered-${i}`,
-        stack_name: 'ordered',
+          timestamp: base + i * 10,
-      });
+          stack_name: 'ordered',
-    }
+        });
+      }
+    });
 
     db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
 
@@ -301,9 +307,11 @@ describe('DatabaseService - notification history cap (periodic)', () => {
   it('uses safe defaults when called with only the retention argument', () => {
     db.deleteAllNotifications(0);
     const base = Date.now();
-    for (let i = 0; i < 600; i++) {
+    db.transaction(() => {
-      db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
+      for (let i = 0; i < 600; i++) {
-    }
+        db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
+      }
+    });
     // Production caller (MonitorService) only passes daysToKeep; the cap defaults must enforce the per-stack 500 limit.
     const summary = db.cleanupOldNotifications(30);
     const after = db.getNotificationHistory(0, 2000).filter((n: any) => n.stack_name === 'default');

backend/src/__tests__/deployed-stack-deletion-service.test.ts

@@ -0,0 +1,156 @@
+/**
+ * Deployed-stack deletion: ready transaction retires both recovery models;
+ * blocking intents gate same-name create.
+ */
+import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
+import { randomUUID } from 'crypto';
+import path from 'path';
+import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
+import type {
+  StackUpdateRecoveryGenerationRow,
+  ServiceUpdateRecoveryRow,
+  StackUpdateCleanupPendingRow,
+} from '../services/DatabaseService';
+
+let tmpDir: string;
+let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
+let DeployedStackDeletionService: typeof import('../services/DeployedStackDeletionService').DeployedStackDeletionService;
+let overrideDeletionContainmentBase: typeof import('../services/DeployedStackDeletionService').overrideDeletionContainmentBase;
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ DatabaseService } = await import('../services/DatabaseService'));
+  ({ DeployedStackDeletionService, overrideDeletionContainmentBase } = await import('../services/DeployedStackDeletionService'));
+});
+
+afterAll(() => cleanupTestDb(tmpDir));
+
+function db() {
+  return DatabaseService.getInstance();
+}
+
+beforeEach(() => {
+  const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
+  raw.prepare('DELETE FROM stack_update_recovery_generations').run();
+  raw.prepare('DELETE FROM service_update_recovery').run();
+  raw.prepare('DELETE FROM stack_update_cleanup_pending').run();
+});
+
+const NODE = 1;
+
+describe('DeployedStackDeletionService ready transaction', () => {
+  it('commitStackDeletionReadyTransaction deletes full-stack and service recovery rows', () => {
+    const now = Date.now();
+    const stackName = 'del-stack';
+    const gen: StackUpdateRecoveryGenerationRow = {
+      id: randomUUID(),
+      node_id: NODE,
+      stack_name: stackName,
+      status: 'active',
+      phase: 'immediate_verified',
+      is_current: 1,
+      backup_slot_id: null,
+      override_path: null,
+      services_json: '[]',
+      health_gate_id: null,
+      gate_retain_until: null,
+      artifact_expires_at: null,
+      operation_lease_expires_at: null,
+      created_at: now,
+      updated_at: now,
+      created_by: null,
+      artifacts_retired: 0,
+    };
+    db().insertStackUpdateRecoveryGeneration(gen);
+    const svc: ServiceUpdateRecoveryRow = {
+      id: randomUUID(),
+      node_id: NODE,
+      stack_name: stackName,
+      service_name: 'web',
+      replicas_json: '[]',
+      majority_image_id: 'sha256:abc',
+      declared_image_ref: 'nginx:latest',
+      weak_floating_tag: 0,
+      health_gate_id: null,
+      status: 'active',
+      expires_at: now + 60_000,
+      claim_expires_at: null,
+      created_at: now,
+      created_by: null,
+    };
+    db().insertServiceUpdateRecovery(svc);
+    const intentId = randomUUID();
+    const intent: StackUpdateCleanupPendingRow = {
+      id: intentId,
+      node_id: NODE,
+      stack_name: stackName,
+      status: 'prepared',
+      target_kind: 'local_socket',
+      rollback_tags_json: '[]',
+      override_paths_json: '[]',
+      prune_volumes_requested: 0,
+      created_at: now,
+      updated_at: now,
+    };
+    db().insertCleanupPending(intent);
+
+    expect(db().commitStackDeletionReadyTransaction(intentId, NODE, stackName)).toBe(true);
+    expect(db().listStackUpdateRecoveryForStack(NODE, stackName)).toHaveLength(0);
+    expect(db().listActiveServiceUpdateRecoveries(NODE, stackName, 'web')).toHaveLength(0);
+    expect(db().getCleanupPending(intentId)?.status).toBe('ready');
+  });
+
+  it('hasBlockingDeletionIntent is true for prepared intents', () => {
+    const now = Date.now();
+    db().insertCleanupPending({
+      id: randomUUID(),
+      node_id: NODE,
+      stack_name: 'blocked',
+      status: 'prepared',
+      target_kind: 'local_socket',
+      rollback_tags_json: '[]',
+      override_paths_json: '[]',
+      prune_volumes_requested: 0,
+      created_at: now,
+      updated_at: now,
+    });
+    expect(db().hasBlockingDeletionIntent(NODE, 'blocked')).toBe(true);
+    expect(db().hasBlockingDeletionIntent(NODE, 'other')).toBe(false);
+  });
+
+  it('assertNoBlockingDeletionIntent throws for prepared stacks', () => {
+    const now = Date.now();
+    db().insertCleanupPending({
+      id: randomUUID(),
+      node_id: NODE,
+      stack_name: 'prep',
+      status: 'prepared',
+      target_kind: 'local_socket',
+      rollback_tags_json: '[]',
+      override_paths_json: '[]',
+      prune_volumes_requested: 0,
+      created_at: now,
+      updated_at: now,
+    });
+    expect(() => {
+      DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(NODE, 'prep');
+    }).toThrow(/deletion in progress/i);
+  });
+});
+
+describe('overrideDeletionContainmentBase', () => {
+  it('confines stack-scoped intents to the stack directory', () => {
+    expect(overrideDeletionContainmentBase('/app/compose', 'my-stack')).toBe(
+      path.resolve('/app/compose', 'my-stack'),
+    );
+    expect(overrideDeletionContainmentBase('/app/compose', null)).toBe(
+      path.resolve('/app/compose'),
+    );
+  });
+
+  it('rejects invalid stack names that could traverse', () => {
+    expect(overrideDeletionContainmentBase('/app/compose', '../other')).toBeNull();
+    expect(overrideDeletionContainmentBase('/app/compose', 'bad/name')).toBeNull();
+  });
+});
+

backend/src/__tests__/docker-controller.test.ts

@@ -1714,3 +1714,23 @@ describe('DockerController - getLegacyOrphanContainersByStack', () => {
     fallbackSpy.mockRestore();
   });
 });
+
+describe('DockerController - classifyLegacyOrphansForUpdate', () => {
+  it('returns none when compose ps already manages containers', async () => {
+    // Reuse the same mocks as getLegacyOrphanContainersByStack tests in this file.
+    const { default: DockerController } = await import('../services/DockerController');
+    const dc = DockerController.getInstance(1);
+    vi.spyOn(dc as any, 'fetchComposePsContainers').mockResolvedValue([{ ID: 'c1' }]);
+    await expect(dc.classifyLegacyOrphansForUpdate('my-stack')).resolves.toEqual({ status: 'none' });
+  });
+
+  it('returns classification_failed when compose ps and fallback both fail', async () => {
+    const { default: DockerController } = await import('../services/DockerController');
+    const dc = DockerController.getInstance(1);
+    vi.spyOn(dc as any, 'fetchComposePsContainers').mockRejectedValue(new Error('compose boom'));
+    vi.spyOn(dc as any, 'smartFallback').mockRejectedValue(new Error('fallback boom'));
+    const result = await dc.classifyLegacyOrphansForUpdate('my-stack');
+    expect(result.status).toBe('classification_failed');
+  });
+});
+

backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts

@@ -0,0 +1,112 @@
+/**
+ * Docker-backed integration: a failed pull must leave the running stack untouched.
+ * Skipped automatically when Docker is unavailable.
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { execFileSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb';
+
+function dockerAvailable(): boolean {
+  try {
+    execFileSync('docker', ['info'], { stdio: 'ignore' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+const hasDocker = dockerAvailable();
+const STACK = 'fpkeep';
+
+function compose(args: string[], cwd: string): string {
+  return execFileSync('docker', ['compose', '-p', STACK, ...args], {
+    cwd,
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+}
+
+describe.skipIf(!hasDocker)('failed pull keeps stack running', () => {
+  let tmpDir: string;
+  let composeDir: string;
+  let stackDir: string;
+  let nodeId: number;
+
+  beforeAll(async () => {
+    tmpDir = await setupTestDb();
+    composeDir = process.env.COMPOSE_DIR!;
+    stackDir = path.join(composeDir, STACK);
+    fs.mkdirSync(stackDir, { recursive: true });
+
+    const { DatabaseService } = await import('../../services/DatabaseService');
+    const db = DatabaseService.getInstance();
+    const local = db.getDefaultNode();
+    if (!local?.id) throw new Error('Test DB has no default local node');
+    nodeId = local.id;
+    db.updateGlobalSetting('prune_on_update', '0');
+
+    fs.writeFileSync(
+      path.join(stackDir, 'compose.yaml'),
+      [
+        'services:',
+        '  web:',
+        '    image: busybox:1.36.1',
+        '    command: ["sleep", "3600"]',
+        '',
+      ].join('\n'),
+      'utf8',
+    );
+
+    compose(['up', '-d', '--pull', 'always'], stackDir);
+  }, 180_000);
+
+  afterAll(async () => {
+    try {
+      if (stackDir && fs.existsSync(stackDir)) {
+        compose(['down', '--remove-orphans'], stackDir);
+      }
+    } catch {
+      // Best-effort cleanup.
+    }
+    if (tmpDir) cleanupTestDb(tmpDir);
+  }, 120_000);
+
+  it('leaves the original container running when ComposeService updateStack pull fails', async () => {
+    const beforeId = compose(['ps', '-q'], stackDir).trim();
+    expect(beforeId.length).toBeGreaterThan(0);
+
+    const beforeInspect = JSON.parse(
+      execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
+    ) as Array<{ State: { Running: boolean; Status: string } }>;
+    expect(beforeInspect[0].State.Running).toBe(true);
+
+    fs.writeFileSync(
+      path.join(stackDir, 'compose.yaml'),
+      [
+        'services:',
+        '  web:',
+        '    image: busybox:sencho-does-not-exist-fpkeep-xyz',
+        '    command: ["sleep", "3600"]',
+        '',
+      ].join('\n'),
+      'utf8',
+    );
+
+    const { StackUpdateRecoveryService } = await import('../../services/StackUpdateRecoveryService');
+    StackUpdateRecoveryService.resetForTests();
+    const { ComposeService } = await import('../../services/ComposeService');
+    await expect(
+      ComposeService.getInstance(nodeId).updateStack(STACK, undefined, true),
+    ).rejects.toThrow();
+
+    const afterId = compose(['ps', '-q'], stackDir).trim();
+    expect(afterId).toBe(beforeId);
+
+    const afterInspect = JSON.parse(
+      execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
+    ) as Array<{ State: { Running: boolean } }>;
+    expect(afterInspect[0].State.Running).toBe(true);
+  }, 180_000);
+});

backend/src/__tests__/image-updates-routes.test.ts

@@ -375,7 +375,7 @@ describe('POST /api/auto-update/execute', () => {
       .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
     const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
       .mockResolvedValue({ hasUpdate: true } as never);
-    const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
+    const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
     const gateSpy = vi.spyOn(PolicyEnforcement, 'enforcePolicyPreDeploy').mockResolvedValue({
       ok: false,
       bypassed: false,
@@ -421,7 +421,7 @@ describe('POST /api/auto-update/execute', () => {
       .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
     const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
       .mockResolvedValue({ hasUpdate: true } as never);
-    const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
+    const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
     const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au');
     try {
       const res = await request(app)

backend/src/__tests__/scheduler-service.test.ts

@@ -54,7 +54,7 @@ const {
   mockStartContainer: vi.fn().mockResolvedValue(undefined),
   mockStopContainer: vi.fn().mockResolvedValue(undefined),
   mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }),
-  mockUpdateStack: vi.fn().mockResolvedValue(undefined),
+  mockUpdateStack: vi.fn().mockResolvedValue({ recoveryId: null }),
   mockGetStacks: vi.fn().mockResolvedValue([]),
   mockGetStackContent: vi.fn().mockResolvedValue(''),
   mockGetEnvContent: vi.fn().mockResolvedValue(''),

backend/src/__tests__/stack-bulk-routes.test.ts

@@ -260,7 +260,7 @@ describe('POST /api/stacks/bulk execution', () => {
   });
 
   it('handles update action (paid tier) by calling ComposeService.updateStack', async () => {
-    mockUpdateStack.mockResolvedValue(undefined);
+    mockUpdateStack.mockResolvedValue({ recoveryId: null });
     const { LicenseService } = await import('../services/LicenseService');
     const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
     try {
@@ -287,7 +287,7 @@ describe('POST /api/stacks/bulk execution', () => {
       policy: { id: 1, name: 'block-criticals', node_id: null, node_identity: '', stack_pattern: null, max_severity: 'HIGH', block_on_deploy: 1, block_on_severity: 1, block_on_kev: 0, block_on_fixable: 0, enabled: 1, replicated_from_control: 0, created_at: Date.now(), updated_at: Date.now() },
       violations: [{ imageRef: 'nginx:latest', severity: 'CRITICAL', criticalCount: 3, highCount: 0, kevCount: 0, fixableCount: 0, reasons: ['severity'], scanId: 1 }],
     });
-    mockUpdateStack.mockResolvedValue(undefined);
+    mockUpdateStack.mockResolvedValue({ recoveryId: null });
     try {
       const res = await request(app)
         .post('/api/stacks/bulk')

backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts

@@ -124,7 +124,9 @@ describe('DELETE /api/stacks/:stackName mesh opt-out cascade (F-1 / F-14)', () =
 
         expect(res.status).toBe(200);
         const sawCascadeWarning = warnSpy.mock.calls.some(args =>
-            typeof args[0] === 'string' && args[0].includes('Mesh opt-out cascade failed')
+            typeof args[0] === 'string'
+            && (args[0].includes('Mesh opt-out cascade failed')
+              || args[0].includes('Mesh opt-out failed'))
         );
         expect(sawCascadeWarning).toBe(true);
     });

backend/src/__tests__/stack-self-protected-routes.test.ts

@@ -164,7 +164,7 @@ describe('self stack lifecycle refusal', () => {
   });
 
   it('allows update on a non-self stack', async () => {
-    mockUpdateStack.mockResolvedValue(undefined);
+    mockUpdateStack.mockResolvedValue({ recoveryId: null });
     const res = await request(app)
       .post('/api/stacks/web/update')
       .set('Cookie', authCookie);
@@ -177,7 +177,7 @@ describe('self stack lifecycle refusal', () => {
 describe('POST /api/stacks/bulk self stack skip', () => {
   beforeEach(() => {
     stubSelfProject('sencho');
-    mockUpdateStack.mockResolvedValue(undefined);
+    mockUpdateStack.mockResolvedValue({ recoveryId: null });
     mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]);
   });
 

backend/src/__tests__/stack-update-orchestrator.test.ts

@@ -35,7 +35,7 @@ function spec(name: string): EffectiveServiceSpec {
 
 beforeEach(() => {
   state.updateStack.mockReset();
-  state.updateStack.mockResolvedValue(undefined);
+  state.updateStack.mockResolvedValue({ recoveryId: null });
   state.model = null;
 });
 
@@ -85,8 +85,9 @@ describe('StackUpdateOrchestrator stack branch', () => {
       { nodeId: 0, stackName: 'web', target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' },
       { atomic: true, terminalWs: null },
     );
-    expect(result).toEqual({ kind: 'stack_compose_done' });
+    expect(result).toEqual({ kind: 'stack_compose_done', recoveryId: null });
     expect(state.updateStack).toHaveBeenCalledWith('web', undefined, true);
+    // recoveryId is forwarded from ComposeService.updateStack
   });
 });
 

backend/src/__tests__/stack-update-recovery-service.test.ts

@@ -0,0 +1,365 @@
+/**
+ * Direct tests for StackUpdateRecoveryService artifact lifecycle and compensation probe.
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const mockTag = vi.fn().mockResolvedValue(undefined);
+const mockRemove = vi.fn().mockResolvedValue(undefined);
+const mockListContainers = vi.fn().mockResolvedValue([]);
+const mockInspectContainer = vi.fn();
+const mockGetContainer = vi.fn(() => ({ inspect: mockInspectContainer }));
+const mockGetImage = vi.fn((ref: string) => ({
+  tag: mockTag,
+  remove: mockRemove,
+  inspect: vi.fn().mockResolvedValue({ RepoDigests: [] }),
+  _ref: ref,
+}));
+
+vi.mock('../services/DockerController', () => ({
+  default: {
+    getInstance: () => ({
+      getDocker: () => ({
+        listContainers: mockListContainers,
+        getContainer: mockGetContainer,
+        getImage: mockGetImage,
+      }),
+    }),
+  },
+}));
+
+vi.mock('../services/FileSystemService', () => ({
+  FileSystemService: {
+    getInstance: () => ({
+      getBaseDir: () => '/test/compose',
+      backupStackFiles: vi.fn().mockResolvedValue(undefined),
+      restoreStackFiles: vi.fn().mockResolvedValue(undefined),
+    }),
+  },
+}));
+
+vi.mock('../services/composeProjectContext', () => ({
+  classifyReferenceKind: () => 'moving_tag',
+  resolveComposeProjectContext: vi.fn().mockResolvedValue({
+    validateForMutation: vi.fn().mockResolvedValue(undefined),
+    backupFromContext: vi.fn().mockResolvedValue('backup-1'),
+    restoreFromContext: vi.fn().mockResolvedValue(undefined),
+  }),
+}));
+
+vi.mock('../services/effectiveServiceModel', () => ({
+  buildEffectiveServiceModel: vi.fn().mockResolvedValue({
+    renderable: true,
+    services: [{ name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1 }],
+  }),
+}));
+
+const mockValidateExact = vi.fn().mockResolvedValue(undefined);
+vi.mock('../services/ComposeService', async () => {
+  const actual = await vi.importActual<typeof import('../services/ComposeService')>('../services/ComposeService');
+  return {
+    ...actual,
+    ComposeService: {
+      getInstance: () => ({
+        validateExactComposeInvocation: mockValidateExact,
+        buildAuthoredComposeArgs: vi.fn(),
+      }),
+    },
+    getComposeCommandTimeoutMs: () => 60_000,
+  };
+});
+
+const mockUnlink = vi.fn().mockResolvedValue(undefined);
+const mockWriteFile = vi.fn().mockResolvedValue(undefined);
+const mockRealpath = vi.fn(async (p: string) => p);
+vi.mock('fs/promises', () => ({
+  default: {
+    unlink: (p: string) => mockUnlink(p),
+    writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc),
+    realpath: (p: string) => mockRealpath(p),
+  },
+  unlink: (p: string) => mockUnlink(p),
+  writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc),
+  realpath: (p: string) => mockRealpath(p),
+}));
+
+import { DatabaseService } from '../services/DatabaseService';
+import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
+
+describe('StackUpdateRecoveryService', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    StackUpdateRecoveryService.resetForTests();
+    mockListContainers.mockResolvedValue([
+      { Id: 'c1', State: 'running', Labels: {} },
+    ]);
+    mockInspectContainer.mockResolvedValue({
+      State: { Status: 'running', ExitCode: 0 },
+      Image: 'sha256:abc',
+    });
+    mockValidateExact.mockResolvedValue(undefined);
+    mockUnlink.mockResolvedValue(undefined);
+    mockWriteFile.mockResolvedValue(undefined);
+    mockRealpath.mockImplementation(async (p: string) => p);
+    mockRemove.mockResolvedValue(undefined);
+    mockTag.mockResolvedValue(undefined);
+  });
+
+  it('validates exact invocation before tagging images', async () => {
+    const order: string[] = [];
+    mockValidateExact.mockImplementation(async () => { order.push('validate'); });
+    mockTag.mockImplementation(async () => { order.push('tag'); });
+    mockWriteFile.mockImplementation(async () => { order.push('write'); });
+
+    const spyInsert = vi.spyOn(DatabaseService.prototype, 'insertStackUpdateRecoveryGeneration')
+      .mockImplementation(() => { order.push('insert'); });
+    vi.spyOn(DatabaseService.prototype, 'getGlobalSettings').mockReturnValue({});
+
+    await StackUpdateRecoveryService.getInstance().captureCandidate({
+      nodeId: 1,
+      stackName: 'my-stack',
+      createdBy: 'test',
+    });
+
+    expect(order.indexOf('validate')).toBeLessThan(order.indexOf('tag'));
+    expect(order.indexOf('tag')).toBeLessThan(order.indexOf('write'));
+    expect(order.indexOf('write')).toBeLessThan(order.indexOf('insert'));
+    spyInsert.mockRestore();
+  });
+
+  it('retires opaque tags and override on abandon', async () => {
+    const row = {
+      id: 'gen-1',
+      node_id: 1,
+      stack_name: 'my-stack',
+      status: 'candidate' as const,
+      phase: 'captured' as const,
+      is_current: 0,
+      backup_slot_id: null,
+      override_path: '/test/compose/my-stack/.sencho-recovery-aaaaaaaaaaaa.yml',
+      services_json: JSON.stringify([{
+        serviceName: 'web',
+        scale: 1,
+        hasBuild: false,
+        declaredImageRef: 'nginx:latest',
+        referenceKind: 'moving_tag',
+        replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold' }],
+      }]),
+      health_gate_id: null,
+      gate_retain_until: null,
+      artifact_expires_at: null,
+      operation_lease_expires_at: null,
+      created_at: Date.now(),
+      updated_at: Date.now(),
+      created_by: null,
+      artifacts_retired: 0,
+    };
+
+    vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
+    vi.spyOn(DatabaseService.prototype, 'abandonStackUpdateRecoveryGeneration').mockReturnValue(true);
+    const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired')
+      .mockReturnValue(true);
+
+    const ok = await StackUpdateRecoveryService.getInstance().abandon('gen-1');
+    expect(ok).toBe(true);
+    expect(mockRemove).toHaveBeenCalled();
+    expect(mockUnlink).toHaveBeenCalledWith(row.override_path);
+    expect(markRetired).toHaveBeenCalledWith('gen-1');
+  });
+
+  it('does not mark restored_current when recovery probe finds crashed containers', async () => {
+    const servicesJson = JSON.stringify([{
+      serviceName: 'web',
+      scale: 1,
+      hasBuild: false,
+      declaredImageRef: 'nginx:latest',
+      referenceKind: 'moving_tag',
+      replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/x/web:hold' }],
+    }]);
+    const row = {
+      id: 'gen-2',
+      node_id: 1,
+      stack_name: 'my-stack',
+      status: 'active' as const,
+      phase: 'reconciling' as const,
+      is_current: 1,
+      backup_slot_id: 'b1',
+      override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml',
+      services_json: servicesJson,
+      health_gate_id: null,
+      gate_retain_until: null,
+      artifact_expires_at: null,
+      operation_lease_expires_at: null,
+      created_at: Date.now(),
+      updated_at: Date.now(),
+      created_by: null,
+      artifacts_retired: 0,
+    };
+    vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
+    const update = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration')
+      .mockImplementation(() => undefined);
+
+    mockListContainers.mockResolvedValue([{ Id: 'c1', State: 'exited', Labels: { 'com.docker.compose.service': 'web' } }]);
+    mockInspectContainer.mockResolvedValue({ State: { ExitCode: 0 } });
+
+    vi.useFakeTimers();
+    const promise = StackUpdateRecoveryService.getInstance().compensateWithCandidate(
+      'gen-2',
+      async () => undefined,
+    );
+    await vi.advanceTimersByTimeAsync(3100);
+    const ok = await promise;
+    vi.useRealTimers();
+
+    expect(ok).toBe(false);
+    expect(update).toHaveBeenCalledWith('gen-2', expect.objectContaining({ status: 'recovery_required' }));
+    expect(update).not.toHaveBeenCalledWith(
+      'gen-2',
+      expect.objectContaining({ status: 'restored_current' }),
+    );
+  });
+
+  const capturedWebReplica = {
+    containerId: 'c1',
+    imageId: 'sha256:oldimg',
+    repoDigest: null,
+    state: 'running' as const,
+    rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold',
+  };
+
+  it('probeRecoveredStack rejects empty runtime when expected replicas were running', async () => {
+    const servicesJson = JSON.stringify([{
+      serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
+      referenceKind: 'moving_tag', replicas: [capturedWebReplica],
+    }]);
+    mockListContainers.mockResolvedValue([]);
+    vi.useFakeTimers();
+    const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
+    await vi.advanceTimersByTimeAsync(3100);
+    await expect(promise).resolves.toBe(false);
+    vi.useRealTimers();
+  });
+
+  it('probeRecoveredStack rejects restarting and unhealthy expected containers', async () => {
+    const servicesJson = JSON.stringify([{
+      serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
+      referenceKind: 'moving_tag', replicas: [capturedWebReplica],
+    }]);
+
+    mockListContainers.mockResolvedValue([
+      { Id: 'c1', State: 'restarting', Labels: { 'com.docker.compose.service': 'web' } },
+    ]);
+    vi.useFakeTimers();
+    let promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
+    await vi.advanceTimersByTimeAsync(3100);
+    await expect(promise).resolves.toBe(false);
+    vi.useRealTimers();
+
+    mockListContainers.mockResolvedValue([
+      { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
+    ]);
+    mockInspectContainer.mockResolvedValue({
+      Image: 'sha256:oldimg',
+      State: { Status: 'running', Health: { Status: 'unhealthy' } },
+    });
+    vi.useFakeTimers();
+    promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
+    await vi.advanceTimersByTimeAsync(3100);
+    await expect(promise).resolves.toBe(false);
+    vi.useRealTimers();
+  });
+
+  it('probeRecoveredStack rejects healthy containers running a mismatched image', async () => {
+    const servicesJson = JSON.stringify([{
+      serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
+      referenceKind: 'moving_tag', replicas: [capturedWebReplica],
+    }]);
+    mockListContainers.mockResolvedValue([
+      { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
+    ]);
+    mockInspectContainer.mockResolvedValue({
+      Image: 'sha256:newfailed',
+      Config: { Image: 'nginx:alpine' },
+      State: { Status: 'running' },
+    });
+    vi.useFakeTimers();
+    const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
+    await vi.advanceTimersByTimeAsync(3100);
+    await expect(promise).resolves.toBe(false);
+    vi.useRealTimers();
+  });
+
+  it('probeRecoveredStack rejects a running replica for a service captured at scale zero', async () => {
+    const servicesJson = JSON.stringify([{
+      serviceName: 'worker', scale: 0, hasBuild: false, declaredImageRef: 'busybox:latest',
+      referenceKind: 'moving_tag',
+      replicas: [{
+        containerId: 'c0', imageId: 'sha256:worker', repoDigest: null, state: 'stopped',
+        rollbackTag: 'sencho-rb/aaaaaaaaaaaa/worker:hold',
+      }],
+    }]);
+    mockListContainers.mockResolvedValue([
+      { Id: 'c0', State: 'running', Labels: { 'com.docker.compose.service': 'worker' } },
+    ]);
+    mockInspectContainer.mockResolvedValue({
+      Image: 'sha256:worker',
+      State: { Status: 'running' },
+    });
+    vi.useFakeTimers();
+    const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
+    await vi.advanceTimersByTimeAsync(3100);
+    await expect(promise).resolves.toBe(false);
+    vi.useRealTimers();
+  });
+
+  it('probeRecoveredStack accepts healthy running replicas matching capture scale and image', async () => {
+    const servicesJson = JSON.stringify([{
+      serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
+      referenceKind: 'moving_tag', replicas: [capturedWebReplica],
+    }]);
+    mockListContainers.mockResolvedValue([
+      { Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
+    ]);
+    mockInspectContainer.mockResolvedValue({
+      Image: 'sha256:oldimg',
+      Config: { Image: 'sencho-rb/aaaaaaaaaaaa/web:hold' },
+      State: { Status: 'running' },
+    });
+    vi.useFakeTimers();
+    const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
+    await vi.advanceTimersByTimeAsync(3100);
+    await expect(promise).resolves.toBe(true);
+    vi.useRealTimers();
+  });
+
+  it('does not mark artifacts retired when tag removal fails', async () => {
+    const row = {
+      id: 'gen-3',
+      node_id: 1,
+      stack_name: 'my-stack',
+      status: 'abandoned' as const,
+      phase: 'captured' as const,
+      is_current: 0,
+      backup_slot_id: null,
+      override_path: null,
+      services_json: JSON.stringify([{
+        serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
+        referenceKind: 'moving_tag',
+        replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/cccccccccccc/web:hold' }],
+      }]),
+      health_gate_id: null,
+      gate_retain_until: null,
+      artifact_expires_at: Date.now() - 1,
+      operation_lease_expires_at: null,
+      created_at: Date.now(),
+      updated_at: Date.now(),
+      created_by: null,
+      artifacts_retired: 0,
+    };
+    mockRemove.mockRejectedValueOnce(Object.assign(new Error('docker busy'), { statusCode: 500 }));
+    const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired')
+      .mockReturnValue(true);
+    const ok = await StackUpdateRecoveryService.getInstance().retireGenerationArtifacts(row);
+    expect(ok).toBe(false);
+    expect(markRetired).not.toHaveBeenCalled();
+  });
+});

backend/src/__tests__/stacks-failure-notifications.test.ts

@@ -270,7 +270,7 @@ describe('health gate begin call sites', () => {
   });
 
   it('begins a gate after a manual update and returns its id', async () => {
-    mockUpdateStack.mockResolvedValue(undefined);
+    mockUpdateStack.mockResolvedValue({ recoveryId: null });
     const res = await request(app)
       .post('/api/stacks/myapp/update')
       .set('Cookie', authCookie)
@@ -281,7 +281,7 @@ describe('health gate begin call sites', () => {
   });
 
   it('begins a gate per stack in a bulk update and carries ids in the results', async () => {
-    mockUpdateStack.mockResolvedValue(undefined);
+    mockUpdateStack.mockResolvedValue({ recoveryId: null });
     const res = await request(app)
       .post('/api/stacks/bulk')
       .set('Cookie', authCookie)
@@ -490,7 +490,7 @@ describe('deploy_failure notification on /update error', () => {
   });
 
   it('uses trusted proxy tier headers for remote atomic updates', async () => {
-    mockUpdateStack.mockResolvedValue(undefined);
+    mockUpdateStack.mockResolvedValue({ recoveryId: null });
     const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
 
     const res = await request(app)

backend/src/__tests__/update-guard-rollback.test.ts

@@ -96,11 +96,12 @@ describe('buildRollbackItems', () => {
     expect(known.detail).toContain('nginx:1.27.1');
   });
 
-  it('downgrades the previous image to not_covered for a moving tag', () => {
+  it('treats a moving tag as ready because full-stack updates retain the prior image ID', () => {
     const item = itemById(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), 'previous_images');
-    expect(item.state).toBe('not_covered');
+    expect(item.state).toBe('ready');
     expect(item.detail).toContain('moving image tag');
     expect(item.detail).toContain('nginx:latest');
+    expect(item.detail).toContain('recovery window');
   });
 
   it('does not mistake an image literally named error for a failed preview', () => {
@@ -139,9 +140,9 @@ describe('aggregateRollbackOverall', () => {
     expect(aggregateRollbackOverall(items)).toBe('partial');
   });
 
-  it('is partial when the rollback target is a moving tag', () => {
+  it('is ready when the rollback target is a moving tag (image ID capture covers it)', () => {
     const items = buildRollbackItems(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), NOW);
-    expect(aggregateRollbackOverall(items)).toBe('partial');
+    expect(aggregateRollbackOverall(items)).toBe('ready');
   });
 
   it('is partial when env coverage is missing', () => {

backend/src/__tests__/update-guard-service.test.ts

@@ -295,12 +295,11 @@ describe('UpdateGuardService.computeRollbackReadiness moving-tag wiring', () =>
     mockListContainers.mockResolvedValue([]);
   });
 
-  it('marks previous_images not_covered (overall partial) when any image uses a moving tag', async () => {
+  it('marks previous_images ready when any image uses a moving tag (full-stack image ID capture)', async () => {
-    // Primary pinned, sidecar on a moving tag: a file rollback cannot revert it.
     mockGetPreview.mockResolvedValue(preview([{ current_tag: '1.2.3' }, { current_tag: 'latest' }]));
     const report = await UpdateGuardService.getInstance().computeRollbackReadiness(0, 'app');
-    expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('not_covered');
+    expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('ready');
-    expect(report.overall).toBe('partial');
+    expect(report.overall).toBe('ready');
   });
 
   it('marks previous_images ready (overall ready) when every image is pinned', async () => {

backend/src/__tests__/webhooks-trigger.test.ts

@@ -481,7 +481,7 @@ describe('WebhookService.execute: health gate begin call sites', () => {
         const { HealthGateService } = await import('../services/HealthGateService');
         vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined);
         vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]);
-        vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue(undefined);
+        vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
         const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook');
 
         const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true);

backend/src/bootstrap/startup.ts

@@ -12,6 +12,8 @@ import { MonitorService } from '../services/MonitorService';
 import { AutoHealService } from '../services/AutoHealService';
 import { HealthGateService } from '../services/HealthGateService';
 import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
+import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
+import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
 import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
 import { DockerEventManager } from '../services/DockerEventManager';
 import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService';
@@ -134,6 +136,16 @@ export async function startServer(server: Server): Promise<void> {
   // Initialize the license service before any tier-gated code can run.
   LicenseService.getInstance().initialize();
 
+  // Deletion-intent reconciliation must finish before mutation-capable
+  // background services or HTTP accept traffic that could recreate a stack
+  // name still covered by a prepared/ready tombstone.
+  try {
+    await DeployedStackDeletionService.getInstance().reconcileAtStartup();
+  } catch (err) {
+    console.error('[Startup] Deployed stack deletion reconcile failed:', (err as Error).message);
+  }
+  StackUpdateRecoveryService.getInstance().start();
+
   // Synchronous starts: schedule background timers and continue. None of
   // these fire their first tick for at least a few seconds, so they
   // safely run alongside the async initializers below.

backend/src/routes/imageUpdates.ts

@@ -426,7 +426,13 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
           continue;
         }
         db.clearStackUpdateStatus(req.nodeId, stackName);
-        HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
+        const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
+        const orchResult = lock.result;
+        const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
+        if (recoveryId) {
+          const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService');
+          StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
+        }
 
         NotificationService.getInstance().broadcastEvent({
           type: 'state-invalidate',

backend/src/routes/nodes.ts

@@ -7,6 +7,7 @@ import { requirePermission } from '../middleware/permissions';
 import { rejectApiTokenScope } from '../middleware/apiTokenScope';
 import { requireAdmin, requirePaid } from '../middleware/tierGates';
 import { enrollmentLimiter } from '../middleware/rateLimiters';
+import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
 import { DatabaseService } from '../services/DatabaseService';
 import { NodeRegistry } from '../services/NodeRegistry';
 import { CacheService } from '../services/CacheService';
@@ -401,7 +402,14 @@ nodesRouter.delete('/:id', async (req: Request, res: Response) => {
     // local node, or one with no active connection). Mirrors the re-enroll path.
     MeshProxyTunnelDialer.getInstance().closeBridge(id, 'node deleted');
     PilotTunnelManager.getInstance().closeTunnel(id, PilotCloseCode.NormalClosure, 'node deleted');
-    DatabaseService.getInstance().deleteNode(id);
+    // Local-socket nodes: ready tombstone + recovery-row retirement in the same
+    // transaction as the node delete, then sweep tags/paths. Remote hub records
+    // create no Docker cleanup tombstone.
+    if (existing.type === 'local') {
+      await DeployedStackDeletionService.getInstance().deleteLocalNode(id);
+    } else {
+      DatabaseService.getInstance().deleteNode(id);
+    }
     NodeRegistry.getInstance().evictConnection(id);
     NodeRegistry.getInstance().notifyNodeRemoved(id);
     CacheService.getInstance().invalidate(`${REMOTE_META_NAMESPACE}:${id}`);

backend/src/routes/stacks.ts

@@ -16,7 +16,6 @@ import { ComposeService, getComposeRollbackInfo } from '../services/ComposeServi
 import { StackUpdateOrchestrator, shortImageId, type OrchestratorResult } from '../services/StackUpdateOrchestrator';
 import DockerController, { type BulkStackInfo } from '../services/DockerController';
 import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
-import { MeshService } from '../services/MeshService';
 import { CacheService, type CacheFetchOutcome } from '../services/CacheService';
 import { UpdatePreviewService } from '../services/UpdatePreviewService';
 import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
@@ -38,6 +37,8 @@ import { labelInventoryOptionsFromRequest, requireRevealAdmin } from '../helpers
 import { EXPOSURE_INTENTS, type ExposureIntent } from '../services/network/types';
 import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService';
 import { HealthGateService } from '../services/HealthGateService';
+import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
+import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
 import { classifyFailure } from '../services/updateGuard/failureClassifier';
 import { requirePermission, checkPermission } from '../middleware/permissions';
 import { NotificationService, type NotificationCategory } from '../services/NotificationService';
@@ -99,8 +100,14 @@ const STACK_OP_PRESENT_PARTICIPLE: Record<StackOpAction, string> = {
   update: 'updating',
   rollback: 'rolling back',
   backup: 'backing up',
+  delete: 'deleting',
 };
 
+function linkStackUpdateRecoveryGate(recoveryId: string | null | undefined, healthGateId: string | null): void {
+  if (!recoveryId) return;
+  StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
+}
+
 function tryAcquireStackOpLock(
   req: Request,
   res: Response,
@@ -474,7 +481,7 @@ async function runStackBulkOp(
         };
       }
       const atomic = true;
-      await StackUpdateOrchestrator.getInstance().execute(
+      const orchResult = await StackUpdateOrchestrator.getInstance().execute(
         { nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'bulk', actor: user },
         { atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
       );
@@ -492,6 +499,8 @@ async function runStackBulkOp(
         console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err),
       );
       const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
+      const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
+      linkStackUpdateRecoveryGate(recoveryId, healthGateId);
       return { stackName, ok: true, healthGateId };
     } else {
       const outcome = await containerActionForStack(req.nodeId, stackName, action);
@@ -929,6 +938,12 @@ stacksRouter.post('/', async (req: Request, res: Response) => {
     if (!isValidStackName(stackName)) {
       return res.status(400).json({ error: 'Stack name can only contain alphanumeric characters, hyphens, and underscores' });
     }
+    try {
+      DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(req.nodeId, stackName);
+    } catch (guardError) {
+      res.status(409).json({ error: getErrorMessage(guardError, 'Stack deletion in progress'), code: 'stack_deletion_in_progress' });
+      return;
+    }
     await FileSystemService.getInstance(req.nodeId).createStack(stackName);
     invalidateNodeCaches(req.nodeId);
     dlog(`[Stacks] Stack created: ${sanitizeForLog(stackName)}`);
@@ -1109,82 +1124,32 @@ stacksRouter.delete('/:stackName', async (req: Request, res: Response) => {
   const sanitizedName = sanitizeForLog(stackName);
   if (debug) console.debug(`[Stacks:debug] Delete starting`, { stackName: sanitizedName, pruneVolumes, nodeId: req.nodeId });
 
-  // Step 1: compose down. Best-effort: a stack with corrupt compose files or
+  const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
-  // a temporarily-unreachable daemon should still be removable. Logged so the
+    nodeId: req.nodeId,
-  // operator can investigate orphaned containers separately.
+    stackName,
-  try {
+    pruneVolumes,
-    await ComposeService.getInstance(req.nodeId).downStack(stackName);
+    actor: req.user?.username ?? 'system',
-    if (debug) console.debug(`[Stacks:debug] Delete: down OK`, { stackName: sanitizedName });
+  });
-  } catch (downErr) {
-    console.warn('[Stacks] Compose down failed or no-op for %s:', sanitizeForLog(stackName), downErr);
-  }
 
-  // Step 2: volume prune (only if requested). Best-effort.
+  if (!result.ok) {
-  if (pruneVolumes) {
+    if (result.code === 'lock_conflict') {
-    try {
+      res.status(409).json({
-      const result = await DockerController.getInstance().pruneManagedOnly('volumes', [stackName]);
+        error: result.error,
-      dlog(`[Stacks] Pruned volumes for ${sanitizeForLog(stackName)}: ${result.reclaimedBytes} bytes reclaimed`);
+        code: 'stack_op_in_progress',
-    } catch (pruneErr) {
+        action: result.existingAction ?? 'delete',
-      console.warn('[Stacks] Volume prune failed for %s, continuing delete:', sanitizeForLog(stackName), pruneErr);
+      });
+      return;
     }
-  }
+    if (result.code === 'fs_failed') {
-
+      res.status(500).json({
-  // Step 3: filesystem delete. If this fails the on-disk compose files are
+        error: `${result.error}. Stack containers may have been stopped but on-disk files remain. Retry the delete or clean the files manually.`,
-  // still present, so we abort BEFORE touching the database — keeping DB and
+      });
-  // FS in sync so the operator can retry. Otherwise a half-deleted stack
+      return;
-  // (rows gone, files remain) becomes invisible to the UI.
+    }
-  try {
+    res.status(500).json({ error: result.error });
-    await FileSystemService.getInstance(req.nodeId).deleteStack(stackName);
-    if (debug) console.debug(`[Stacks:debug] Delete: fs OK`, { stackName: sanitizedName });
-  } catch (fsErr) {
-    console.error('[Stacks] File deletion failed for %s; database state untouched:', sanitizeForLog(stackName), fsErr);
-    const message = getErrorMessage(fsErr, 'Failed to remove stack files');
-    res.status(500).json({
-      error: `${message}. Stack containers may have been stopped but on-disk files remain. Retry the delete or clean the files manually.`,
-    });
     return;
   }
 
-  // Step 4: database cleanup. Per-call idempotent; safe to run sequentially.
-  try {
-    DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
-    DatabaseService.getInstance().clearStackScanAttempts(req.nodeId, stackName);
-    DatabaseService.getInstance().deleteServiceUpdateRecoveries(req.nodeId, stackName);
-    DatabaseService.getInstance().deleteRoleAssignmentsByResource('stack', stackName);
-    DatabaseService.getInstance().deleteGitSource(stackName);
-    DatabaseService.getInstance().deleteStackDossier(req.nodeId, stackName);
-    DatabaseService.getInstance().deleteStackDriftFindings(req.nodeId, stackName);
-    DatabaseService.getInstance().deleteStackExposureIntents(req.nodeId, stackName);
-    DatabaseService.getInstance().deleteStackExposure(req.nodeId, stackName);
-    DatabaseService.getInstance().deleteStackProjectEnvFiles(req.nodeId, stackName);
-    DatabaseService.getInstance().deleteStackScans(req.nodeId, stackName);
-    if (debug) console.debug(`[Stacks:debug] Delete: db OK`, { stackName: sanitizedName });
-  } catch (dbErr) {
-    console.error('[Stacks] Database cleanup failed for %s; files already removed:', sanitizeForLog(stackName), dbErr);
-    const message = getErrorMessage(dbErr, 'Failed to clear stack database state');
-    res.status(500).json({
-      error: `${message}. Stack files have been removed; some database rows for this stack may remain. Recreating the stack with the same name will reuse those rows.`,
-    });
-    return;
-  }
-
-  // Step 5: mesh opt-out cascade. Idempotent; best-effort cleanup of derived
-  // aliases / override files. A mesh failure here must not flip the delete
-  // result, since the stack itself is already gone.
-  try {
-    await MeshService.getInstance().optOutStack(
-      req.nodeId,
-      stackName,
-      req.user?.username ?? 'system',
-    );
-  } catch (meshErr) {
-    console.warn(
-      '[Stacks] Mesh opt-out cascade failed for %s, continuing delete:',
-      sanitizeForLog(stackName),
-      meshErr,
-    );
-  }
-
   invalidateNodeCaches(req.nodeId);
   dlog(`[Stacks] Stack deleted: ${sanitizeForLog(stackName)}`);
   res.json({ success: true });
@@ -2267,7 +2232,7 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
     const debug = isDebugEnabled();
     const atomic = true;
     if (debug) console.debug('[Stacks:debug] Update starting', { stackName, atomic, nodeId: req.nodeId });
-    await StackUpdateOrchestrator.getInstance().execute(
+    const orchResult = await StackUpdateOrchestrator.getInstance().execute(
       { nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null },
       { atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
     );
@@ -2285,6 +2250,8 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
     if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`);
     ok = true;
     const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
+    const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
+    linkStackUpdateRecoveryGate(recoveryId, healthGateId);
     res.json({ status: 'Update completed', healthGateId });
     notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system');
     if (!skipScan) {

backend/src/services/BlueprintService.ts

@@ -10,6 +10,7 @@ import {
 } from './DatabaseService';
 import { ComposeService } from './ComposeService';
 import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService';
+import { DeployedStackDeletionService } from './DeployedStackDeletionService';
 import { FileSystemService } from './FileSystemService';
 import { NodeRegistry } from './NodeRegistry';
 import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers';
@@ -463,32 +464,17 @@ export class BlueprintService {
     }
 
     private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<void> {
-        // Hold the per-stack lock across both the compose down and the directory
+        const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
-        // delete so a withdraw cannot race a manual operation, nor tear the
+            nodeId: node.id,
-        // files out from under one that starts mid-withdraw.
+            stackName: blueprint.name,
-        const lock = await StackOpLockService.getInstance().runExclusive(
+            pruneVolumes: false,
-            node.id, blueprint.name, 'down', 'system',
+            actor: 'system:blueprint',
-            async () => {
+        });
-                try {
+        if (!result.ok) {
-                    await ComposeService.getInstance(node.id).downStack(blueprint.name);
+            if (result.code === 'lock_conflict') {
-                } catch (err) {
+                throw new Error(result.error);
-                    // best-effort: continue to delete the directory even if down fails
+            }
-                    console.warn(`[BlueprintService] downStack failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`);
+            throw new Error(result.error);
-                }
-                if (await this.stackDirExists(node.id, blueprint.name)) {
-                    await FileSystemService.getInstance(node.id).deleteStack(blueprint.name);
-                }
-                // Remove the exposure descriptor so a withdrawn blueprint
-                // stack does not leave a stale row that escalates posture.
-                try {
-                    DatabaseService.getInstance().deleteStackExposure(node.id, blueprint.name);
-                } catch (e) {
-                    console.warn(`[BlueprintService] deleteStackExposure failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(e)}`);
-                }
-            },
-        );
-        if (!lock.ran) {
-            throw new Error(stackOpSkipMessage(blueprint.name, lock.existing.action));
         }
     }
 

backend/src/services/ComposeService.ts

@@ -143,6 +143,33 @@ export class ComposeService {
    * same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh
    * override, rendering the user's authored model without mesh injection.
    */
+  /** Public wrapper for dual-arg assembly and recovery Compose invocations. */
+  public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise<string[]> {
+    return this.authoredComposeArgs(stackName, action);
+  }
+
+  public async validateStackForMutation(stackName: string): Promise<void> {
+    await this.assertRequiredEnvPresent(stackName);
+    await this.assertSafePilotBindMapping(stackName);
+  }
+
+  /**
+   * Render/validate the exact Compose invocation used by mutating operations
+   * (authored files, env pins, and generated Mesh override) before capture.
+   */
+  public async validateExactComposeInvocation(stackName: string): Promise<void> {
+    if (!isValidStackName(stackName)) {
+      throw new Error('Invalid stack path');
+    }
+    const baseResolved = path.resolve(this.baseDir);
+    const stackDir = path.resolve(baseResolved, stackName);
+    if (!stackDir.startsWith(baseResolved + path.sep)) {
+      throw new Error('Invalid stack path');
+    }
+    const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']);
+    await this.execute('docker', args, stackDir, undefined, true);
+  }
+
   private async authoredComposeArgs(stackName: string, action: string[]): Promise<string[]> {
     const args: string[] = ['compose'];
     const filePrefix = authoredComposeFileArgs(stackName, this.nodeId);
@@ -151,12 +178,23 @@ export class ComposeService {
     // directory, so deploy/update resolve the same effective config the validator did.
     args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId));
 
+    const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName);
     let overridePath: string | null = null;
     try {
       overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName);
     } catch (err) {
+      if (meshEnabled) {
+        throw err instanceof Error
+          ? err
+          : new Error(`Mesh override generation failed: ${String(err)}`);
+      }
       console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message));
     }
+    if (meshEnabled && !overridePath) {
+      throw new Error(
+        `Mesh override is required for stack "${stackName}" but could not be generated`,
+      );
+    }
     if (overridePath) {
       if (filePrefix.length === 0) {
         // Single-file stack: passing any -f disables compose's auto-discovery, so name
@@ -799,7 +837,50 @@ export class ComposeService {
     startStream();
   }
 
-  async updateStack(stackName: string, ws?: WebSocket, atomic?: boolean): Promise<void> {
+
+  /**
+   * Authored (+ mesh) compose args with an optional recovery override layered LAST.
+   * Used for pinned lifecycle / compensation ups (`--pull never --no-build`).
+   */
+  public async buildComposeArgsWithRecoveryOverride(
+    stackName: string,
+    action: string[],
+    recoveryOverridePath: string | null,
+  ): Promise<string[]> {
+    const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']);
+    const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__');
+    const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel;
+    const out = [...prefix];
+    if (recoveryOverridePath) {
+      if (!out.includes('-f')) {
+        const fsSvc = FileSystemService.getInstance(this.nodeId);
+        const baseFilename = await fsSvc.getComposeFilename(stackName);
+        out.push('-f', baseFilename);
+        try {
+          const userOverride = await fsSvc.getOverrideFilename(stackName);
+          if (userOverride) out.push('-f', userOverride);
+        } catch (err) {
+          const code = (err as { code?: string }).code;
+          if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') {
+            throw err;
+          }
+          console.warn(
+            '[ComposeService] could not resolve user compose override for recovery args:',
+            sanitizeForLog((err as Error).message),
+          );
+        }
+      }
+      out.push('-f', recoveryOverridePath);
+    }
+    out.push(...action);
+    return out;
+  }
+
+  async updateStack(
+    stackName: string,
+    ws?: WebSocket,
+    atomic?: boolean,
+  ): Promise<{ recoveryId: string | null }> {
     await this.assertRequiredEnvPresent(stackName);
     await this.assertSafePilotBindMapping(stackName);
     const stackDir = path.join(this.baseDir, stackName);
@@ -810,48 +891,106 @@ export class ComposeService {
       if (ws && ws.readyState === WebSocket.OPEN) ws.send(data);
     };
 
-    if (atomic) {
+    // Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs).
-      await this.createAtomicBackup(stackName, 'update', sendOutput);
+    const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
-    }
+    const recoverySvc = StackUpdateRecoveryService.getInstance();
+    let recoveryId: string | null = null;
+    let handedOff = false;
 
     try {
-      try {
+      sendOutput('=== Validating stack for update ===\n');
-        const dockerController = DockerController.getInstance(this.nodeId);
+      sendOutput('=== Capturing rollback generation ===\n');
-        const legacyContainers = await dockerController.getContainersByStack(stackName);
+      const candidate = await recoverySvc.captureCandidate({
-        if (legacyContainers && legacyContainers.length > 0) {
+        nodeId: this.nodeId,
-          sendOutput(`=== Cleaning up existing containers for clean update ===\n`);
+        stackName,
-          await dockerController.removeContainers(legacyContainers.map((c: any) => c.Id));
+        createdBy: null,
-        }
+      });
-      } catch (e) {
+      recoveryId = candidate.id;
-        console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e);
-      }
 
       const buildServices = await loadStackBuildServices(this.nodeId, stackName);
       const buildAware = buildServices.length > 0;
 
-      await this.withRegistryAuth(async (env) => {
+      try {
-        if (buildAware) {
+        await this.withRegistryAuth(async (env) => {
-          sendOutput('=== Building images ===\n');
+          if (buildAware) {
-          await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull']), stackDir, ws, true, env, getComposeStallTimeoutMs());
+            sendOutput('=== Building images ===\n');
+            await this.execute(
+              'docker',
+              await this.authoredComposeArgs(stackName, ['build', '--pull']),
+              stackDir, ws, true, env, getComposeStallTimeoutMs(),
+            );
+            sendOutput('=== Pulling registry images ===\n');
+            await this.execute(
+              'docker',
+              await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']),
+              stackDir, ws, true, env, getComposeStallTimeoutMs(),
+            );
+          } else {
+            sendOutput('=== Pulling latest images ===\n');
+            await this.execute(
+              'docker',
+              await this.authoredComposeArgs(stackName, ['pull']),
+              stackDir, ws, true, env, getComposeStallTimeoutMs(),
+            );
+          }
+        }, sendOutput);
+      } catch (acquireError) {
+        // Acquisition failure: abandon candidate; leave runtime untouched.
+        await recoverySvc.abandon(candidate.id);
+        recoveryId = null;
+        throw acquireError;
+      }
 
-          sendOutput('=== Pulling registry images ===\n');
+      if (!recoverySvc.markAcquired(candidate.id)) {
-          await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), stackDir, ws, true, env, getComposeStallTimeoutMs());
+        await recoverySvc.abandon(candidate.id);
-        } else {
+        throw new Error('Failed to mark recovery generation as acquired');
-          sendOutput('=== Pulling latest images ===\n');
+      }
-          await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull']), stackDir, ws, true, env, getComposeStallTimeoutMs());
-        }
-
-        sendOutput('=== Recreating containers ===\n');
-        await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs());
-      }, sendOutput);
-
-      // Post-Update Health Probe
-      await new Promise(resolve => setTimeout(resolve, 3000));
 
       const dockerController = DockerController.getInstance(this.nodeId);
+      sendOutput('=== Classifying legacy orphans ===\n');
+      const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName);
+      if (classified.status === 'classification_failed') {
+        await recoverySvc.abandon(candidate.id);
+        recoveryId = null;
+        throw new Error(`Legacy orphan classification failed: ${classified.error}`);
+      }
+
+      if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) {
+        await recoverySvc.abandon(candidate.id);
+        recoveryId = null;
+        throw new Error('Failed to hand off recovery generation');
+      }
+      handedOff = true;
+      if (!recoverySvc.markReconciling(candidate.id)) {
+        throw new Error('Failed to mark recovery generation as reconciling after handoff');
+      }
+
+      if (classified.status === 'orphans') {
+        sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`);
+        const results = await dockerController.removeContainers(classified.ids);
+        const failed = results.filter((r) => !r.success);
+        if (failed.length > 0) {
+          throw new Error(
+            `Failed to remove ${failed.length} legacy orphan container(s) after handoff`,
+          );
+        }
+      }
+
+      await this.withRegistryAuth(async (env) => {
+        sendOutput('=== Recreating containers ===\n');
+        await this.execute(
+          'docker',
+          await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']),
+          stackDir, ws, true, env, getComposeStallTimeoutMs(),
+        );
+      }, sendOutput);
+
+      // Immediate verification probe
+      await new Promise((resolve) => setTimeout(resolve, 3000));
+
       const containers = await dockerController.getDocker().listContainers({
         all: true,
-        filters: { label: [`com.docker.compose.project=${stackName}`] }
+        filters: { label: [`com.docker.compose.project=${stackName}`] },
       });
 
       for (const containerInfo of containers) {
@@ -859,56 +998,90 @@ export class ComposeService {
           const container = dockerController.getDocker().getContainer(containerInfo.Id);
           const inspectData = await container.inspect();
           const exitCode = inspectData.State.ExitCode;
-
           if (exitCode !== 0) {
             throw this.createContainerCrashError(exitCode);
           }
         }
       }
 
+      if (!recoverySvc.markImmediateVerified(candidate.id)) {
+        console.warn(
+          '[ComposeService] Could not CAS immediate_verified for recovery %s',
+          sanitizeForLog(candidate.id),
+        );
+      }
+
       sendOutput('=== Stack updated successfully ===\n');
-      // Opt-out (default ON): after a clean update, prune the node's dangling
+
-      // (untagged) image layers, including the one this pull just orphaned. Read
+      // Defer prune until gate retention / gate link: only prune when no active holds
-      // fresh each run so a remote node honors its own setting. Wrapped so a
+      // would be violated. Still honor prune_on_update, but use unified holds so
-      // prune failure can never reach the atomic-rollback catch below.
+      // candidate/current rollback images are retained.
       try {
         const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
         if (pruneOnUpdate) {
-          // Dynamic import: ServiceUpdateRecoveryService imports getComposeCommandTimeoutMs
+          const isImageHeld = recoverySvc.buildUnifiedHeldImagePredicate(this.nodeId);
-          // from this module, so a static import here would be circular.
-          const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService');
-          const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(this.nodeId);
           const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld);
-          // The Docker prune API does not report SpaceReclaimed on the containerd
-          // image store, so only show the figure when the daemon actually returns one.
           const reclaimed = result.reclaimedBytes > 0
             ? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB`
             : '';
           sendOutput(`=== Pruned dangling images${reclaimed} ===\n`);
         }
       } catch (pruneError) {
-        console.warn('Failed to prune dangling images after update for %s:', sanitizeForLog(stackName), pruneError);
+        console.warn(
+          'Failed to prune dangling images after update for %s:',
+          sanitizeForLog(stackName),
+          pruneError,
+        );
+      }
+
+      if (debug) {
+        console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName });
       }
-      if (debug) console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName });
     } catch (updateError) {
-      if (atomic) {
+      if (!handedOff && recoveryId) {
-        sendOutput('\n=== Update failed - restoring previous compose and env files ===\n');
+        await recoverySvc.abandon(recoveryId);
-        const rolledBack = await this.restoreAtomicBackup(stackName, stackDir, ws, sendOutput);
+        recoveryId = null;
+      }
+      if (handedOff && recoveryId) {
+        sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n');
+        const rolledBack = await recoverySvc.compensateWithCandidate(
+          recoveryId,
+          async (overridePath) => {
+            await this.withRegistryAuth(async (env) => {
+              await this.execute(
+                'docker',
+                await this.buildComposeArgsWithRecoveryOverride(
+                  stackName,
+                  ['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'],
+                  overridePath,
+                ),
+                stackDir,
+                ws,
+                true,
+                env,
+                getComposeStallTimeoutMs(),
+              );
+            }, sendOutput);
+          },
+        );
         throw new ComposeRollbackError(updateError, true, rolledBack);
       }
+      // Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched.
       throw updateError;
     }
-    // Reached only on a successful update; re-baseline so temporal drift compares
+
-    // against what is now deployed (see deployStack for why this lives here), then
-    // reconcile the ledger against the updated runtime.
     await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName);
     await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName);
     try {
       await this.refreshExposureCache(stackName);
     } catch (err) {
-      console.warn('[ComposeService] Exposure refresh failed after update for %s:',
+      console.warn(
-        sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown')));
+        '[ComposeService] Exposure refresh failed after update for %s:',
+        sanitizeForLog(stackName),
+        sanitizeForLog(getErrorMessage(err, 'unknown')),
+      );
     }
+    return { recoveryId };
   }
 
   /**

backend/src/services/DatabaseService.ts

@@ -213,6 +213,42 @@ export interface HealthGateRunRow {
 }
 
 /** Pre-update image snapshot enabling a manual per-service restore after a service-scoped update. */
+
+/** Full-stack update recovery generation. Separate from service_update_recovery. */
+export interface StackUpdateRecoveryGenerationRow {
+    id: string;
+    node_id: number;
+    stack_name: string;
+    status: 'candidate' | 'active' | 'restored_current' | 'superseded' | 'abandoned' | 'recovery_required';
+    phase: 'captured' | 'acquired' | 'handoff_committed' | 'reconciling' | 'immediate_verified';
+    is_current: number;
+    backup_slot_id: string | null;
+    override_path: string | null;
+    services_json: string;
+    health_gate_id: string | null;
+    gate_retain_until: number | null;
+    artifact_expires_at: number | null;
+    operation_lease_expires_at: number | null;
+    created_at: number;
+    updated_at: number;
+    created_by: string | null;
+    artifacts_retired: number;
+}
+
+/** Durable cleanup tombstone for stack/node deletion artifact sweep. */
+export interface StackUpdateCleanupPendingRow {
+    id: string;
+    node_id: number | null;
+    stack_name: string | null;
+    status: 'prepared' | 'ready' | 'cancelled';
+    target_kind: 'local_socket';
+    rollback_tags_json: string;
+    override_paths_json: string;
+    prune_volumes_requested: number;
+    created_at: number;
+    updated_at: number;
+}
+
 export interface ServiceUpdateRecoveryRow {
     id: string;
     node_id: number;
@@ -1647,6 +1683,50 @@ export class DatabaseService {
       CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_claim_expires
         ON service_update_recovery(status, claim_expires_at);
 
+
+      CREATE TABLE IF NOT EXISTS stack_update_recovery_generations (
+        id TEXT PRIMARY KEY,
+        node_id INTEGER NOT NULL,
+        stack_name TEXT NOT NULL,
+        status TEXT NOT NULL CHECK (status IN ('candidate','active','restored_current','superseded','abandoned','recovery_required')),
+        phase TEXT NOT NULL CHECK (phase IN ('captured','acquired','handoff_committed','reconciling','immediate_verified')),
+        is_current INTEGER NOT NULL DEFAULT 0,
+        backup_slot_id TEXT,
+        override_path TEXT,
+        services_json TEXT NOT NULL,
+        health_gate_id TEXT,
+        gate_retain_until INTEGER,
+        artifact_expires_at INTEGER,
+        operation_lease_expires_at INTEGER,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        created_by TEXT,
+        artifacts_retired INTEGER NOT NULL DEFAULT 0
+      );
+      CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_node_stack_current
+        ON stack_update_recovery_generations(node_id, stack_name, is_current);
+      CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_status_expires
+        ON stack_update_recovery_generations(status, artifact_expires_at);
+      CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_phase_lease
+        ON stack_update_recovery_generations(phase, operation_lease_expires_at);
+
+      CREATE TABLE IF NOT EXISTS stack_update_cleanup_pending (
+        id TEXT PRIMARY KEY,
+        node_id INTEGER,
+        stack_name TEXT,
+        status TEXT NOT NULL CHECK (status IN ('prepared','ready','cancelled')),
+        target_kind TEXT NOT NULL CHECK (target_kind IN ('local_socket')),
+        rollback_tags_json TEXT NOT NULL,
+        override_paths_json TEXT NOT NULL,
+        prune_volumes_requested INTEGER NOT NULL DEFAULT 0,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_status
+        ON stack_update_cleanup_pending(status);
+      CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_node_stack
+        ON stack_update_cleanup_pending(node_id, stack_name);
+
       CREATE TABLE IF NOT EXISTS secrets (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
         name TEXT NOT NULL UNIQUE,
@@ -1699,6 +1779,8 @@ export class DatabaseService {
             try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); } catch (e) { /* ignore */ }
         };
 
+        maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0');
+
         // Distributed API model columns
         maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''");
         maybeAddCol('nodes', 'api_token', "TEXT DEFAULT ''");
@@ -3510,6 +3592,268 @@ export class DatabaseService {
         this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
     }
 
+    public insertStackUpdateRecoveryGeneration(row: StackUpdateRecoveryGenerationRow): void {
+        this.db.prepare(
+            `INSERT INTO stack_update_recovery_generations (
+                id, node_id, stack_name, status, phase, is_current, backup_slot_id, override_path,
+                services_json, health_gate_id, gate_retain_until, artifact_expires_at,
+                operation_lease_expires_at, created_at, updated_at, created_by, artifacts_retired
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+        ).run(
+            row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current,
+            row.backup_slot_id, row.override_path, row.services_json, row.health_gate_id,
+            row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at,
+            row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0,
+        );
+    }
+
+    public getStackUpdateRecoveryGeneration(id: string): StackUpdateRecoveryGenerationRow | undefined {
+        return this.db.prepare('SELECT * FROM stack_update_recovery_generations WHERE id = ?').get(id) as StackUpdateRecoveryGenerationRow | undefined;
+    }
+
+    public getCurrentStackUpdateRecovery(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined {
+        return this.db.prepare(
+            'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? AND is_current = 1 LIMIT 1'
+        ).get(nodeId, stackName) as StackUpdateRecoveryGenerationRow | undefined;
+    }
+
+    public listStackUpdateRecoveryForStack(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow[] {
+        return this.db.prepare(
+            'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? ORDER BY created_at DESC'
+        ).all(nodeId, stackName) as StackUpdateRecoveryGenerationRow[];
+    }
+
+    public listStackUpdateRecoveryGenerationsForNode(nodeId: number): StackUpdateRecoveryGenerationRow[] {
+        return this.db.prepare(
+            'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? ORDER BY created_at DESC'
+        ).all(nodeId) as StackUpdateRecoveryGenerationRow[];
+    }
+
+    public updateStackUpdateRecoveryGeneration(
+        id: string,
+        patch: Partial<Pick<StackUpdateRecoveryGenerationRow,
+            'status' | 'phase' | 'is_current' | 'override_path' | 'health_gate_id' |
+            'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json'>>,
+    ): void {
+        const keys = Object.keys(patch) as Array<keyof typeof patch>;
+        if (keys.length === 0) return;
+        const sets = keys.map((k) => `${String(k)} = ?`).join(', ');
+        const values = keys.map((k) => patch[k]);
+        this.db.prepare(
+            `UPDATE stack_update_recovery_generations SET ${sets}, updated_at = ? WHERE id = ?`
+        ).run(...values, Date.now(), id);
+    }
+
+    public casStackUpdateRecoveryPhase(
+        id: string,
+        expectedPhase: StackUpdateRecoveryGenerationRow['phase'],
+        nextPhase: StackUpdateRecoveryGenerationRow['phase'],
+    ): boolean {
+        const result = this.db.prepare(
+            'UPDATE stack_update_recovery_generations SET phase = ?, updated_at = ? WHERE id = ? AND phase = ?'
+        ).run(nextPhase, Date.now(), id, expectedPhase);
+        return result.changes === 1;
+    }
+
+    public casHandoffGeneration(candidateId: string, nodeId: number, stackName: string): boolean {
+        const handoff = this.db.transaction(() => {
+            const candidate = this.getStackUpdateRecoveryGeneration(candidateId);
+            if (!candidate || candidate.node_id !== nodeId || candidate.stack_name !== stackName) return false;
+            if (candidate.status !== 'candidate' || candidate.phase !== 'acquired') return false;
+            this.db.prepare(
+                `UPDATE stack_update_recovery_generations
+                 SET status = 'superseded', is_current = 0, artifact_expires_at = ?, updated_at = ?
+                 WHERE node_id = ? AND stack_name = ? AND is_current = 1 AND id != ?`
+            ).run(Date.now() + 7 * 24 * 60 * 60 * 1000, Date.now(), nodeId, stackName, candidateId);
+            const result = this.db.prepare(
+                `UPDATE stack_update_recovery_generations
+                 SET status = 'active', is_current = 1, phase = 'handoff_committed', updated_at = ?
+                 WHERE id = ? AND status = 'candidate' AND phase = 'acquired'`
+            ).run(Date.now(), candidateId);
+            return result.changes === 1;
+        });
+        return handoff();
+    }
+
+
+    /** Generations whose Docker/FS artifacts can be retired (not actively held). */
+    public listStackUpdateRecoveryGenerationsForArtifactRetirement(now: number): StackUpdateRecoveryGenerationRow[] {
+        return this.db.prepare(
+            `SELECT * FROM stack_update_recovery_generations
+             WHERE artifacts_retired = 0
+               AND is_current = 0
+               AND status IN ('abandoned', 'superseded')
+               AND (artifact_expires_at IS NULL OR artifact_expires_at <= ?)
+               AND (gate_retain_until IS NULL OR gate_retain_until <= ?)`
+        ).all(now, now) as StackUpdateRecoveryGenerationRow[];
+    }
+
+    public markStackUpdateRecoveryArtifactsRetired(id: string): boolean {
+        const result = this.db.prepare(
+            `UPDATE stack_update_recovery_generations
+             SET artifacts_retired = 1, override_path = NULL, updated_at = ?
+             WHERE id = ? AND artifacts_retired = 0`
+        ).run(Date.now(), id);
+        return result.changes === 1;
+    }
+
+    public abandonStackUpdateRecoveryGeneration(id: string): boolean {
+        const now = Date.now();
+        const result = this.db.prepare(
+            `UPDATE stack_update_recovery_generations
+             SET status = 'abandoned', is_current = 0, artifact_expires_at = ?,
+                 operation_lease_expires_at = NULL, updated_at = ?
+             WHERE id = ? AND status = 'candidate'`
+        ).run(now, now, id);
+        return result.changes === 1;
+    }
+
+    /** Pre-handoff candidates whose operation lease has expired. */
+    public listStaleStackUpdateRecoveryCandidates(now: number): StackUpdateRecoveryGenerationRow[] {
+        return this.db.prepare(
+            `SELECT * FROM stack_update_recovery_generations
+             WHERE status = 'candidate'
+               AND operation_lease_expires_at IS NOT NULL
+               AND operation_lease_expires_at <= ?`
+        ).all(now) as StackUpdateRecoveryGenerationRow[];
+    }
+
+    /** Post-handoff generations stuck before immediate_verified with an expired lease. */
+    public listStuckStackUpdateRecoveryGenerations(now: number): StackUpdateRecoveryGenerationRow[] {
+        return this.db.prepare(
+            `SELECT * FROM stack_update_recovery_generations
+             WHERE status IN ('active', 'restored_current')
+               AND phase IN ('handoff_committed', 'reconciling')
+               AND operation_lease_expires_at IS NOT NULL
+               AND operation_lease_expires_at <= ?`
+        ).all(now) as StackUpdateRecoveryGenerationRow[];
+    }
+
+    public linkStackUpdateRecoveryHealthGate(id: string, healthGateId: string): void {
+        this.db.prepare(
+            'UPDATE stack_update_recovery_generations SET health_gate_id = ?, updated_at = ? WHERE id = ?'
+        ).run(healthGateId, Date.now(), id);
+    }
+
+    public setStackUpdateRecoveryGateRetainUntil(id: string, until: number): void {
+        this.db.prepare(
+            'UPDATE stack_update_recovery_generations SET gate_retain_until = ?, updated_at = ? WHERE id = ?'
+        ).run(until, Date.now(), id);
+    }
+
+    public listHeldStackUpdateRecoveryImageIds(nodeId: number, now: number): string[] {
+        const rows = this.db.prepare(
+            `SELECT services_json FROM stack_update_recovery_generations
+             WHERE node_id = ?
+               AND status IN ('candidate','active','restored_current','recovery_required')
+               AND (artifact_expires_at IS NULL OR artifact_expires_at > ? OR gate_retain_until > ? OR is_current = 1)`
+        ).all(nodeId, now, now) as Array<{ services_json: string }>;
+        const ids = new Set<string>();
+        for (const row of rows) {
+            try {
+                const parsed: unknown = JSON.parse(row.services_json);
+                if (!Array.isArray(parsed)) continue;
+                for (const item of parsed) {
+                    if (!item || typeof item !== 'object') continue;
+                    const replicas = (item as { replicas?: unknown }).replicas;
+                    if (Array.isArray(replicas)) {
+                        for (const replica of replicas) {
+                            if (replica && typeof replica === 'object'
+                                && typeof (replica as { imageId?: unknown }).imageId === 'string'
+                                && (replica as { imageId: string }).imageId.trim()) {
+                                ids.add((replica as { imageId: string }).imageId);
+                            }
+                        }
+                    } else if (typeof (item as { imageId?: unknown }).imageId === 'string') {
+                        ids.add((item as { imageId: string }).imageId);
+                    }
+                }
+            } catch {
+                // Corrupt JSON: skip.
+            }
+        }
+        return [...ids];
+    }
+
+    public deleteStackUpdateRecoveryGenerations(nodeId: number, stackName: string): void {
+        this.db.prepare(
+            'DELETE FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ?'
+        ).run(nodeId, stackName);
+    }
+
+    public insertCleanupPending(row: StackUpdateCleanupPendingRow): void {
+        this.db.prepare(
+            `INSERT INTO stack_update_cleanup_pending (
+                id, node_id, stack_name, status, target_kind, rollback_tags_json,
+                override_paths_json, prune_volumes_requested, created_at, updated_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+        ).run(
+            row.id, row.node_id, row.stack_name, row.status, row.target_kind,
+            row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested,
+            row.created_at, row.updated_at,
+        );
+    }
+
+    public getCleanupPending(id: string): StackUpdateCleanupPendingRow | undefined {
+        return this.db.prepare('SELECT * FROM stack_update_cleanup_pending WHERE id = ?').get(id) as StackUpdateCleanupPendingRow | undefined;
+    }
+
+    public listReadyCleanupPending(): StackUpdateCleanupPendingRow[] {
+        return this.db.prepare(
+            "SELECT * FROM stack_update_cleanup_pending WHERE status = 'ready' ORDER BY created_at ASC"
+        ).all() as StackUpdateCleanupPendingRow[];
+    }
+
+    public listPreparedCleanupPending(): StackUpdateCleanupPendingRow[] {
+        return this.db.prepare(
+            "SELECT * FROM stack_update_cleanup_pending WHERE status = 'prepared' ORDER BY created_at ASC"
+        ).all() as StackUpdateCleanupPendingRow[];
+    }
+
+    public updateCleanupPendingStatus(id: string, status: StackUpdateCleanupPendingRow['status']): void {
+        this.db.prepare(
+            'UPDATE stack_update_cleanup_pending SET status = ?, updated_at = ? WHERE id = ?'
+        ).run(status, Date.now(), id);
+    }
+
+    public deleteCleanupPending(id: string): void {
+        this.db.prepare('DELETE FROM stack_update_cleanup_pending WHERE id = ?').run(id);
+    }
+
+    public hasBlockingDeletionIntent(nodeId: number, stackName: string): boolean {
+        const row = this.db.prepare(
+            `SELECT id FROM stack_update_cleanup_pending
+             WHERE node_id = ? AND stack_name = ? AND status IN ('prepared','ready') LIMIT 1`
+        ).get(nodeId, stackName);
+        return !!row;
+    }
+
+    public getPreparedDeletionIntent(nodeId: number, stackName: string): StackUpdateCleanupPendingRow | undefined {
+        return this.db.prepare(
+            `SELECT * FROM stack_update_cleanup_pending
+             WHERE node_id = ? AND stack_name = ? AND status = 'prepared' LIMIT 1`
+        ).get(nodeId, stackName) as StackUpdateCleanupPendingRow | undefined;
+    }
+
+    public getDeletionIntentById(id: string): StackUpdateCleanupPendingRow | undefined {
+        return this.getCleanupPending(id);
+    }
+
+    public commitStackDeletionReadyTransaction(intentId: string, nodeId: number, stackName: string): boolean {
+        const run = this.db.transaction(() => {
+            const intent = this.getCleanupPending(intentId);
+            if (!intent || intent.status !== 'prepared') return false;
+            if (intent.node_id !== nodeId || intent.stack_name !== stackName) return false;
+            this.db.prepare(
+                "UPDATE stack_update_cleanup_pending SET status = 'ready', updated_at = ? WHERE id = ? AND status = 'prepared'"
+            ).run(Date.now(), intentId);
+            this.deleteStackUpdateRecoveryGenerations(nodeId, stackName);
+            this.deleteServiceUpdateRecoveries(nodeId, stackName);
+            return true;
+        });
+        return run();
+    }
+
     // --- Notification History ---
 
     private mapNotificationRow(row: any): NotificationHistory {
@@ -3878,7 +4222,16 @@ export class DatabaseService {
         this.db.prepare(`UPDATE nodes SET ${fields.join(', ')} WHERE id = ?`).run(...values);
     }
 
-    public deleteNode(id: number): void {
+    /**
+     * Delete a node row and related per-node state.
+     * For a non-default local-socket node, pass localCleanup so a ready tombstone
+     * is inserted in the same transaction (caller enumerates tags/paths first).
+     * Remote hub node records must omit localCleanup (no Docker tombstone).
+     */
+    public deleteNode(
+        id: number,
+        localCleanup?: { tombstoneId: string; tags: string[]; overridePaths: string[] },
+    ): void {
         const node = this.getNode(id);
         // Protect the last local node regardless of is_default flag.
         if (node && node.type === 'local' && this.getLocalNodeCount() <= 1) {
@@ -3887,7 +4240,25 @@ export class DatabaseService {
         if (node?.is_default) {
             throw new Error('Cannot delete the default node');
         }
+        if (localCleanup && node?.type !== 'local') {
+            throw new Error('Local cleanup tombstones are only valid for local-socket nodes');
+        }
         this.db.transaction(() => {
+            if (localCleanup && node?.type === 'local') {
+                const now = Date.now();
+                this.insertCleanupPending({
+                    id: localCleanup.tombstoneId,
+                    node_id: id,
+                    stack_name: null,
+                    status: 'ready',
+                    target_kind: 'local_socket',
+                    rollback_tags_json: JSON.stringify(localCleanup.tags),
+                    override_paths_json: JSON.stringify(localCleanup.overridePaths),
+                    prune_volumes_requested: 0,
+                    created_at: now,
+                    updated_at: now,
+                });
+            }
             this.db.prepare('DELETE FROM scheduled_task_runs WHERE task_id IN (SELECT id FROM scheduled_tasks WHERE node_id = ?)').run(id);
             this.db.prepare('DELETE FROM scheduled_tasks WHERE node_id = ?').run(id);
             this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ?').run(id);
@@ -3901,6 +4272,7 @@ export class DatabaseService {
             this.db.prepare('DELETE FROM preflight_acknowledgements WHERE node_id = ?').run(id);
             this.db.prepare('DELETE FROM stack_exposure WHERE node_id = ?').run(id);
             this.db.prepare('DELETE FROM health_gate_runs WHERE node_id = ?').run(id);
+            this.db.prepare('DELETE FROM stack_update_recovery_generations WHERE node_id = ?').run(id);
             this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ?').run(id);
             this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id);
             this.deleteRoleAssignmentsByResource('node', String(id));

backend/src/services/DeployedStackDeletionService.ts

@@ -0,0 +1,521 @@
+/**
+ * Shared deployed-stack deletion lifecycle (manual DELETE + Blueprint withdrawLocal).
+ *
+ * prepared -> (down, optional volume prune, FS delete) -> ready transaction that
+ * retires full-stack recovery generations and service_update_recovery rows, then
+ * sweeper removes ready tombstone tags/overrides.
+ */
+import { randomUUID } from 'crypto';
+import Docker from 'dockerode';
+import fs from 'fs/promises';
+import path from 'path';
+import {
+  DatabaseService,
+  type StackUpdateCleanupPendingRow,
+} from './DatabaseService';
+import { ComposeService } from './ComposeService';
+import DockerController from './DockerController';
+import { FileSystemService } from './FileSystemService';
+import { NodeRegistry } from './NodeRegistry';
+import { MeshService } from './MeshService';
+import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService';
+import { getErrorMessage } from '../utils/errors';
+import { sanitizeForLog } from '../utils/safeLog';
+import { isPathWithinBase, isValidStackName } from '../utils/validation';
+
+/**
+ * Directory that may contain recovery override files for a tombstone sweep.
+ * Stack-scoped intents are limited to `<composeDir>/<stackName>/`;
+ * node-wide intents (null stack) use the whole compose root.
+ */
+export function overrideDeletionContainmentBase(
+  composeDir: string,
+  stackName: string | null,
+): string | null {
+  const resolvedCompose = path.resolve(composeDir);
+  if (!stackName) return resolvedCompose;
+  if (!isValidStackName(stackName)) return null;
+  const stackDir = path.resolve(resolvedCompose, stackName);
+  if (!isPathWithinBase(stackDir, resolvedCompose)) return null;
+  return stackDir;
+}
+
+export interface DeleteDeployedStackInput {
+  nodeId: number;
+  stackName: string;
+  pruneVolumes: boolean;
+  actor: string;
+  /** When true, skip acquiring a new lock (caller already holds delete via continuation). */
+  continuationIntentId?: string;
+}
+
+export type DeleteDeployedStackResult =
+  | { ok: true }
+  | { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string };
+
+function collectArtifactsFromGenerations(
+  generations: Array<{ override_path: string | null; services_json: string }>,
+): { tags: string[]; overridePaths: string[] } {
+  const tags = new Set<string>();
+  const overridePaths = new Set<string>();
+
+  for (const gen of generations) {
+    if (gen.override_path) overridePaths.add(gen.override_path);
+    try {
+      const parsed: unknown = JSON.parse(gen.services_json);
+      if (!Array.isArray(parsed)) continue;
+      for (const svc of parsed) {
+        if (!svc || typeof svc !== 'object') continue;
+        const replicas = (svc as { replicas?: unknown }).replicas;
+        if (!Array.isArray(replicas)) continue;
+        for (const replica of replicas) {
+          const tag = replica && typeof replica === 'object'
+            ? (replica as { rollbackTag?: unknown }).rollbackTag
+            : null;
+          if (typeof tag === 'string' && tag.trim()) tags.add(tag);
+        }
+      }
+    } catch {
+      // Corrupt capture JSON: skip tags for this generation.
+    }
+  }
+
+  return { tags: [...tags], overridePaths: [...overridePaths] };
+}
+
+function collectArtifacts(nodeId: number, stackName: string): { tags: string[]; overridePaths: string[] } {
+  return collectArtifactsFromGenerations(
+    DatabaseService.getInstance().listStackUpdateRecoveryForStack(nodeId, stackName),
+  );
+}
+
+function parseJsonStringArray(raw: string): string[] {
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    if (!Array.isArray(parsed)) return [];
+    return parsed.filter((item): item is string => typeof item === 'string');
+  } catch {
+    return [];
+  }
+}
+
+export class DeployedStackDeletionService {
+  private static instance: DeployedStackDeletionService;
+
+  public static getInstance(): DeployedStackDeletionService {
+    if (!DeployedStackDeletionService.instance) {
+      DeployedStackDeletionService.instance = new DeployedStackDeletionService();
+    }
+    return DeployedStackDeletionService.instance;
+  }
+
+  public assertNoBlockingDeletionIntent(nodeId: number, stackName: string): void {
+    if (DatabaseService.getInstance().hasBlockingDeletionIntent(nodeId, stackName)) {
+      throw new Error(
+        `Stack "${stackName}" has a deletion in progress and cannot be created or mutated until cleanup finishes.`,
+      );
+    }
+  }
+
+  public async deleteDeployedStack(input: DeleteDeployedStackInput): Promise<DeleteDeployedStackResult> {
+    const { nodeId, stackName, actor } = input;
+    const locks = StackOpLockService.getInstance();
+
+    if (input.continuationIntentId) {
+      const cont = locks.tryAcquireDeletionContinuation({
+        intentId: input.continuationIntentId,
+        nodeId,
+        stackName,
+      });
+      if (!cont.acquired) {
+        return {
+          ok: false,
+          code: 'lock_conflict',
+          error: stackOpSkipMessage(stackName, cont.existing.action),
+          existingAction: cont.existing.action,
+        };
+      }
+      try {
+        return await this.runDeletionBody(input, input.continuationIntentId);
+      } finally {
+        locks.release(nodeId, stackName);
+      }
+    }
+
+    const exclusive = await locks.runExclusive(nodeId, stackName, 'delete', actor, async () => {
+      return this.runDeletionBody(input);
+    });
+    if (!exclusive.ran) {
+      return {
+        ok: false,
+        code: 'lock_conflict',
+        error: stackOpSkipMessage(stackName, exclusive.existing.action),
+        existingAction: exclusive.existing.action,
+      };
+    }
+    return exclusive.result;
+  }
+
+  private async runDeletionBody(
+    input: DeleteDeployedStackInput,
+    existingIntentId?: string,
+  ): Promise<DeleteDeployedStackResult> {
+    const { nodeId, stackName, pruneVolumes } = input;
+    const db = DatabaseService.getInstance();
+
+    let intentId = existingIntentId;
+    if (!intentId) {
+      const { tags, overridePaths } = collectArtifacts(nodeId, stackName);
+      const now = Date.now();
+      const row: StackUpdateCleanupPendingRow = {
+        id: randomUUID(),
+        node_id: nodeId,
+        stack_name: stackName,
+        status: 'prepared',
+        target_kind: 'local_socket',
+        rollback_tags_json: JSON.stringify(tags),
+        override_paths_json: JSON.stringify(overridePaths),
+        prune_volumes_requested: pruneVolumes ? 1 : 0,
+        created_at: now,
+        updated_at: now,
+      };
+      try {
+        db.insertCleanupPending(row);
+      } catch (error) {
+        return {
+          ok: false,
+          code: 'tombstone_failed',
+          error: getErrorMessage(error, 'Failed to prepare stack deletion'),
+        };
+      }
+      intentId = row.id;
+    }
+
+    const intent = db.getDeletionIntentById(intentId);
+    if (!intent || intent.status !== 'prepared') {
+      return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
+    }
+
+    try {
+      await ComposeService.getInstance(nodeId).downStack(stackName);
+    } catch (downErr) {
+      console.warn(
+        '[DeployedStackDeletion] Compose down failed or no-op for %s:',
+        sanitizeForLog(stackName),
+        downErr,
+      );
+    }
+
+    if (intent.prune_volumes_requested === 1) {
+      try {
+        await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
+      } catch (pruneErr) {
+        console.warn(
+          '[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
+          sanitizeForLog(stackName),
+          pruneErr,
+        );
+      }
+    }
+
+    try {
+      await FileSystemService.getInstance(nodeId).deleteStack(stackName);
+    } catch (fsErr) {
+      db.updateCleanupPendingStatus(intentId, 'cancelled');
+      return {
+        ok: false,
+        code: 'fs_failed',
+        error: getErrorMessage(fsErr, 'Failed to remove stack files'),
+      };
+    }
+
+    if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) {
+      return {
+        ok: false,
+        code: 'db_failed',
+        error: 'Failed to commit stack deletion ready transaction',
+      };
+    }
+
+    try {
+      db.clearStackUpdateStatus(nodeId, stackName);
+      db.clearStackScanAttempts(nodeId, stackName);
+      db.deleteRoleAssignmentsByResource('stack', stackName);
+      db.deleteGitSource(stackName);
+      db.deleteStackDossier(nodeId, stackName);
+      db.deleteStackDriftFindings(nodeId, stackName);
+      db.deleteStackExposureIntents(nodeId, stackName);
+      db.deleteStackExposure(nodeId, stackName);
+      db.deleteStackProjectEnvFiles(nodeId, stackName);
+      db.deleteStackScans(nodeId, stackName);
+    } catch (dbErr) {
+      console.error(
+        '[DeployedStackDeletion] Secondary DB cleanup failed for %s; recovery rows already retired:',
+        sanitizeForLog(stackName),
+        dbErr,
+      );
+      return {
+        ok: false,
+        code: 'db_failed',
+        error: getErrorMessage(dbErr, 'Failed to clear stack database state'),
+      };
+    }
+
+    try {
+      await MeshService.getInstance().optOutStack(nodeId, stackName, input.actor);
+    } catch (meshErr) {
+      console.warn(
+        '[DeployedStackDeletion] Mesh opt-out failed for %s:',
+        sanitizeForLog(stackName),
+        meshErr,
+      );
+    }
+
+    await this.sweepReadyIntent(intentId);
+    return { ok: true };
+  }
+
+  /**
+   * Remove rollback tags + override paths for a ready tombstone, then drop the row.
+   * When the node row is already gone (local-node delete), pass a preserved local
+   * Docker handle and composeDir so we never fall back to a remote default node.
+   */
+  public async sweepReadyIntent(
+    intentId: string,
+    opts?: { docker?: Docker; composeDir?: string },
+  ): Promise<void> {
+    const db = DatabaseService.getInstance();
+    const intent = db.getCleanupPending(intentId);
+    if (!intent || intent.status !== 'ready') return;
+    if (intent.node_id == null) {
+      db.deleteCleanupPending(intentId);
+      return;
+    }
+
+    const tags = parseJsonStringArray(intent.rollback_tags_json);
+    const overridePaths = parseJsonStringArray(intent.override_paths_json);
+
+    let docker: Docker;
+    if (opts?.docker) {
+      docker = opts.docker;
+    } else if (db.getNode(intent.node_id)) {
+      docker = DockerController.getInstance(intent.node_id).getDocker();
+    } else if (intent.target_kind === 'local_socket') {
+      // Deleted local node: local_socket tombstones always target the host Docker socket.
+      docker = new Docker();
+    } else {
+      console.warn(
+        '[DeployedStackDeletion] Cannot sweep tombstone %s: node gone and target is not local_socket',
+        sanitizeForLog(intentId),
+      );
+      return;
+    }
+
+    let incomplete = false;
+
+    for (const tag of tags) {
+      try {
+        await docker.getImage(tag).remove({ force: true });
+      } catch (error) {
+        const status = (error as { statusCode?: number }).statusCode;
+        const message = getErrorMessage(error, 'unknown').toLowerCase();
+        if (status === 404 || message.includes('no such image') || message.includes('not found')) {
+          continue;
+        }
+        incomplete = true;
+        console.warn(
+          '[DeployedStackDeletion] Failed to remove rollback tag %s: %s',
+          sanitizeForLog(tag),
+          sanitizeForLog(getErrorMessage(error, 'unknown')),
+        );
+      }
+    }
+
+    const baseDir = opts?.composeDir
+      ?? (db.getNode(intent.node_id)
+        ? FileSystemService.getInstance(intent.node_id).getBaseDir()
+        : null);
+
+    const containmentBase = baseDir
+      ? overrideDeletionContainmentBase(baseDir, intent.stack_name)
+      : null;
+    if (baseDir && !containmentBase) {
+      incomplete = true;
+      console.warn(
+        '[DeployedStackDeletion] Refusing override sweep: invalid stack containment for tombstone %s',
+        sanitizeForLog(intentId),
+      );
+    }
+
+    for (const overridePath of overridePaths) {
+      try {
+        const resolved = path.resolve(overridePath);
+        const basename = path.basename(resolved);
+        if (!/^\.sencho-recovery-[a-f0-9]+\.yml$/i.test(basename)) {
+          incomplete = true;
+          console.warn(
+            '[DeployedStackDeletion] Refusing to delete non-recovery override: %s',
+            sanitizeForLog(overridePath),
+          );
+          continue;
+        }
+        if (containmentBase && !isPathWithinBase(resolved, containmentBase)) {
+          incomplete = true;
+          console.warn(
+            '[DeployedStackDeletion] Refusing to delete override outside stack containment: %s',
+            sanitizeForLog(overridePath),
+          );
+          continue;
+        }
+        if (!baseDir && !path.isAbsolute(resolved)) {
+          incomplete = true;
+          console.warn(
+            '[DeployedStackDeletion] Refusing relative override without compose dir: %s',
+            sanitizeForLog(overridePath),
+          );
+          continue;
+        }
+        if (!containmentBase && baseDir) {
+          incomplete = true;
+          continue;
+        }
+        await fs.unlink(resolved);
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+          incomplete = true;
+          console.warn(
+            '[DeployedStackDeletion] Failed to delete override %s: %s',
+            sanitizeForLog(overridePath),
+            sanitizeForLog(getErrorMessage(error, 'unknown')),
+          );
+        }
+      }
+    }
+
+    // Keep the ready tombstone when cleanup is incomplete so startup can retry.
+    if (!incomplete) {
+      db.deleteCleanupPending(intentId);
+    }
+  }
+
+  /** Enumerate rollback tags and override paths for every stack recovery on a node. */
+  public collectNodeArtifacts(nodeId: number): { tags: string[]; overridePaths: string[] } {
+    return collectArtifactsFromGenerations(
+      DatabaseService.getInstance().listStackUpdateRecoveryGenerationsForNode(nodeId),
+    );
+  }
+
+  /**
+   * Delete a local-socket node with an atomic ready tombstone, then sweep.
+   * Remote node records call DatabaseService.deleteNode without cleanup.
+   */
+  public async deleteLocalNode(nodeId: number): Promise<void> {
+    const db = DatabaseService.getInstance();
+    const node = db.getNode(nodeId);
+    if (!node) throw new Error('Node not found');
+    if (node.type !== 'local') {
+      db.deleteNode(nodeId);
+      return;
+    }
+    // Preserve Docker + compose dir before the row disappears so sweep never
+    // targets a remote default node.
+    const docker = DockerController.getInstance(nodeId).getDocker();
+    const composeDir = FileSystemService.getInstance(nodeId).getBaseDir();
+    const { tags, overridePaths } = this.collectNodeArtifacts(nodeId);
+    const tombstoneId = randomUUID();
+    db.deleteNode(nodeId, { tombstoneId, tags, overridePaths });
+    NodeRegistry.getInstance().evictConnection(nodeId);
+    try {
+      await this.sweepReadyIntent(tombstoneId, { docker, composeDir });
+    } catch (error) {
+      // Node row is already gone; leave the ready tombstone for startup resume.
+      console.error(
+        '[DeployedStackDeletion] Sweep after local-node delete deferred for tombstone %s: %s',
+        sanitizeForLog(tombstoneId),
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+    }
+  }
+
+  /**
+   * Startup reconciliation: resume prepared intents, complete ready transitions
+   * when the directory is already gone, and sweep ready tombstones.
+   */
+  public async reconcileAtStartup(): Promise<void> {
+    const db = DatabaseService.getInstance();
+    for (const intent of db.listPreparedCleanupPending()) {
+      if (intent.node_id == null || !intent.stack_name) continue;
+      const nodeId = intent.node_id;
+      const stackName = intent.stack_name;
+      const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName);
+      let dirExists = true;
+      try {
+        await fs.access(stackDir);
+      } catch (accessError) {
+        const code = (accessError as NodeJS.ErrnoException).code;
+        if (code === 'ENOENT') {
+          dirExists = false;
+        } else {
+          console.warn(
+            '[DeployedStackDeletion] Startup access error for %s (not treating as absent): %s',
+            sanitizeForLog(stackName),
+            sanitizeForLog(getErrorMessage(accessError, 'unknown')),
+          );
+          continue;
+        }
+      }
+
+      if (!dirExists) {
+        if (!db.commitStackDeletionReadyTransaction(intent.id, nodeId, stackName)) {
+          console.warn(
+            '[DeployedStackDeletion] Startup ready commit failed for %s/%s',
+            nodeId,
+            sanitizeForLog(stackName),
+          );
+          continue;
+        }
+        try {
+          await MeshService.getInstance().optOutStack(nodeId, stackName, 'system:startup');
+        } catch (meshErr) {
+          console.warn(
+            '[DeployedStackDeletion] Startup mesh opt-out failed for %s: %s',
+            sanitizeForLog(stackName),
+            sanitizeForLog(getErrorMessage(meshErr, 'unknown')),
+          );
+        }
+        await this.sweepReadyIntent(intent.id);
+        continue;
+      }
+
+      const result = await this.deleteDeployedStack({
+        nodeId,
+        stackName,
+        pruneVolumes: intent.prune_volumes_requested === 1,
+        actor: 'system:startup',
+        continuationIntentId: intent.id,
+      });
+      if (!result.ok) {
+        console.warn(
+          '[DeployedStackDeletion] Startup resume failed for %s: %s',
+          sanitizeForLog(stackName),
+          result.error,
+        );
+      }
+    }
+
+    for (const intent of db.listReadyCleanupPending()) {
+      if (intent.node_id != null && intent.stack_name) {
+        try {
+          await MeshService.getInstance().optOutStack(intent.node_id, intent.stack_name, 'system:startup');
+        } catch (meshErr) {
+          console.warn(
+            '[DeployedStackDeletion] Ready-resume mesh opt-out failed for %s: %s',
+            sanitizeForLog(intent.stack_name),
+            sanitizeForLog(getErrorMessage(meshErr, 'unknown')),
+          );
+        }
+      }
+      await this.sweepReadyIntent(intent.id);
+    }
+  }
+}

backend/src/services/DockerController.ts

@@ -24,6 +24,7 @@ import {
 import type { NetworkingNetworkBase } from './network/networkingTypes';
 import { isPathWithinBase } from '../utils/validation';
 import { isDebugEnabled } from '../utils/debug';
+import { getErrorMessage } from '../utils/errors';
 import { sanitizeForLog } from '../utils/safeLog';
 import { describeSpawnError } from '../utils/spawnErrors';
 import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
@@ -2258,6 +2259,57 @@ class DockerController {
     }
   }
 
+  /**
+   * Strict Result API for full-stack updates. Never converts Compose-ps + fallback
+   * failure into empty success. Compose-managed containers are never orphan IDs.
+   */
+  public async classifyLegacyOrphansForUpdate(
+    stackName: string,
+  ): Promise<
+    | { status: 'none' }
+    | { status: 'orphans'; ids: string[] }
+    | { status: 'classification_failed'; error: string }
+  > {
+    const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName);
+    const toIds = (list: Array<{ Id?: string }>) =>
+      list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0)
+        .map((c) => c.Id);
+
+    const fallbackOrphans = async (): Promise<
+      | { status: 'none' }
+      | { status: 'orphans'; ids: string[] }
+      | { status: 'classification_failed'; error: string }
+    > => {
+      try {
+        const ids = toIds(await this.smartFallback(stackName, stackDir));
+        return ids.length === 0 ? { status: 'none' } : { status: 'orphans', ids };
+      } catch (fallbackError) {
+        return {
+          status: 'classification_failed',
+          error: getErrorMessage(fallbackError, 'Legacy orphan classification failed'),
+        };
+      }
+    };
+
+    try {
+      const composeContainers = await this.fetchComposePsContainers(stackName, stackDir);
+      // Compose already manages this stack: no legacy orphan cleanup (same as deploy).
+      if (composeContainers.length > 0) return { status: 'none' };
+      return await fallbackOrphans();
+    } catch (error) {
+      const execError = error as NodeJS.ErrnoException & { stderr?: string };
+      const mapped = describeSpawnError(execError, { command: 'docker compose ps' });
+      const detail = execError.stderr || mapped.message || getErrorMessage(error, 'docker compose ps failed');
+      console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail));
+      // Unlike getLegacyOrphanContainersByStack, never convert dual failure into empty success.
+      const fallback = await fallbackOrphans();
+      if (fallback.status === 'classification_failed') {
+        return { status: 'classification_failed', error: String(detail) };
+      }
+      return fallback;
+    }
+  }
+
   public async getContainersByStack(stackName: string) {
     // Resolve the compose dir and the authored prefix for THIS controller's node,
     // not the process default, so a non-default local node sees its own stack dir

backend/src/services/FileSystemService.ts

@@ -545,6 +545,11 @@ export class FileSystemService {
   }
 
   async createStack(stackName: string): Promise<void> {
+    if (DatabaseService.getInstance().hasBlockingDeletionIntent(this.nodeId, stackName)) {
+      throw new Error(
+        `Stack "${stackName}" has a deletion in progress and cannot be created until cleanup finishes.`,
+      );
+    }
     const stackDir = this.resolveStackDir(stackName);
     await this.assertRealWithinBase(stackDir);
 

backend/src/services/SchedulerService.ts

@@ -1104,7 +1104,13 @@ export class SchedulerService {
         );
         if (!lock.ran) return skipMessage(stackName, lock.existing.action);
         db.clearStackUpdateStatus(nodeId, stackName);
-        HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
+        const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
+        const orchResult = lock.result;
+        const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
+        if (recoveryId) {
+            const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
+            StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
+        }
 
         this.safeDispatch(
             'info',

backend/src/services/ServiceUpdateRecoveryService.ts

@@ -237,7 +237,17 @@ export class ServiceUpdateRecoveryService {
     return (imageId: string) => {
       const held = this.getHeldImageIds(nodeId);
       if (held === null) return true;
-      return held.has(imageId);
+      if (held.has(imageId)) return true;
+      try {
+        // Dynamic import avoids a static cycle with StackUpdateRecoveryService.
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
+        const { StackUpdateRecoveryService } = require('./StackUpdateRecoveryService') as typeof import('./StackUpdateRecoveryService');
+        const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
+        if (stackHeld === null) return true;
+        return stackHeld.has(imageId);
+      } catch {
+        return true;
+      }
     };
   }
 

backend/src/services/StackOpLockService.ts

@@ -1,3 +1,4 @@
+import { DatabaseService } from './DatabaseService';
 /**
  * Tracks in-flight stack lifecycle operations (deploy, down, restart, stop,
  * start, update, rollback, backup) per (nodeId, stackName). A second request to
@@ -9,7 +10,7 @@
  * which matches the lifecycle of any in-flight `docker compose` child process.
  */
 
-export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup';
+export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete';
 
 /**
  * Note returned by a background path that skipped its operation because a manual
@@ -70,6 +71,31 @@ export class StackOpLockService {
     this.locks.delete(this.key(nodeId, stackName));
   }
 
+  /**
+   * Matching-intent continuation for crash-resume of a prepared deletion.
+   * Succeeds only when the persisted intent matches and no other in-memory lock holds.
+   */
+  public tryAcquireDeletionContinuation(args: {
+    intentId: string;
+    nodeId: number;
+    stackName: string;
+  }): AcquireResult {
+    const intent = DatabaseService.getInstance().getDeletionIntentById(args.intentId);
+    if (
+      !intent
+      || intent.status !== 'prepared'
+      || intent.node_id !== args.nodeId
+      || intent.stack_name !== args.stackName
+    ) {
+      const existing = this.locks.get(this.key(args.nodeId, args.stackName));
+      return {
+        acquired: false,
+        existing: existing ?? { action: 'delete', startedAt: Date.now(), user: 'system' },
+      };
+    }
+    return this.tryAcquire(args.nodeId, args.stackName, 'delete', 'system:deletion-continuation');
+  }
+
   /**
    * Acquire the per-(nodeId, stackName) lock for the duration of `fn`, then
    * release it. Returns `{ ran: true, result }` when the lock was free, or

backend/src/services/StackUpdateOrchestrator.ts

@@ -47,7 +47,7 @@ export interface ServiceUpdateOptions {
 }
 
 export type OrchestratorResult =
-  | { kind: 'stack_compose_done' }
+  | { kind: 'stack_compose_done'; recoveryId: string | null }
   | {
       kind: 'service_done';
       serviceName: string;
@@ -182,10 +182,11 @@ export class StackUpdateOrchestrator {
     ctx: UpdateOperationContext,
     options: StackComposeOptions,
   ): Promise<OrchestratorResult> {
-    await ComposeService.getInstance(ctx.nodeId).updateStack(
+    const updateResult = await ComposeService.getInstance(ctx.nodeId).updateStack(
       ctx.stackName, options.terminalWs ?? undefined, options.atomic,
     );
-    return { kind: 'stack_compose_done' };
+    const recoveryId = updateResult?.recoveryId ?? null;
+    return { kind: 'stack_compose_done', recoveryId };
   }
 
   private async executeServiceUpdate(
@@ -195,6 +196,16 @@ export class StackUpdateOrchestrator {
   ): Promise<OrchestratorResult> {
     const { nodeId, stackName } = ctx;
 
+    {
+      const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
+      if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) {
+        return serviceFailed(
+          'stack_recovery_pin_active',
+          'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.',
+        );
+      }
+    }
+
     const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName);
     if (!loaded.ok) return loaded.result;
     const { spec, services } = loaded;
@@ -308,6 +319,16 @@ export class StackUpdateOrchestrator {
     const { nodeId, stackName } = ctx;
     const recoveryId = options.recoveryId as string;
 
+    {
+      const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
+      if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) {
+        return serviceFailed(
+          'stack_recovery_pin_active',
+          'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.',
+          { recoveryId },
+        );
+      }
+    }
     const recovery = ServiceUpdateRecoveryService.getInstance().get(recoveryId);
     if (
       !recovery ||

backend/src/services/StackUpdateRecoveryService.ts

@@ -0,0 +1,767 @@
+/**
+ * Full-stack update recovery generations: capture, opaque rollback tags,
+ * stack-local recovery override, handoff, compensation, and prune holds.
+ *
+ * Separate from ServiceUpdateRecoveryService (service-scoped snapshots).
+ * Does not run Compose; ComposeService / orchestrator own Docker mutations.
+ *
+ * Recovery fidelity contract (supported):
+ * - Prior images are restored via opaque hold tags (`--pull never --no-build`).
+ * - Observed running replica count is restored via compose `scale`.
+ * - Services that were fully stopped at capture are kept at `scale: 0`.
+ * - Authored replica counts are not used when they diverge from observed state.
+ */
+import { randomUUID } from 'crypto';
+import fs from 'fs/promises';
+import path from 'path';
+import {
+  DatabaseService,
+  type StackUpdateRecoveryGenerationRow,
+} from './DatabaseService';
+import DockerController from './DockerController';
+import { FileSystemService } from './FileSystemService';
+import { buildEffectiveServiceModel } from './effectiveServiceModel';
+import {
+  classifyReferenceKind,
+  type ImageReferenceKind,
+  resolveComposeProjectContext,
+} from './composeProjectContext';
+import { getComposeCommandTimeoutMs } from './ComposeService';
+import { getErrorMessage } from '../utils/errors';
+import { sanitizeForLog } from '../utils/safeLog';
+import { isValidStackName } from '../utils/validation';
+
+const SWEEP_INTERVAL_MS = 5 * 60_000;
+const INITIAL_SWEEP_DELAY_MS = 30_000;
+const MIN_RECOVERY_WINDOW_SECONDS = 90;
+const RECOVERY_TTL_BUFFER_MS = 30 * 60_000;
+const GATE_RETAIN_DEFAULT_MS = 2 * 60 * 60_000;
+const RECOVERY_PROBE_DELAY_MS = 3_000;
+
+export interface StackRecoveryReplicaCapture {
+  containerId: string | null;
+  imageId: string | null;
+  repoDigest: string | null;
+  state: 'running' | 'stopped' | 'none';
+  rollbackTag: string | null;
+}
+
+export interface StackRecoveryServiceCapture {
+  serviceName: string;
+  /** Observed running replica count at capture (supported restore scale). */
+  scale: number;
+  hasBuild: boolean;
+  declaredImageRef: string | null;
+  referenceKind: ImageReferenceKind;
+  replicas: StackRecoveryReplicaCapture[];
+}
+
+export interface CaptureStackUpdateInput {
+  nodeId: number;
+  stackName: string;
+  createdBy: string | null;
+}
+
+function yamlQuote(value: string): string {
+  return JSON.stringify(value);
+}
+
+function sanitizeServiceSlug(name: string): string {
+  return name.replace(/[^a-zA-Z0-9._-]/g, '-').toLowerCase() || 'svc';
+}
+
+function opaqueRollbackTag(generationId: string, serviceName: string): string {
+  const short = generationId.replace(/-/g, '').slice(0, 12);
+  return `sencho-rb/${short}/${sanitizeServiceSlug(serviceName)}:hold`;
+}
+
+function parseServicesJson(raw: string): StackRecoveryServiceCapture[] {
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    if (!Array.isArray(parsed)) return [];
+    return parsed as StackRecoveryServiceCapture[];
+  } catch {
+    return [];
+  }
+}
+
+export function collectImageIdsFromServicesJson(servicesJson: string): string[] {
+  const ids = new Set<string>();
+  for (const svc of parseServicesJson(servicesJson)) {
+    for (const replica of svc.replicas ?? []) {
+      if (replica.imageId && replica.imageId.trim()) ids.add(replica.imageId);
+    }
+  }
+  return [...ids];
+}
+
+function collectRollbackTags(services: StackRecoveryServiceCapture[]): string[] {
+  const tags = new Set<string>();
+  for (const svc of services) {
+    for (const replica of svc.replicas ?? []) {
+      if (replica.rollbackTag) tags.add(replica.rollbackTag);
+    }
+  }
+  return [...tags];
+}
+
+export class StackUpdateRecoveryService {
+  private static instance: StackUpdateRecoveryService;
+  private started = false;
+  private intervalId: NodeJS.Timeout | null = null;
+  private initialTimer: NodeJS.Timeout | null = null;
+
+  private constructor() {}
+
+  public static getInstance(): StackUpdateRecoveryService {
+    if (!StackUpdateRecoveryService.instance) {
+      StackUpdateRecoveryService.instance = new StackUpdateRecoveryService();
+    }
+    return StackUpdateRecoveryService.instance;
+  }
+
+  public static resetForTests(): void {
+    if (StackUpdateRecoveryService.instance) {
+      StackUpdateRecoveryService.instance.stop();
+    }
+    StackUpdateRecoveryService.instance = new StackUpdateRecoveryService();
+  }
+
+  public start(): void {
+    this.started = true;
+    if (this.initialTimer || this.intervalId) return;
+    this.initialTimer = setTimeout(() => {
+      void this.reconcileIncomplete();
+      this.intervalId = setInterval(() => {
+        void this.reconcileIncomplete();
+      }, SWEEP_INTERVAL_MS);
+    }, INITIAL_SWEEP_DELAY_MS);
+  }
+
+  public stop(): void {
+    this.started = false;
+    if (this.initialTimer) {
+      clearTimeout(this.initialTimer);
+      this.initialTimer = null;
+    }
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+  }
+
+  /**
+   * Validate exact Compose invocation, backup files, snapshot runtime, create
+   * opaque tags + recovery override, insert candidate generation.
+   */
+  public async captureCandidate(input: CaptureStackUpdateInput): Promise<StackUpdateRecoveryGenerationRow> {
+    const { nodeId, stackName, createdBy } = input;
+    if (!isValidStackName(stackName)) {
+      throw new Error('Invalid stack name');
+    }
+
+    const context = await resolveComposeProjectContext(nodeId, stackName);
+    await context.validateForMutation();
+    // Exact mutating invocation (authored files + env + generated Mesh override)
+    // must validate before any backup, tag, or override write.
+    const { ComposeService } = await import('./ComposeService');
+    await ComposeService.getInstance(nodeId).validateExactComposeInvocation(stackName);
+
+    const backupSlotId = await context.backupFromContext('update');
+
+    const model = await buildEffectiveServiceModel(nodeId, stackName);
+    if (!model.renderable) {
+      throw new Error(model.error || 'Effective Compose model failed to render');
+    }
+
+    const generationId = randomUUID();
+    const docker = DockerController.getInstance(nodeId).getDocker();
+    const services: StackRecoveryServiceCapture[] = [];
+    const createdTags: string[] = [];
+    let overridePath: string | null = null;
+
+    try {
+      for (const spec of model.services) {
+        const declaredImageRef = spec.declaredImage;
+        const referenceKind = classifyReferenceKind(declaredImageRef);
+        const listed = await docker.listContainers({
+          all: true,
+          filters: {
+            label: [
+              `com.docker.compose.project=${stackName}`,
+              `com.docker.compose.service=${spec.name}`,
+            ],
+          },
+        });
+
+        const replicas: StackRecoveryReplicaCapture[] = [];
+        for (const info of listed) {
+          try {
+            const inspect = await docker.getContainer(info.Id).inspect();
+            const status = inspect.State?.Status;
+            const state: StackRecoveryReplicaCapture['state'] =
+              status === 'running' ? 'running' : status ? 'stopped' : 'none';
+            const imageId = typeof inspect.Image === 'string' && inspect.Image.length > 0
+              ? inspect.Image
+              : null;
+            if ((state === 'running' || state === 'stopped') && !imageId) {
+              throw new Error(
+                `Service "${spec.name}" replica ${info.Id.slice(0, 12)} has no protectable image id`,
+              );
+            }
+            let repoDigest: string | null = null;
+            if (imageId) {
+              try {
+                const image = await docker.getImage(imageId).inspect();
+                const digests = (image.RepoDigests ?? []) as string[];
+                repoDigest = digests.length > 0 ? digests[0] : null;
+              } catch {
+                repoDigest = null;
+              }
+            }
+            replicas.push({
+              containerId: info.Id,
+              imageId,
+              repoDigest,
+              state,
+              rollbackTag: null,
+            });
+          } catch (error) {
+            if ((error as { statusCode?: number })?.statusCode === 404) continue;
+            throw error;
+          }
+        }
+
+        const runningCount = replicas.filter((r) => r.state === 'running').length;
+        services.push({
+          serviceName: spec.name,
+          scale: runningCount,
+          hasBuild: spec.hasBuild,
+          declaredImageRef,
+          referenceKind,
+          replicas,
+        });
+      }
+
+      const taggedIds = new Set<string>();
+      for (const svc of services) {
+        const primary = svc.replicas.find((r) => r.imageId) ?? null;
+        if (!primary?.imageId) continue;
+
+        const tag = opaqueRollbackTag(generationId, svc.serviceName);
+        const tagKey = `${primary.imageId}|${tag}`;
+        if (!taggedIds.has(tagKey)) {
+          const { repo, tagName } = splitOpaqueTag(tag);
+          await docker.getImage(primary.imageId).tag({ repo, tag: tagName });
+          taggedIds.add(tagKey);
+          createdTags.push(tag);
+        }
+        for (const replica of svc.replicas) {
+          if (replica.imageId === primary.imageId) {
+            replica.rollbackTag = tag;
+          }
+        }
+      }
+
+      overridePath = await this.writeRecoveryOverride(nodeId, stackName, generationId, services);
+
+      const now = Date.now();
+      const row: StackUpdateRecoveryGenerationRow = {
+        id: generationId,
+        node_id: nodeId,
+        stack_name: stackName,
+        status: 'candidate',
+        phase: 'captured',
+        is_current: 0,
+        backup_slot_id: backupSlotId,
+        override_path: overridePath,
+        services_json: JSON.stringify(services),
+        health_gate_id: null,
+        gate_retain_until: null,
+        artifact_expires_at: null,
+        operation_lease_expires_at: now + getComposeCommandTimeoutMs() + RECOVERY_TTL_BUFFER_MS,
+        created_at: now,
+        updated_at: now,
+        created_by: createdBy,
+        artifacts_retired: 0,
+      };
+      DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row);
+      return row;
+    } catch (error) {
+      await this.bestEffortRemoveTags(nodeId, createdTags);
+      if (overridePath) {
+        try {
+          await fs.unlink(overridePath);
+        } catch {
+          // Best-effort mid-capture cleanup.
+        }
+      }
+      throw error;
+    }
+  }
+
+  private async writeRecoveryOverride(
+    nodeId: number,
+    stackName: string,
+    generationId: string,
+    services: StackRecoveryServiceCapture[],
+  ): Promise<string> {
+    if (!isValidStackName(stackName)) {
+      throw new Error('Invalid stack name');
+    }
+    // Canonical inline path barrier (same pattern as ComposeService.renderConfig).
+    const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir());
+    const stackDir = path.resolve(baseResolved, stackName);
+    if (!stackDir.startsWith(baseResolved + path.sep)) {
+      throw new Error('Invalid stack path');
+    }
+    let stackDirReal: string;
+    let baseReal: string;
+    try {
+      [stackDirReal, baseReal] = await Promise.all([
+        fs.realpath(stackDir),
+        fs.realpath(baseResolved),
+      ]);
+    } catch {
+      throw new Error('Stack directory not found');
+    }
+    if (stackDirReal !== baseReal && !stackDirReal.startsWith(baseReal + path.sep)) {
+      throw new Error('Stack directory escapes compose base');
+    }
+
+    const short = generationId.replace(/-/g, '').slice(0, 12);
+    if (!/^[a-f0-9]{12}$/i.test(short)) {
+      throw new Error('Invalid recovery generation id');
+    }
+    const filename = `.sencho-recovery-${short}.yml`;
+    const abs = path.resolve(stackDirReal, filename);
+    if (!abs.startsWith(stackDirReal + path.sep)) {
+      throw new Error('Recovery override path escapes stack directory');
+    }
+
+    const lines: string[] = ['services:'];
+    let wroteAny = false;
+    for (const svc of services) {
+      const tag = svc.replicas.find((r) => r.rollbackTag)?.rollbackTag ?? null;
+      const stoppedOnly =
+        svc.replicas.length > 0
+        && svc.replicas.every((r) => r.state === 'stopped' || r.state === 'none');
+      if (!tag && svc.scale !== 0 && !stoppedOnly) continue;
+      const key = /^[a-zA-Z0-9._-]+$/.test(svc.serviceName) ? svc.serviceName : yamlQuote(svc.serviceName);
+      lines.push(`  ${key}:`);
+      if (tag) {
+        lines.push(`    image: ${yamlQuote(tag)}`);
+      }
+      // Observed running count; fully stopped services stay at scale 0.
+      if (svc.scale === 0 || stoppedOnly) {
+        lines.push('    scale: 0');
+      } else if (svc.scale > 0) {
+        lines.push(`    scale: ${svc.scale}`);
+      }
+      wroteAny = true;
+    }
+    const body = wroteAny ? `${lines.join('\n')}\n` : 'services: {}\n';
+    await fs.writeFile(abs, body, 'utf8');
+    return abs;
+  }
+
+  public markAcquired(id: string): boolean {
+    return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'captured', 'acquired');
+  }
+
+  public handoff(candidateId: string, nodeId: number, stackName: string): boolean {
+    return DatabaseService.getInstance().casHandoffGeneration(candidateId, nodeId, stackName);
+  }
+
+  public markReconciling(id: string): boolean {
+    return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'handoff_committed', 'reconciling');
+  }
+
+  public markImmediateVerified(id: string): boolean {
+    const ok = DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'reconciling', 'immediate_verified');
+    if (ok) {
+      DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(id, {
+        artifact_expires_at: Date.now() + this.activeRecoveryTtlMs() + RECOVERY_TTL_BUFFER_MS,
+        operation_lease_expires_at: null,
+      });
+    }
+    return ok;
+  }
+
+  /** Mark candidate abandoned in DB and retire Docker/FS artifacts. */
+  public async abandon(id: string): Promise<boolean> {
+    const row = this.get(id);
+    const ok = DatabaseService.getInstance().abandonStackUpdateRecoveryGeneration(id);
+    if (ok && row) {
+      await this.retireGenerationArtifacts({ ...row, status: 'abandoned', artifacts_retired: 0 });
+    }
+    return ok;
+  }
+
+  public linkHealthGate(id: string, healthGateId: string): void {
+    DatabaseService.getInstance().linkStackUpdateRecoveryHealthGate(id, healthGateId);
+  }
+
+  public setGateRetainUntil(id: string, until: number = Date.now() + GATE_RETAIN_DEFAULT_MS): void {
+    DatabaseService.getInstance().setStackUpdateRecoveryGateRetainUntil(id, until);
+  }
+
+  /** Link a health gate when present; otherwise set bounded gate_retain_until. */
+  public linkGateOrRetain(id: string, healthGateId: string | null): void {
+    try {
+      if (healthGateId) {
+        this.linkHealthGate(id, healthGateId);
+      } else {
+        this.setGateRetainUntil(id);
+      }
+    } catch (error) {
+      console.warn(
+        '[StackUpdateRecovery] linkGateOrRetain failed for %s; setting retain window:',
+        sanitizeForLog(id),
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+      try {
+        this.setGateRetainUntil(id);
+      } catch (retainError) {
+        console.warn(
+          '[StackUpdateRecovery] setGateRetainUntil failed for %s:',
+          sanitizeForLog(id),
+          sanitizeForLog(getErrorMessage(retainError, 'unknown')),
+        );
+      }
+    }
+  }
+
+  public get(id: string): StackUpdateRecoveryGenerationRow | undefined {
+    return DatabaseService.getInstance().getStackUpdateRecoveryGeneration(id);
+  }
+
+  public getCurrent(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined {
+    return DatabaseService.getInstance().getCurrentStackUpdateRecovery(nodeId, stackName);
+  }
+
+  public isRestoredCurrentPinActive(nodeId: number, stackName: string): boolean {
+    const current = this.getCurrent(nodeId, stackName);
+    return !!current && current.status === 'restored_current';
+  }
+
+  public getHeldImageIds(nodeId: number): Set<string> | null {
+    try {
+      const now = Date.now();
+      const fromStack = DatabaseService.getInstance().listHeldStackUpdateRecoveryImageIds(nodeId, now);
+      return new Set(fromStack);
+    } catch (error) {
+      console.warn(
+        '[StackUpdateRecovery] Failed to compute held image ids for node %d:',
+        nodeId,
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+      return null;
+    }
+  }
+
+  /**
+   * Unified held-image predicate: service-scoped + full-stack holds.
+   * Fail closed (skip prune) when either lookup fails.
+   */
+  public buildUnifiedHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
+    // Dynamic require avoids a static cycle with ServiceUpdateRecoveryService.
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { ServiceUpdateRecoveryService } = require('./ServiceUpdateRecoveryService') as typeof import('./ServiceUpdateRecoveryService');
+    const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
+    const stackHeld = this.getHeldImageIds(nodeId);
+    if (serviceHeld === null || stackHeld === null) {
+      return () => true;
+    }
+    return (imageId: string) => serviceHeld.has(imageId) || stackHeld.has(imageId);
+  }
+
+  /**
+   * Post-handoff compensation: restore files + pinned up, then probe before
+   * reporting restored_current / immediate_verified.
+   */
+  public async compensateWithCandidate(
+    generationId: string,
+    composeUp: (overridePath: string) => Promise<void>,
+  ): Promise<boolean> {
+    const row = this.get(generationId);
+    if (!row) return false;
+    try {
+      const context = await resolveComposeProjectContext(row.node_id, row.stack_name);
+      await context.restoreFromContext();
+      if (!row.override_path) {
+        throw new Error('Recovery generation has no override path');
+      }
+      await composeUp(row.override_path);
+      const probeOk = await this.probeRecoveredStack(
+        row.node_id,
+        row.stack_name,
+        row.services_json,
+      );
+      if (!probeOk) {
+        DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
+          status: 'recovery_required',
+        });
+        return false;
+      }
+      DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
+        status: 'restored_current',
+        phase: 'immediate_verified',
+        is_current: 1,
+        artifact_expires_at: null,
+      });
+      return true;
+    } catch (error) {
+      console.error(
+        '[StackUpdateRecovery] Compensation failed for %s: %s',
+        sanitizeForLog(generationId),
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+      DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
+        status: 'recovery_required',
+      });
+      return false;
+    }
+  }
+
+  /**
+   * Verify recovered runtime against the captured generation.
+   * Rejects absent, restarting, dead, exited, or unhealthy expected replicas,
+   * image-id mismatches vs capture, and any running replica of a scale-0 service.
+   */
+  public async probeRecoveredStack(
+    nodeId: number,
+    stackName: string,
+    servicesJson: string,
+  ): Promise<boolean> {
+    await new Promise((resolve) => setTimeout(resolve, RECOVERY_PROBE_DELAY_MS));
+    try {
+      const expected = parseServicesJson(servicesJson);
+      const expectedRunning = new Map<string, number>();
+      const expectedImageIds = new Map<string, Set<string>>();
+      const scaleZeroServices = new Set<string>();
+
+      for (const svc of expected) {
+        const imageIds = new Set<string>();
+        for (const replica of svc.replicas ?? []) {
+          if (replica.imageId?.trim()) imageIds.add(replica.imageId);
+        }
+        if (svc.scale > 0) {
+          // Fail closed when we cannot verify image identity for expected runners.
+          if (imageIds.size === 0) return false;
+          expectedRunning.set(svc.serviceName, svc.scale);
+          expectedImageIds.set(svc.serviceName, imageIds);
+        } else {
+          scaleZeroServices.add(svc.serviceName);
+        }
+      }
+
+      const docker = DockerController.getInstance(nodeId).getDocker();
+      const containers = await docker.listContainers({
+        all: true,
+        filters: { label: [`com.docker.compose.project=${stackName}`] },
+      });
+
+      if (expectedRunning.size > 0 && containers.length === 0) {
+        return false;
+      }
+
+      const runningByService = new Map<string, number>();
+      for (const containerInfo of containers) {
+        const labels = (containerInfo.Labels ?? {}) as Record<string, string>;
+        const serviceName = labels['com.docker.compose.service'];
+        const state = (containerInfo.State || '').toLowerCase();
+
+        if (state === 'restarting' || state === 'dead') {
+          return false;
+        }
+
+        if (state === 'exited' || state === 'created' || state === 'removing') {
+          if (serviceName && expectedRunning.has(serviceName)) {
+            return false;
+          }
+          continue;
+        }
+
+        if (state !== 'running') {
+          if (serviceName && expectedRunning.has(serviceName)) {
+            return false;
+          }
+          continue;
+        }
+
+        // Captured at scale 0 must stay stopped; any running replica fails the probe.
+        if (serviceName && scaleZeroServices.has(serviceName)) {
+          return false;
+        }
+
+        const inspectData = await docker.getContainer(containerInfo.Id).inspect();
+        const health = inspectData.State?.Health?.Status;
+        if (health === 'unhealthy') {
+          return false;
+        }
+
+        if (serviceName && expectedImageIds.has(serviceName)) {
+          const allowedIds = expectedImageIds.get(serviceName)!;
+          const actualImageId = typeof inspectData.Image === 'string' ? inspectData.Image : '';
+          if (!actualImageId || !allowedIds.has(actualImageId)) {
+            return false;
+          }
+        }
+
+        if (serviceName) {
+          runningByService.set(serviceName, (runningByService.get(serviceName) ?? 0) + 1);
+        }
+      }
+
+      for (const [serviceName, need] of expectedRunning) {
+        if ((runningByService.get(serviceName) ?? 0) < need) {
+          return false;
+        }
+      }
+      return true;
+    } catch (error) {
+      console.warn(
+        '[StackUpdateRecovery] Recovery probe failed for %s: %s',
+        sanitizeForLog(stackName),
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+      return false;
+    }
+  }
+
+  /**
+   * Idempotent removal of opaque tags and override files for a generation.
+   * Marks artifacts_retired only when every artifact is removed or confirmed absent,
+   * so transient failures remain retryable via reconcileIncomplete.
+   */
+  public async retireGenerationArtifacts(row: StackUpdateRecoveryGenerationRow): Promise<boolean> {
+    if (row.artifacts_retired === 1) return true;
+    const services = parseServicesJson(row.services_json);
+    const tagsOk = await this.removeRollbackTags(row.node_id, collectRollbackTags(services));
+    let overrideOk = true;
+    if (row.override_path) {
+      try {
+        await fs.unlink(row.override_path);
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+          overrideOk = false;
+          console.warn(
+            '[StackUpdateRecovery] Failed to delete override %s: %s',
+            sanitizeForLog(row.override_path),
+            sanitizeForLog(getErrorMessage(error, 'unknown')),
+          );
+        }
+      }
+    }
+    if (!tagsOk || !overrideOk) return false;
+    DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id);
+    return true;
+  }
+
+  /**
+   * Abandon lease-expired candidates, flag stuck post-handoff generations,
+   * and retire expired abandoned/superseded artifacts.
+   */
+  public async reconcileIncomplete(): Promise<void> {
+    if (!this.started) return;
+    try {
+      const db = DatabaseService.getInstance();
+      const now = Date.now();
+      let abandoned = 0;
+      for (const row of db.listStaleStackUpdateRecoveryCandidates(now)) {
+        if (await this.abandon(row.id)) abandoned += 1;
+      }
+      let flagged = 0;
+      for (const row of db.listStuckStackUpdateRecoveryGenerations(now)) {
+        db.updateStackUpdateRecoveryGeneration(row.id, {
+          status: 'recovery_required',
+          operation_lease_expires_at: null,
+        });
+        flagged += 1;
+      }
+      let retired = 0;
+      for (const row of db.listStackUpdateRecoveryGenerationsForArtifactRetirement(now)) {
+        // Never retire an active/current or recovery_required hold target.
+        if (row.is_current === 1 || row.status === 'recovery_required') continue;
+        if (await this.retireGenerationArtifacts(row)) retired += 1;
+      }
+      if (abandoned > 0 || flagged > 0 || retired > 0) {
+        console.log(
+          `[StackUpdateRecovery] Reconciled ${abandoned} stale candidate(s), `
+          + `${flagged} stuck generation(s), retired ${retired} artifact set(s)`,
+        );
+      }
+    } catch (error) {
+      console.error(
+        '[StackUpdateRecovery] Reconcile failed: %s',
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+    }
+  }
+
+  /** Returns true when every tag is removed or already absent. */
+  private async removeRollbackTags(nodeId: number, tags: string[]): Promise<boolean> {
+    if (tags.length === 0) return true;
+    try {
+      const docker = DockerController.getInstance(nodeId).getDocker();
+      let allOk = true;
+      for (const tag of tags) {
+        try {
+          await docker.getImage(tag).remove({ force: true });
+        } catch (error) {
+          const status = (error as { statusCode?: number }).statusCode;
+          const message = getErrorMessage(error, 'unknown').toLowerCase();
+          if (status === 404 || message.includes('no such image') || message.includes('not found')) {
+            continue;
+          }
+          allOk = false;
+          console.warn(
+            '[StackUpdateRecovery] Failed to remove rollback tag %s: %s',
+            sanitizeForLog(tag),
+            sanitizeForLog(getErrorMessage(error, 'unknown')),
+          );
+        }
+      }
+      return allOk;
+    } catch (error) {
+      console.warn(
+        '[StackUpdateRecovery] Docker unavailable while removing tags: %s',
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+      return false;
+    }
+  }
+
+  private async bestEffortRemoveTags(nodeId: number, tags: string[]): Promise<void> {
+    await this.removeRollbackTags(nodeId, tags);
+  }
+
+  private activeRecoveryTtlMs(): number {
+    return Math.max(getComposeCommandTimeoutMs(), this.readRecoveryWindowSeconds() * 1000);
+  }
+
+  private readRecoveryWindowSeconds(): number {
+    try {
+      const raw = parseInt(
+        DatabaseService.getInstance().getGlobalSettings()['health_gate_window_seconds'] ?? '',
+        10,
+      );
+      return Number.isFinite(raw) ? Math.max(raw, MIN_RECOVERY_WINDOW_SECONDS) : MIN_RECOVERY_WINDOW_SECONDS;
+    } catch (error) {
+      console.warn(
+        '[StackUpdateRecovery] Settings read failed; using default window: %s',
+        sanitizeForLog(getErrorMessage(error, 'unknown')),
+      );
+      return MIN_RECOVERY_WINDOW_SECONDS;
+    }
+  }
+}
+
+function splitOpaqueTag(tag: string): { repo: string; tagName: string } {
+  const lastColon = tag.lastIndexOf(':');
+  if (lastColon > 0) {
+    return { repo: tag.slice(0, lastColon), tagName: tag.slice(lastColon + 1) };
+  }
+  return { repo: tag, tagName: 'hold' };
+}

backend/src/services/WebhookService.ts

@@ -174,18 +174,24 @@ export class WebhookService {
                         case 'start':
                             await compose.runCommand(stackName, 'start');
                             break;
-                        case 'pull':
+                        case 'pull': {
                             await assertPolicyGateAllows(
                                 stackName,
                                 nodeId,
                                 buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
                             );
-                            await StackUpdateOrchestrator.getInstance().execute(
+                            const orchResult = await StackUpdateOrchestrator.getInstance().execute(
                                 { nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' },
                                 { atomic: atomic ?? false, terminalWs: null },
                             );
-                            HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
+                            const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
+                            const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
+                            if (recoveryId) {
+                                const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
+                                StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
+                            }
                             break;
+                        }
                     }
                 },
             );

backend/src/services/composeProjectContext.ts

@@ -0,0 +1,94 @@
+/**
+ * Thin Compose project context for safe full-stack updates.
+ *
+ * Wraps the current authored compose argument path and atomic file backup/restore.
+ * When a richer shared Compose project context lands, migrate callers to that type;
+ * this module must not become a competing full-manifest resolver.
+ */
+import path from 'path';
+import { randomUUID } from 'crypto';
+import { ComposeService } from './ComposeService';
+import { FileSystemService } from './FileSystemService';
+import { buildEffectiveServiceModel } from './effectiveServiceModel';
+import { getErrorMessage } from '../utils/errors';
+
+export type ImageReferenceKind = 'moving_tag' | 'digest_pinned' | 'none';
+
+const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i;
+
+export function classifyReferenceKind(declaredImageRef: string | null): ImageReferenceKind {
+  if (!declaredImageRef) return 'none';
+  if (DIGEST_PIN_PATTERN.test(declaredImageRef)) return 'digest_pinned';
+  return 'moving_tag';
+}
+
+async function requireRenderableModel(nodeId: number, stackName: string) {
+  const model = await buildEffectiveServiceModel(nodeId, stackName);
+  if (!model.renderable) {
+    throw new Error(model.error || 'Effective Compose model failed to render');
+  }
+  return model;
+}
+
+export interface ComposeProjectContext {
+  readonly nodeId: number;
+  readonly stackName: string;
+  readonly stackDir: string;
+  backupSlotId: string | null;
+  toComposeArgs(action: string[]): Promise<string[]>;
+  validateForMutation(): Promise<void>;
+  backupFromContext(operation: 'update' | 'deployment'): Promise<string>;
+  restoreFromContext(): Promise<void>;
+  resolveServiceImageMap(): Promise<Map<string, string | null>>;
+}
+
+class AuthoredComposeProjectContext implements ComposeProjectContext {
+  backupSlotId: string | null = null;
+
+  constructor(
+    readonly nodeId: number,
+    readonly stackName: string,
+    readonly stackDir: string,
+  ) {}
+
+  async toComposeArgs(action: string[]): Promise<string[]> {
+    return ComposeService.getInstance(this.nodeId).buildAuthoredComposeArgs(this.stackName, action);
+  }
+
+  async validateForMutation(): Promise<void> {
+    await ComposeService.getInstance(this.nodeId).validateStackForMutation(this.stackName);
+    await requireRenderableModel(this.nodeId, this.stackName);
+  }
+
+  async backupFromContext(_operation: 'update' | 'deployment'): Promise<string> {
+    await FileSystemService.getInstance(this.nodeId).backupStackFiles(this.stackName);
+    const slotId = randomUUID();
+    this.backupSlotId = slotId;
+    return slotId;
+  }
+
+  async restoreFromContext(): Promise<void> {
+    await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName);
+  }
+
+  async resolveServiceImageMap(): Promise<Map<string, string | null>> {
+    const model = await requireRenderableModel(this.nodeId, this.stackName);
+    const map = new Map<string, string | null>();
+    for (const svc of model.services) {
+      map.set(svc.name, svc.declaredImage);
+    }
+    return map;
+  }
+}
+
+export async function resolveComposeProjectContext(
+  nodeId: number,
+  stackName: string,
+): Promise<ComposeProjectContext> {
+  const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName);
+  return new AuthoredComposeProjectContext(nodeId, stackName, stackDir);
+}
+
+export function describeContextError(error: unknown): string {
+  return getErrorMessage(error, 'Compose project context failed');
+}

backend/src/services/updateGuard/readiness.ts

@@ -304,11 +304,11 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
   if (inputs.rollbackTarget === 'error') {
     items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The update preview is unavailable.' });
   } else if (inputs.rollbackTarget.target && inputs.rollbackTarget.moving) {
-    items.push({ id: 'previous_images', state: 'not_covered', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag, so restoring the compose and env files does not revert the image: the local tag still resolves to the newer digest. Pin every image to an immutable version tag for a true image rollback.` });
+    items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag. Full-stack updates capture the running image ID before pull/build and retain it through the recovery window so an immediate restore can retarget the exact prior image, not only the tag string.` });
   } else if (inputs.rollbackTarget.target) {
     items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Known rollback target: ${inputs.rollbackTarget.target}. The compose file pins an immutable tag, so restoring files also restores the image.` });
   } else {
-    items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. A rollback restores compose and env files; a moving tag may keep the newer image.' });
+    items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. When no prior image is recorded, a restore may fall back to compose and env files alone; a moving tag can still resolve to a newer digest.' });
   }
 
   if (inputs.lastDeployAt === 'error') {

backend/vitest.config.ts

@@ -5,7 +5,7 @@ export default defineConfig({
     environment: 'node',
     // Only run TypeScript sources - exclude the compiled dist/ output.
     include: ['src/__tests__/**/*.test.ts'],
-    exclude: ['dist/**', 'node_modules/**'],
+    exclude: ['dist/**', 'node_modules/**', 'src/__tests__/docker-integration/**'],
     // Build the baseline DB (schema + migrations + admin seed) once; each
     // test file's setupTestDb copies it instead of re-running migrations.
     globalSetup: ['./src/__tests__/helpers/vitestGlobalSetup.ts'],

backend/vitest.docker-integration.config.ts

@@ -0,0 +1,16 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    environment: 'node',
+    include: ['src/__tests__/docker-integration/**/*.test.ts'],
+    exclude: ['dist/**', 'node_modules/**'],
+    globalSetup: ['./src/__tests__/helpers/vitestGlobalSetup.ts'],
+    pool: 'forks',
+    maxWorkers: 1,
+    minWorkers: 1,
+    testTimeout: 180_000,
+    hookTimeout: 180_000,
+    sequence: { concurrent: false },
+  },
+});

frontend/src/components/EditorLayout/hooks/useStackActions.test.ts

@@ -19,6 +19,7 @@ vi.mock('@/components/ui/toast-store', () => ({
 }));
 
 import { apiFetch } from '@/lib/api';
+import { toast } from '@/components/ui/toast-store';
 
 type EditorState = ReturnType<typeof useEditorViewState>;
 type StackListState = ReturnType<typeof useStackListState>;
@@ -314,6 +315,18 @@ describe('useStackActions policy-block dialog wiring', () => {
     expect(overlayState.setPolicyBlock).not.toHaveBeenCalled();
   });
 
+  it('toasts a deletion-in-progress message when the conflict action is delete', async () => {
+    const inProgress = JSON.stringify({
+      code: 'stack_op_in_progress',
+      inProgress: { action: 'delete', startedAt: Date.now(), user: 'admin' },
+    });
+    vi.mocked(apiFetch).mockResolvedValueOnce(new Response(inProgress, { status: 409 }));
+    const { result, overlayState } = setup();
+    await act(async () => { await result.current.updateStack(mouseEvent); });
+    expect(overlayState.setPolicyBlock).not.toHaveBeenCalled();
+    expect(toast.error).toHaveBeenCalledWith(expect.stringMatching(/already deleting/i));
+  });
+
   it('opens the dialog with action "update" via the sidebar update entry point', async () => {
     vi.mocked(apiFetch).mockResolvedValueOnce(new Response(JSON.stringify(policyPayload), { status: 409 }));
     const { result, overlayState } = setup();

frontend/src/components/EditorLayout/hooks/useStackActions.ts

@@ -107,7 +107,7 @@ const parseFailureClassification = (value: unknown): FailureClassification | und
   return undefined;
 };
 
-type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update';
+type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'delete';
 
 interface StackOpInProgressInfo {
   action: StackOpAction;
@@ -122,6 +122,7 @@ const STACK_OP_PRESENT_PARTICIPLE: Record<StackOpAction, string> = {
   stop: 'stopping',
   start: 'starting',
   update: 'updating',
+  delete: 'deleting',
 };
 
 const VALID_STACK_OP_ACTIONS: ReadonlySet<string> = new Set(

frontend/src/components/EditorLayout/operation-phase.ts

@@ -38,7 +38,7 @@ export function classifyOperationPhase(rows: ParsedLogRow[], action: ActionVerb)
         if (stage === 'BUILD') {
             return 'Building images';
         }
-        if (message.includes('Backup created for atomic') || message.includes('Cleaning up existing containers')) {
+        if (message.includes('Backup created for atomic') || message.includes('Validating stack for update') || message.includes('Capturing rollback generation')) {
             return 'Preparing';
         }
     }