fix: keep running containers until stack pull/build succeeds (#1657)

* fix: keep running containers until stack pull/build succeeds

Acquire images before reconcile, capture a recovery generation for
compensation, and only remove classified orphans after handoff.

* fix: address recovery audit blockers for safe stack updates

Retire abandoned and expired recovery artifacts, probe compensated
runtimes before reporting rollback success, preserve local Docker when
deleting a node, validate the exact Compose invocation before capture,
and repair updateStack return-contract fixtures.

* fix: resolve ESLint errors blocking CI on this branch

Unused-import and unused-variable errors left over from the stack
deletion refactor: MeshService in stacks.ts (its opt-out cascade moved
into DeployedStackDeletionService), a redundant pruneVolumes
destructure in deleteDeployedStack (the real one is re-derived from
the same input object inside runDeletionBody), and an unused beforeAll
import in a Docker-integration test stub. Also scopes the webhook
pull-action case body in a block to satisfy no-case-declarations;
purely syntactic, no behavior change.

* fix: harden recovery probe, cleanup retry, and failed-pull Docker test

Reject absent or unhealthy expected replicas before reporting rollback
success, keep cleanup records until artifacts are actually removed, fail
closed when a mesh override cannot be generated, and assert a real
failed pull leaves the original container running.

* fix: verify recovery probe image identity and stack-scoped override paths

Reject recovered runtimes that use the wrong image or leave scale-zero
services running, and confine tombstone override deletion to the intent
stack directory so forged cross-stack paths cannot be swept.

* test: batch notification cap fixtures in a SQLite transaction

Unbatched 1200-row inserts were timing out at the default 30s under
CI load even though the same assertions pass in under 2s when green.

backend/src/__tests__/deployed-stack-deletion-service.test.ts

@@ -0,0 +1,156 @@
+/**
+ * Deployed-stack deletion: ready transaction retires both recovery models;
+ * blocking intents gate same-name create.
+ */
+import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
+import { randomUUID } from 'crypto';
+import path from 'path';
+import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
+import type {
+  StackUpdateRecoveryGenerationRow,
+  ServiceUpdateRecoveryRow,
+  StackUpdateCleanupPendingRow,
+} from '../services/DatabaseService';
+
+let tmpDir: string;
+let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
+let DeployedStackDeletionService: typeof import('../services/DeployedStackDeletionService').DeployedStackDeletionService;
+let overrideDeletionContainmentBase: typeof import('../services/DeployedStackDeletionService').overrideDeletionContainmentBase;
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ DatabaseService } = await import('../services/DatabaseService'));
+  ({ DeployedStackDeletionService, overrideDeletionContainmentBase } = await import('../services/DeployedStackDeletionService'));
+});
+
+afterAll(() => cleanupTestDb(tmpDir));
+
+function db() {
+  return DatabaseService.getInstance();
+}
+
+beforeEach(() => {
+  const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
+  raw.prepare('DELETE FROM stack_update_recovery_generations').run();
+  raw.prepare('DELETE FROM service_update_recovery').run();
+  raw.prepare('DELETE FROM stack_update_cleanup_pending').run();
+});
+
+const NODE = 1;
+
+describe('DeployedStackDeletionService ready transaction', () => {
+  it('commitStackDeletionReadyTransaction deletes full-stack and service recovery rows', () => {
+    const now = Date.now();
+    const stackName = 'del-stack';
+    const gen: StackUpdateRecoveryGenerationRow = {
+      id: randomUUID(),
+      node_id: NODE,
+      stack_name: stackName,
+      status: 'active',
+      phase: 'immediate_verified',
+      is_current: 1,
+      backup_slot_id: null,
+      override_path: null,
+      services_json: '[]',
+      health_gate_id: null,
+      gate_retain_until: null,
+      artifact_expires_at: null,
+      operation_lease_expires_at: null,
+      created_at: now,
+      updated_at: now,
+      created_by: null,
+      artifacts_retired: 0,
+    };
+    db().insertStackUpdateRecoveryGeneration(gen);
+    const svc: ServiceUpdateRecoveryRow = {
+      id: randomUUID(),
+      node_id: NODE,
+      stack_name: stackName,
+      service_name: 'web',
+      replicas_json: '[]',
+      majority_image_id: 'sha256:abc',
+      declared_image_ref: 'nginx:latest',
+      weak_floating_tag: 0,
+      health_gate_id: null,
+      status: 'active',
+      expires_at: now + 60_000,
+      claim_expires_at: null,
+      created_at: now,
+      created_by: null,
+    };
+    db().insertServiceUpdateRecovery(svc);
+    const intentId = randomUUID();
+    const intent: StackUpdateCleanupPendingRow = {
+      id: intentId,
+      node_id: NODE,
+      stack_name: stackName,
+      status: 'prepared',
+      target_kind: 'local_socket',
+      rollback_tags_json: '[]',
+      override_paths_json: '[]',
+      prune_volumes_requested: 0,
+      created_at: now,
+      updated_at: now,
+    };
+    db().insertCleanupPending(intent);
+
+    expect(db().commitStackDeletionReadyTransaction(intentId, NODE, stackName)).toBe(true);
+    expect(db().listStackUpdateRecoveryForStack(NODE, stackName)).toHaveLength(0);
+    expect(db().listActiveServiceUpdateRecoveries(NODE, stackName, 'web')).toHaveLength(0);
+    expect(db().getCleanupPending(intentId)?.status).toBe('ready');
+  });
+
+  it('hasBlockingDeletionIntent is true for prepared intents', () => {
+    const now = Date.now();
+    db().insertCleanupPending({
+      id: randomUUID(),
+      node_id: NODE,
+      stack_name: 'blocked',
+      status: 'prepared',
+      target_kind: 'local_socket',
+      rollback_tags_json: '[]',
+      override_paths_json: '[]',
+      prune_volumes_requested: 0,
+      created_at: now,
+      updated_at: now,
+    });
+    expect(db().hasBlockingDeletionIntent(NODE, 'blocked')).toBe(true);
+    expect(db().hasBlockingDeletionIntent(NODE, 'other')).toBe(false);
+  });
+
+  it('assertNoBlockingDeletionIntent throws for prepared stacks', () => {
+    const now = Date.now();
+    db().insertCleanupPending({
+      id: randomUUID(),
+      node_id: NODE,
+      stack_name: 'prep',
+      status: 'prepared',
+      target_kind: 'local_socket',
+      rollback_tags_json: '[]',
+      override_paths_json: '[]',
+      prune_volumes_requested: 0,
+      created_at: now,
+      updated_at: now,
+    });
+    expect(() => {
+      DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(NODE, 'prep');
+    }).toThrow(/deletion in progress/i);
+  });
+});
+
+describe('overrideDeletionContainmentBase', () => {
+  it('confines stack-scoped intents to the stack directory', () => {
+    expect(overrideDeletionContainmentBase('/app/compose', 'my-stack')).toBe(
+      path.resolve('/app/compose', 'my-stack'),
+    );
+    expect(overrideDeletionContainmentBase('/app/compose', null)).toBe(
+      path.resolve('/app/compose'),
+    );
+  });
+
+  it('rejects invalid stack names that could traverse', () => {
+    expect(overrideDeletionContainmentBase('/app/compose', '../other')).toBeNull();
+    expect(overrideDeletionContainmentBase('/app/compose', 'bad/name')).toBeNull();
+  });
+});
+