fix: keep running containers until stack pull/build succeeds (#1657)

* fix: keep running containers until stack pull/build succeeds

Acquire images before reconcile, capture a recovery generation for
compensation, and only remove classified orphans after handoff.

* fix: address recovery audit blockers for safe stack updates

Retire abandoned and expired recovery artifacts, probe compensated
runtimes before reporting rollback success, preserve local Docker when
deleting a node, validate the exact Compose invocation before capture,
and repair updateStack return-contract fixtures.

* fix: resolve ESLint errors blocking CI on this branch

Unused-import and unused-variable errors left over from the stack
deletion refactor: MeshService in stacks.ts (its opt-out cascade moved
into DeployedStackDeletionService), a redundant pruneVolumes
destructure in deleteDeployedStack (the real one is re-derived from
the same input object inside runDeletionBody), and an unused beforeAll
import in a Docker-integration test stub. Also scopes the webhook
pull-action case body in a block to satisfy no-case-declarations;
purely syntactic, no behavior change.

* fix: harden recovery probe, cleanup retry, and failed-pull Docker test

Reject absent or unhealthy expected replicas before reporting rollback
success, keep cleanup records until artifacts are actually removed, fail
closed when a mesh override cannot be generated, and assert a real
failed pull leaves the original container running.

* fix: verify recovery probe image identity and stack-scoped override paths

Reject recovered runtimes that use the wrong image or leave scale-zero
services running, and confine tombstone override deletion to the intent
stack directory so forged cross-stack paths cannot be swept.

* test: batch notification cap fixtures in a SQLite transaction

Unbatched 1200-row inserts were timing out at the default 30s under
CI load even though the same assertions pass in under 2s when green.

backend/src/__tests__/database-metrics.test.ts

@@ -225,23 +225,25 @@ describe('DatabaseService - notification history cap (periodic)', () => {
 
     // A chatty stack writes 600 events.
     const base = Date.now();
-    for (let i = 0; i < 600; i++) {
-      db.addNotificationHistory(0, {
-        level: 'info',
-        message: `chatty-${i}`,
-        timestamp: base + i,
-        stack_name: 'chatty',
-      });
-    }
-    // A quiet stack writes 3 events long before the chatty burst.
-    for (let i = 0; i < 3; i++) {
-      db.addNotificationHistory(0, {
-        level: 'info',
-        message: `quiet-${i}`,
-        timestamp: base - 10_000 + i,
-        stack_name: 'quiet',
-      });
-    }
+    db.transaction(() => {
+      for (let i = 0; i < 600; i++) {
+        db.addNotificationHistory(0, {
+          level: 'info',
+          message: `chatty-${i}`,
+          timestamp: base + i,
+          stack_name: 'chatty',
+        });
+      }
+      // A quiet stack writes 3 events long before the chatty burst.
+      for (let i = 0; i < 3; i++) {
+        db.addNotificationHistory(0, {
+          level: 'info',
+          message: `quiet-${i}`,
+          timestamp: base - 10_000 + i,
+          stack_name: 'quiet',
+        });
+      }
+    });
 
     // No per-insert prune: every row is present.
     const beforeCleanup = db.getNotificationHistory(0, 2000);
@@ -261,13 +263,15 @@ describe('DatabaseService - notification history cap (periodic)', () => {
     db.deleteAllNotifications(0);
 
     const base = Date.now();
-    for (let i = 0; i < 1200; i++) {
-      db.addNotificationHistory(0, {
-        level: 'info',
-        message: `system-${i}`,
-        timestamp: base + i,
-      });
-    }
+    db.transaction(() => {
+      for (let i = 0; i < 1200; i++) {
+        db.addNotificationHistory(0, {
+          level: 'info',
+          message: `system-${i}`,
+          timestamp: base + i,
+        });
+      }
+    });
 
     db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
 
@@ -279,14 +283,16 @@ describe('DatabaseService - notification history cap (periodic)', () => {
   it('keeps the newest entries per (node, stack) after periodic cap', () => {
     db.deleteAllNotifications(0);
     const base = Date.now();
-    for (let i = 0; i < 600; i++) {
-      db.addNotificationHistory(0, {
-        level: 'info',
-        message: `ordered-${i}`,
-        timestamp: base + i * 10,
-        stack_name: 'ordered',
-      });
-    }
+    db.transaction(() => {
+      for (let i = 0; i < 600; i++) {
+        db.addNotificationHistory(0, {
+          level: 'info',
+          message: `ordered-${i}`,
+          timestamp: base + i * 10,
+          stack_name: 'ordered',
+        });
+      }
+    });
 
     db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
 
@@ -301,9 +307,11 @@ describe('DatabaseService - notification history cap (periodic)', () => {
   it('uses safe defaults when called with only the retention argument', () => {
     db.deleteAllNotifications(0);
     const base = Date.now();
-    for (let i = 0; i < 600; i++) {
-      db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
-    }
+    db.transaction(() => {
+      for (let i = 0; i < 600; i++) {
+        db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
+      }
+    });
     // Production caller (MonitorService) only passes daysToKeep; the cap defaults must enforce the per-stack 500 limit.
     const summary = db.cleanupOldNotifications(30);
     const after = db.getNotificationHistory(0, 2000).filter((n: any) => n.stack_name === 'default');