fix: keep running containers until stack pull/build succeeds (#1657)

* fix: keep running containers until stack pull/build succeeds Acquire images before reconcile, capture a recovery generation for compensation, and only remove classified orphans after handoff. * fix: address recovery audit blockers for safe stack updates Retire abandoned and expired recovery artifacts, probe compensated runtimes before reporting rollback success, preserve local Docker when deleting a node, validate the exact Compose invocation before capture, and repair updateStack return-contract fixtures. * fix: resolve ESLint errors blocking CI on this branch Unused-import and unused-variable errors left over from the stack deletion refactor: MeshService in stacks.ts (its opt-out cascade moved into DeployedStackDeletionService), a redundant pruneVolumes destructure in deleteDeployedStack (the real one is re-derived from the same input object inside runDeletionBody), and an unused beforeAll import in a Docker-integration test stub. Also scopes the webhook pull-action case body in a block to satisfy no-case-declarations; purely syntactic, no behavior change. * fix: harden recovery probe, cleanup retry, and failed-pull Docker test Reject absent or unhealthy expected replicas before reporting rollback success, keep cleanup records until artifacts are actually removed, fail closed when a mesh override cannot be generated, and assert a real failed pull leaves the original container running. * fix: verify recovery probe image identity and stack-scoped override paths Reject recovered runtimes that use the wrong image or leave scale-zero services running, and confine tombstone override deletion to the intent stack directory so forged cross-stack paths cannot be swept. * test: batch notification cap fixtures in a SQLite transaction Unbatched 1200-row inserts were timing out at the default 30s under CI load even though the same assertions pass in under 2s when green.
2026-08-21 07:36:40 +00:00 · 2026-07-21 12:18:01 -04:00
parent b1decbb32a
commit 3f1f15a6f4
41 changed files with 3087 additions and 244 deletions
@@ -0,0 +1,112 @@
+/**
+ * Docker-backed integration: a failed pull must leave the running stack untouched.
+ * Skipped automatically when Docker is unavailable.
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { execFileSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb';
+
+function dockerAvailable(): boolean {
+  try {
+    execFileSync('docker', ['info'], { stdio: 'ignore' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+const hasDocker = dockerAvailable();
+const STACK = 'fpkeep';
+
+function compose(args: string[], cwd: string): string {
+  return execFileSync('docker', ['compose', '-p', STACK, ...args], {
+    cwd,
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+}
+
+describe.skipIf(!hasDocker)('failed pull keeps stack running', () => {
+  let tmpDir: string;
+  let composeDir: string;
+  let stackDir: string;
+  let nodeId: number;
+
+  beforeAll(async () => {
+    tmpDir = await setupTestDb();
+    composeDir = process.env.COMPOSE_DIR!;
+    stackDir = path.join(composeDir, STACK);
+    fs.mkdirSync(stackDir, { recursive: true });
+
+    const { DatabaseService } = await import('../../services/DatabaseService');
+    const db = DatabaseService.getInstance();
+    const local = db.getDefaultNode();
+    if (!local?.id) throw new Error('Test DB has no default local node');
+    nodeId = local.id;
+    db.updateGlobalSetting('prune_on_update', '0');
+
+    fs.writeFileSync(
+      path.join(stackDir, 'compose.yaml'),
+      [
+        'services:',
+        '  web:',
+        '    image: busybox:1.36.1',
+        '    command: ["sleep", "3600"]',
+        '',
+      ].join('\n'),
+      'utf8',
+    );
+
+    compose(['up', '-d', '--pull', 'always'], stackDir);
+  }, 180_000);
+
+  afterAll(async () => {
+    try {
+      if (stackDir && fs.existsSync(stackDir)) {
+        compose(['down', '--remove-orphans'], stackDir);
+      }
+    } catch {
+      // Best-effort cleanup.
+    }
+    if (tmpDir) cleanupTestDb(tmpDir);
+  }, 120_000);
+
+  it('leaves the original container running when ComposeService updateStack pull fails', async () => {
+    const beforeId = compose(['ps', '-q'], stackDir).trim();
+    expect(beforeId.length).toBeGreaterThan(0);
+
+    const beforeInspect = JSON.parse(
+      execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
+    ) as Array<{ State: { Running: boolean; Status: string } }>;
+    expect(beforeInspect[0].State.Running).toBe(true);
+
+    fs.writeFileSync(
+      path.join(stackDir, 'compose.yaml'),
+      [
+        'services:',
+        '  web:',
+        '    image: busybox:sencho-does-not-exist-fpkeep-xyz',
+        '    command: ["sleep", "3600"]',
+        '',
+      ].join('\n'),
+      'utf8',
+    );
+
+    const { StackUpdateRecoveryService } = await import('../../services/StackUpdateRecoveryService');
+    StackUpdateRecoveryService.resetForTests();
+    const { ComposeService } = await import('../../services/ComposeService');
+    await expect(
+      ComposeService.getInstance(nodeId).updateStack(STACK, undefined, true),
+    ).rejects.toThrow();
+
+    const afterId = compose(['ps', '-q'], stackDir).trim();
+    expect(afterId).toBe(beforeId);
+
+    const afterInspect = JSON.parse(
+      execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
+    ) as Array<{ State: { Running: boolean } }>;
+    expect(afterInspect[0].State.Running).toBe(true);
+  }, 180_000);
+});