fix(drift): resolve image tags via effective compose model (#1574)

Drift compared raw compose YAML to runtime, so ${VAR:-default} image
expressions false-positive as image-mismatch. Source the declared side
from docker compose config instead, matching deploy-time resolution.

Fixes #1572

backend/src/__tests__/drift-detection.test.ts

@@ -1,7 +1,7 @@
 /**
  * Unit tests for the spatial drift engine: per-finding and per-status diff
  * behaviour of assembleStackDrift, image-reference normalization, and the
- * fail-soft boundaries of buildStackDriftReport (compose read failure → drifted,
+ * fail-soft boundaries of buildStackDriftReport (render failure → drifted,
  * Docker failure → unreachable).
  */
 import { describe, it, expect, vi } from 'vitest';
@@ -13,7 +13,11 @@ import {
 import DockerController from '../services/DockerController';
 import type { DependencyContainer, DependencyNetwork, DependencySnapshot } from '../services/DockerController';
 import { FileSystemService } from '../services/FileSystemService';
+import { ComposeService } from '../services/ComposeService';
+import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompose';
 import type { DeclaredCompose, DeclaredService, DeclaredPort } from '../helpers/composeDependencyParse';
+import type { EffectiveModel, EffService } from '../services/preflight/effectiveModel';
+import { fromDeclaredCompose, fromEffectiveModel } from '../services/network/normalize';
 
 // ── builders ────────────────────────────────────────────────────────────
 
@@ -38,6 +42,37 @@ function container(p: Partial<DependencyContainer> & { id: string }): Dependency
 
 const findingKinds = (r: { findings: { kind: string }[] }): string[] => r.findings.map((f) => f.kind).sort();
 
+function effSvc(over: Partial<EffService> = {}): EffService {
+  return {
+    name: 'web', image: 'nginx:1.27', ports: [], binds: [], namedVolumes: [], storageMounts: [],
+    privileged: false, hasHealthcheck: true, restart: 'unless-stopped', envKeys: [],
+    networks: [], extraHosts: [], labelKeys: [], ...over,
+  };
+}
+
+function stubDockerRender(
+  rendered: object | null,
+  stderr = '',
+) {
+  vi.spyOn(ComposeService, 'getInstance').mockReturnValue({
+    renderConfig: vi.fn().mockResolvedValue({
+      rendered: rendered === null ? null : JSON.stringify(rendered),
+      stderr,
+      code: rendered === null ? 1 : 0,
+      timedOut: false,
+    }),
+  } as unknown as ComposeService);
+}
+
+function stubFsAndSnapshot(snapshot: DependencySnapshot) {
+  vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
+    getStacks: vi.fn().mockResolvedValue(['app']),
+  } as unknown as FileSystemService);
+  vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+    getDependencySnapshot: vi.fn().mockResolvedValue(snapshot),
+  } as unknown as DockerController);
+}
+
 // ── assembleStackDrift: statuses ──────────────────────────────────────────
 
 describe('assembleStackDrift - status', () => {
@@ -143,6 +178,17 @@ describe('assembleStackDrift - findings', () => {
     expect(report.findings).toEqual([]);
   });
 
+  it('reports in-sync when declared image matches a resolved default tag (#1572)', () => {
+    const image = 'ghcr.io/karakeep-app/karakeep:release';
+    const report = assembleStackDrift({
+      stack: 'karakeep',
+      declared: declared([service({ name: 'karakeep', image })]),
+      containers: [container({ id: 'c1', service: 'karakeep', image })],
+    });
+    expect(report.status).toBe('in-sync');
+    expect(report.findings).toEqual([]);
+  });
+
   it('skips the image check for a build-only service (no declared image)', () => {
     const report = assembleStackDrift({
       stack: 'app',
@@ -366,6 +412,40 @@ describe('assembleStackDrift - network drift', () => {
   });
 });
 
+// ── declaredFromEffectiveModel ─────────────────────────────────────────────
+
+describe('declaredFromEffectiveModel', () => {
+  it('maps resolved images and ports from the effective model', () => {
+    const model: EffectiveModel = {
+      projectName: 'app',
+      services: [effSvc({
+        name: 'karakeep',
+        image: 'ghcr.io/karakeep-app/karakeep:release',
+        ports: [{ startPort: 8080, endPort: 8082, hostIp: '127.0.0.1', protocol: 'tcp' }],
+      })],
+      networks: { default: { name: 'app_default', external: false, internal: false } },
+      volumes: {},
+    };
+    const converted = declaredFromEffectiveModel(model);
+    expect(converted.projectName).toBe('app');
+    expect(converted.services[0].image).toBe('ghcr.io/karakeep-app/karakeep:release');
+    expect(converted.services[0].ports).toEqual([{ hostIp: '127.0.0.1', publishedPort: 8080, protocol: 'tcp' }]);
+  });
+
+  it('normalizes networks so drift matches the rendered model', () => {
+    const model: EffectiveModel = {
+      projectName: 'myapp',
+      services: [effSvc({ name: 'web', networks: [{ key: 'backend', aliases: [] }, { key: 'shared', aliases: [] }] })],
+      networks: {
+        backend: { name: 'myapp_backend', external: false, internal: false },
+        shared: { name: 'shared_net', external: true, internal: false },
+      },
+      volumes: {},
+    };
+    expect(fromDeclaredCompose(declaredFromEffectiveModel(model), 'myapp')).toEqual(fromEffectiveModel(model));
+  });
+});
+
 // ── normalizeImageRef ─────────────────────────────────────────────────────
 
 describe('normalizeImageRef', () => {
@@ -391,8 +471,8 @@ describe('normalizeImageRef', () => {
 
 describe('buildStackDriftReport - boundaries', () => {
   it('reports unreachable when the Docker snapshot fails', async () => {
+    stubDockerRender({ name: 'app', services: { web: { image: 'nginx:1.25' } } });
     vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
-      getStackContent: vi.fn().mockResolvedValue('services:\n  web:\n    image: nginx:1.25\n'),
       getStacks: vi.fn().mockResolvedValue(['app']),
     } as unknown as FileSystemService);
     vi.spyOn(DockerController, 'getInstance').mockReturnValue({
@@ -401,20 +481,29 @@ describe('buildStackDriftReport - boundaries', () => {
 
     const report = await buildStackDriftReport(0, 'app');
     expect(report.status).toBe('unreachable');
+    expect(report.hasComposeFile).toBe(true);
     expect(report.findings).toEqual([]);
     vi.restoreAllMocks();
   });
 
-  it('reports drifted with a parseError when the compose file cannot be read', async () => {
-    vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
-      getStackContent: vi.fn().mockRejectedValue(new Error('ENOENT')),
-      getStacks: vi.fn().mockResolvedValue(['app']),
-    } as unknown as FileSystemService);
+  it('reports drifted with a parseError when the effective model cannot be rendered', async () => {
+    stubDockerRender(null, 'invalid compose');
 
     const report = await buildStackDriftReport(0, 'app');
     expect(report.status).toBe('drifted');
     expect(report.hasComposeFile).toBe(false);
-    expect(report.parseError).toBe('ENOENT');
+    expect(report.parseError).toContain('could not render');
+    expect(report.findings).toEqual([]);
+    vi.restoreAllMocks();
+  });
+
+  it('names missing required variables when render fails for unset interpolation', async () => {
+    stubDockerRender(null, 'required variable REQ is missing a value: must be provided');
+
+    const report = await buildStackDriftReport(0, 'app');
+    expect(report.status).toBe('drifted');
+    expect(report.parseError).toContain('REQ');
+    expect(report.parseError).toContain('no value');
     vi.restoreAllMocks();
   });
 
@@ -424,13 +513,8 @@ describe('buildStackDriftReport - boundaries', () => {
       networks: [],
       volumes: [],
     };
-    vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
-      getStackContent: vi.fn().mockResolvedValue('services:\n  web:\n    image: nginx:1.25\n'),
-      getStacks: vi.fn().mockResolvedValue(['app']),
-    } as unknown as FileSystemService);
-    vi.spyOn(DockerController, 'getInstance').mockReturnValue({
-      getDependencySnapshot: vi.fn().mockResolvedValue(snapshot),
-    } as unknown as DockerController);
+    stubDockerRender({ name: 'app', services: { web: { image: 'nginx:1.25' } } });
+    stubFsAndSnapshot(snapshot);
 
     const report = await buildStackDriftReport(0, 'app');
     expect(report.status).toBe('drifted');
@@ -438,19 +522,34 @@ describe('buildStackDriftReport - boundaries', () => {
     vi.restoreAllMocks();
   });
 
+  it('reports in-sync when render resolves a default image tag (#1572)', async () => {
+    const image = 'ghcr.io/karakeep-app/karakeep:release';
+    const snapshot: DependencySnapshot = {
+      containers: [container({ id: 'c1', service: 'karakeep', stack: 'app', image })],
+      networks: [],
+      volumes: [],
+    };
+    stubDockerRender({ name: 'app', services: { karakeep: { image } } });
+    stubFsAndSnapshot(snapshot);
+
+    const report = await buildStackDriftReport(0, 'app');
+    expect(report.status).toBe('in-sync');
+    expect(report.findings).toEqual([]);
+    vi.restoreAllMocks();
+  });
+
   it('threads the snapshot networks through into a network-undeclared finding', async () => {
     const snapshot: DependencySnapshot = {
       containers: [container({ id: 'c1', service: 'web', stack: 'app', image: 'nginx:1.25', networks: [{ name: 'app_rogue', id: 'r', ip: '' }] })],
       networks: [depNet('app_rogue')],
       volumes: [],
     };
-    vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({
-      getStackContent: vi.fn().mockResolvedValue('services:\n  web:\n    image: nginx:1.25\n'),
-      getStacks: vi.fn().mockResolvedValue(['app']),
-    } as unknown as FileSystemService);
-    vi.spyOn(DockerController, 'getInstance').mockReturnValue({
-      getDependencySnapshot: vi.fn().mockResolvedValue(snapshot),
-    } as unknown as DockerController);
+    stubDockerRender({
+      name: 'app',
+      services: { web: { image: 'nginx:1.25', networks: ['default'] } },
+      networks: { default: { name: 'app_default' } },
+    });
+    stubFsAndSnapshot(snapshot);
 
     const report = await buildStackDriftReport(0, 'app');
     expect(findingKinds(report)).toContain('network-undeclared');

backend/src/helpers/effectiveToDeclaredCompose.ts

@@ -0,0 +1,52 @@
+import type { EffectiveModel, EffResource } from '../services/preflight/effectiveModel';
+import type { DeclaredCompose, DeclaredPort, DeclaredResource, DeclaredService } from './composeDependencyParse';
+
+/** Map one rendered network/volume entry to the declared shape drift expects. */
+function toDeclaredResource(key: string, res: EffResource, projectName: string): DeclaredResource {
+  const defaultName = res.external ? key : `${projectName}_${key}`;
+  if (res.name === defaultName) {
+    return { external: res.external };
+  }
+  return { external: res.external, name: res.name };
+}
+
+function toDeclaredResourceRecord(
+  resources: Record<string, EffResource>,
+  projectName: string,
+): Record<string, DeclaredResource> {
+  return Object.fromEntries(
+    Object.entries(resources).map(([key, res]) => [key, toDeclaredResource(key, res, projectName)]),
+  );
+}
+
+/**
+ * Convert a fully-rendered effective model (`docker compose config`) into the
+ * {@link DeclaredCompose} shape consumed by the spatial drift engine. Variable
+ * substitution, defaults, multi-file merge, and profile filtering are already
+ * applied upstream.
+ */
+export function declaredFromEffectiveModel(model: EffectiveModel): DeclaredCompose {
+  const { projectName } = model;
+  const services: DeclaredService[] = model.services.map((s) => ({
+    name: s.name,
+    dependsOn: [],
+    networks: s.networks.map((n) => n.key),
+    volumes: [],
+    ports: s.ports.map(
+      (p): DeclaredPort => ({
+        hostIp: p.hostIp,
+        publishedPort: p.startPort,
+        protocol: p.protocol,
+      }),
+    ),
+    image: s.image,
+    networkMode: s.networkMode,
+  }));
+
+  return {
+    services,
+    networks: toDeclaredResourceRecord(model.networks, projectName),
+    volumes: toDeclaredResourceRecord(model.volumes, projectName),
+    projectName,
+  };
+}

backend/src/services/DriftDetectionService.ts

@@ -1,16 +1,20 @@
 import DockerController from './DockerController';
 import type { DependencyContainer, DependencyNetwork } from './DockerController';
 import { FileSystemService } from './FileSystemService';
-import { parseComposeDependencies } from '../helpers/composeDependencyParse';
+import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompose';
 import type { DeclaredCompose, DeclaredService } from '../helpers/composeDependencyParse';
+import { parseMissingRequiredVars } from '../helpers/envVarParse';
+import { parseEffectiveModel } from './preflight/effectiveModel';
 import { compareStackNetworks, fromDeclaredCompose } from './network/normalize';
-import { sanitizeForLog } from '../utils/safeLog';
+import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
 import { getErrorMessage } from '../utils/errors';
 
+const MAX_RENDER_ERROR = 600;
+
 /**
- * Spatial drift engine: compares a stack's on-disk compose model against the
- * live Docker runtime and reports where the two diverge. Read-only and
- * stateless. It does NOT persist findings, track change-over-time, or compare
+ * Spatial drift engine: compares a stack's effective compose model (from
+ * `docker compose config`) against the live Docker runtime and reports where
+ * the two diverge. Read-only and stateless. It does NOT persist findings, track change-over-time, or compare
  * against a last-applied hash: temporal drift ("the file changed since you
  * deployed"), env-key/value comparison, the cross-fleet rollup, and
  * unknown-source (orphan containers with no on-disk stack) belong to the
@@ -113,7 +117,10 @@ export interface AssembleStackDriftInput {
 /** Add a network to a service's accumulated undeclared-attachment set. */
 function addNetwork(map: Map<string, Set<string>>, service: string, network: string): void {
   let set = map.get(service);
-  if (!set) { set = new Set(); map.set(service, set); }
+  if (!set) {
+    set = new Set();
+    map.set(service, set);
+  }
   set.add(network);
 }
 
@@ -276,32 +283,70 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
   return { stack, status, hasComposeFile: true, hasContainers, findings };
 }
 
+type RenderDeclaredResult =
+  | { ok: true; declared: DeclaredCompose }
+  | { ok: false; parseError: string };
+
+async function renderDeclaredCompose(nodeId: number, stackName: string): Promise<RenderDeclaredResult> {
+  try {
+    const { ComposeService } = await import('./ComposeService');
+    const result = await ComposeService.getInstance(nodeId).renderConfig(stackName);
+    if (result.rendered === null) {
+      const missing = parseMissingRequiredVars(result.stderr);
+      return {
+        ok: false,
+        parseError: missing.length
+          ? `Required variable${missing.length > 1 ? 's' : ''} ${missing.join(', ')} ${missing.length > 1 ? 'have' : 'has'} no value, so the effective model cannot be rendered.`
+          : 'Sencho could not render the effective Compose model. Check the compose and env files for a YAML syntax error, an unresolved include or merge, or a required variable with no value.',
+      };
+    }
+    try {
+      const model = parseEffectiveModel(JSON.parse(result.rendered), stackName);
+      return { ok: true, declared: declaredFromEffectiveModel(model) };
+    } catch (parseErr) {
+      console.warn('[Drift] Effective model parse failed for %s:',
+        sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(parseErr, 'unknown')));
+      return { ok: false, parseError: 'Sencho could not parse the rendered Compose model.' };
+    }
+  } catch (err) {
+    console.error('[Drift] Effective model render failed for stack %s:',
+      sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'render failed')));
+    const message = redactSensitiveText(getErrorMessage(err, 'docker compose could not be started.'))
+      .slice(0, MAX_RENDER_ERROR)
+      .trim();
+    return { ok: false, parseError: message || 'Sencho could not run docker compose on this node.' };
+  }
+}
+
 /**
- * Builds the drift report for one stack on one node: reads the compose file,
- * takes a Docker snapshot, and diffs them. Fails closed at each boundary: a
- * compose read failure is reported as a parse error (drifted, never in-sync),
- * and a Docker failure is reported as 'unreachable' rather than crashing.
+ * Builds the drift report for one stack on one node: renders the effective
+ * compose model, takes a Docker snapshot, and diffs them. Fails closed at each
+ * boundary: a render failure is reported as a parse error (drifted, never
+ * in-sync), and a Docker failure is reported as 'unreachable' rather than crashing.
  */
 export async function buildStackDriftReport(nodeId: number, stackName: string): Promise<StackDriftReport> {
   const fs = FileSystemService.getInstance(nodeId);
+  const render = await renderDeclaredCompose(nodeId, stackName);
 
-  let content: string;
-  try {
-    content = await fs.getStackContent(stackName);
-  } catch (error) {
-    console.error('[Drift] Failed to read compose for stack %s:', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'read failed')));
+  if (!render.ok) {
+    let hasContainers = false;
+    try {
+      const stacks = await fs.getStacks();
+      const snapshot = await DockerController.getInstance(nodeId).getDependencySnapshot(stacks);
+      hasContainers = snapshot.containers.some((c) => c.stack === stackName && RUNNING_STATES.has(c.state));
+    } catch {
+      // Docker unreachable: hasContainers stays false; render error is still the headline.
+    }
     return {
       stack: stackName,
       status: 'drifted',
       hasComposeFile: false,
-      hasContainers: false,
+      hasContainers,
       findings: [],
-      parseError: getErrorMessage(error, 'Failed to read compose file'),
+      parseError: render.parseError,
     };
   }
 
-  const declared = parseComposeDependencies(content);
-
   let containers: DependencyContainer[];
   let networks: DependencyNetwork[] = [];
   try {
@@ -313,18 +358,15 @@ export async function buildStackDriftReport(nodeId: number, stackName: string):
     containers = snapshot.containers.filter((c) => c.stack === stackName);
     networks = snapshot.networks;
   } catch (error) {
-    // Docker is unreachable, so runtime drift cannot be assessed. The headline
-    // failure is reachability; a separate parse error (if any) surfaces as
-    // drifted once Docker is back, keeping the parseError-implies-drifted invariant.
     console.error('[Drift] Docker snapshot failed for stack %s:', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'snapshot failed')));
     return {
       stack: stackName,
       status: 'unreachable',
-      hasComposeFile: !declared.parseError,
+      hasComposeFile: true,
       hasContainers: false,
       findings: [],
     };
   }
 
-  return assembleStackDrift({ stack: stackName, declared, containers, networks, parseError: declared.parseError });
+  return assembleStackDrift({ stack: stackName, declared: render.declared, containers, networks });
 }