sencho/backend/src/__tests__/blueprints.test.ts

/**
 * BlueprintReconciler decision-logic tests.
 *
 * The reconciler's `computeDecision` is the load-bearing pure logic. Given
 * a blueprint, an actual deployment table, and a desired node set, it must
 * decide for each node whether to deploy, withdraw, drift-check, state-review,
 * or evict-block. We test that decision in isolation by accessing the
 * private method via a type-cast, mirroring the AutoHealService.shouldHeal
 * pattern.
 *
 * Local deploy / remote HTTP / actual `docker compose` invocation are not
 * exercised here; they're integration concerns covered by the manual
 * lifecycle in the plan.
 */
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import type { Blueprint, Node } from '../services/DatabaseService';
import type { ReconcileDecision } from '../services/BlueprintReconciler';

type ReconcilerWithCompute = { computeDecision: (blueprint: Blueprint, allNodes: Node[]) => ReconcileDecision };

let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let BlueprintReconciler: typeof import('../services/BlueprintReconciler').BlueprintReconciler;
let NodeLabelService: typeof import('../services/NodeLabelService').NodeLabelService;
let BlueprintService: typeof import('../services/BlueprintService').BlueprintService;
let StackOpLockService: typeof import('../services/StackOpLockService').StackOpLockService;
let counter = 0;

beforeAll(async () => {
    tmpDir = await setupTestDb();
    ({ DatabaseService } = await import('../services/DatabaseService'));
    ({ BlueprintReconciler } = await import('../services/BlueprintReconciler'));
    ({ NodeLabelService } = await import('../services/NodeLabelService'));
    ({ BlueprintService } = await import('../services/BlueprintService'));
    ({ StackOpLockService } = await import('../services/StackOpLockService'));
});

afterAll(() => cleanupTestDb(tmpDir));

beforeEach(() => {
    const db = DatabaseService.getInstance().getDb();
    db.prepare('DELETE FROM blueprint_deployments').run();
    db.prepare('DELETE FROM blueprints').run();
    db.prepare('DELETE FROM node_labels').run();
    db.prepare("DELETE FROM nodes WHERE is_default = 0").run();
    db.prepare("UPDATE global_settings SET value = '0' WHERE key = 'developer_mode'").run();
    StackOpLockService.resetForTests();
    vi.restoreAllMocks();
});

function seedNode(): number {
    counter += 1;
    const db = DatabaseService.getInstance().getDb();
    const result = db.prepare(
        `INSERT INTO nodes (name, type, mode, compose_dir, is_default, status, created_at)
         VALUES (?, 'local', 'proxy', '/tmp/compose', 0, 'online', ?)`
    ).run(`bp-test-${counter}`, Date.now());
    return result.lastInsertRowid as number;
}

function seedBlueprint(opts: {
    name?: string;
    classification?: 'stateless' | 'stateful' | 'unknown';
    drift_mode?: 'observe' | 'suggest' | 'enforce';
    nodeIds?: number[];
    revision?: number;
}) {
    counter += 1;
    const name = opts.name ?? `bp-${counter}`;
    return DatabaseService.getInstance().createBlueprint({
        name,
        description: null,
        compose_content: 'services:\n  app:\n    image: nginx\n',
        selector: { type: 'nodes', ids: opts.nodeIds ?? [] },
        drift_mode: opts.drift_mode ?? 'suggest',
        classification: opts.classification ?? 'stateless',
        classification_reasons: [],
        enabled: true,
        created_by: null,
    });
}

describe('BlueprintReconciler.computeDecision', () => {
    it('queues deploy for a stateless blueprint targeting a fresh node', () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.deploy.map((n: { id: number }) => n.id)).toContain(nodeId);
        expect(decision.stateReview).toEqual([]);
    });

    it('queues state-review (not deploy) for a stateful blueprint targeting a fresh node', () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateful', nodeIds: [nodeId] });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.stateReview.map((n: { id: number }) => n.id)).toContain(nodeId);
        expect(decision.deploy).toEqual([]);
    });

    it('queues drift-check for an active deployment whose revision matches', () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id,
            node_id: nodeId,
            status: 'active',
            applied_revision: bp.revision,
        });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.check.map((n: { id: number }) => n.id)).toContain(nodeId);
        expect(decision.deploy).toEqual([]);
    });

    it('queues redeploy when the blueprint revision moved past the deployed revision', () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id,
            node_id: nodeId,
            status: 'active',
            applied_revision: bp.revision - 1,
        });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.deploy.map((n: { id: number }) => n.id)).toContain(nodeId);
    });

    it('queues state-review when a stateful deployment revision moves', () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateful', nodeIds: [nodeId] });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id,
            node_id: nodeId,
            status: 'active',
            applied_revision: bp.revision - 1,
        });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.stateReview.map((n: { id: number }) => n.id)).toContain(nodeId);
        expect(decision.deploy).toEqual([]);
    });

    it('queues stateless eviction when a node leaves the selector', () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [] });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id,
            node_id: nodeId,
            status: 'active',
            applied_revision: bp.revision,
        });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.withdraw.map((n: { id: number }) => n.id)).toContain(nodeId);
        expect(decision.evictBlocked).toEqual([]);
    });

    it('queues evict_blocked (not auto-withdraw) when a STATEFUL deployment leaves the selector', () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateful', nodeIds: [] });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id,
            node_id: nodeId,
            status: 'active',
            applied_revision: bp.revision,
        });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.evictBlocked.map((n: { id: number }) => n.id)).toContain(nodeId);
        expect(decision.withdraw).toEqual([]);
    });

    it('skips deployments already in pending_state_review or evict_blocked or name_conflict', () => {
        const nodeA = seedNode();
        const nodeB = seedNode();
        const nodeC = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeA, nodeB, nodeC] });
        DatabaseService.getInstance().upsertDeployment({ blueprint_id: bp.id, node_id: nodeA, status: 'pending_state_review' });
        DatabaseService.getInstance().upsertDeployment({ blueprint_id: bp.id, node_id: nodeB, status: 'evict_blocked' });
        DatabaseService.getInstance().upsertDeployment({ blueprint_id: bp.id, node_id: nodeC, status: 'name_conflict' });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.deploy).toEqual([]);
        expect(decision.check).toEqual([]);
        expect(decision.withdraw).toEqual([]);
    });

    it('skips new placements onto cordoned nodes (cordon filter)', () => {
        const nodeId = seedNode();
        DatabaseService.getInstance().setNodeCordoned(nodeId, true, 'maintenance');
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.deploy).toEqual([]);
        expect(decision.stateReview).toEqual([]);
    });

    it('skips state-review for stateful blueprints landing on cordoned nodes', () => {
        const nodeId = seedNode();
        DatabaseService.getInstance().setNodeCordoned(nodeId, true, null);
        const bp = seedBlueprint({ classification: 'stateful', nodeIds: [nodeId] });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.stateReview).toEqual([]);
        expect(decision.deploy).toEqual([]);
    });

    it('still redeploys for revision drift on a cordoned node (existing deployment, not a new placement)', () => {
        const nodeId = seedNode();
        DatabaseService.getInstance().setNodeCordoned(nodeId, true, null);
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id,
            node_id: nodeId,
            status: 'active',
            applied_revision: bp.revision - 1,
        });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.deploy.map((n: { id: number }) => n.id)).toContain(nodeId);
    });

    it('still drift-checks active deployments on a cordoned node', () => {
        const nodeId = seedNode();
        DatabaseService.getInstance().setNodeCordoned(nodeId, true, null);
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id,
            node_id: nodeId,
            status: 'active',
            applied_revision: bp.revision,
        });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.check.map((n: { id: number }) => n.id)).toContain(nodeId);
    });

    it('honors pin override: desired set is exactly the pinned node, regardless of selector', () => {
        const nodeA = seedNode();
        const nodeB = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeA] });
        DatabaseService.getInstance().setBlueprintPinnedNode(bp.id, nodeB);
        const refreshed = DatabaseService.getInstance().getBlueprint(bp.id)!;
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(refreshed, allNodes);
        expect(decision.deploy.map((n: { id: number }) => n.id)).toContain(nodeB);
        expect(decision.deploy.map((n: { id: number }) => n.id)).not.toContain(nodeA);
    });

    it('pin overrides cordon: pinned blueprint deploys onto a cordoned node', () => {
        const nodeId = seedNode();
        DatabaseService.getInstance().setNodeCordoned(nodeId, true, null);
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [] });
        DatabaseService.getInstance().setBlueprintPinnedNode(bp.id, nodeId);
        const refreshed = DatabaseService.getInstance().getBlueprint(bp.id)!;
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(refreshed, allNodes);
        expect(decision.deploy.map((n: { id: number }) => n.id)).toContain(nodeId);
    });

    it('pin to a non-existent node yields an empty desired set', () => {
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [] });
        DatabaseService.getInstance().setBlueprintPinnedNode(bp.id, 999_999);
        const refreshed = DatabaseService.getInstance().getBlueprint(bp.id)!;
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(refreshed, allNodes);
        expect(decision.deploy).toEqual([]);
        expect(decision.stateReview).toEqual([]);
    });

    it('pin shrinks the desired set: stateless deployments on non-pinned nodes are queued for withdraw', () => {
        const nodeA = seedNode();
        const nodeB = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeA, nodeB] });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id, node_id: nodeA, status: 'active', applied_revision: bp.revision,
        });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id, node_id: nodeB, status: 'active', applied_revision: bp.revision,
        });
        DatabaseService.getInstance().setBlueprintPinnedNode(bp.id, nodeA);
        const refreshed = DatabaseService.getInstance().getBlueprint(bp.id)!;
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(refreshed, allNodes);
        expect(decision.check.map((n: { id: number }) => n.id)).toContain(nodeA);
        expect(decision.withdraw.map((n: { id: number }) => n.id)).toContain(nodeB);
        expect(decision.evictBlocked).toEqual([]);
    });

    it('pin shrinks the desired set: stateful deployments on non-pinned nodes are queued for evict_blocked', () => {
        const nodeA = seedNode();
        const nodeB = seedNode();
        const bp = seedBlueprint({ classification: 'stateful', nodeIds: [nodeA, nodeB] });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id, node_id: nodeA, status: 'active', applied_revision: bp.revision,
        });
        DatabaseService.getInstance().upsertDeployment({
            blueprint_id: bp.id, node_id: nodeB, status: 'active', applied_revision: bp.revision,
        });
        DatabaseService.getInstance().setBlueprintPinnedNode(bp.id, nodeA);
        const refreshed = DatabaseService.getInstance().getBlueprint(bp.id)!;
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(refreshed, allNodes);
        expect(decision.evictBlocked.map((n: { id: number }) => n.id)).toContain(nodeB);
        expect(decision.withdraw).toEqual([]);
    });

    it('deleting the pinned node clears the pin from the blueprint', () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [] });
        DatabaseService.getInstance().setBlueprintPinnedNode(bp.id, nodeId);
        expect(DatabaseService.getInstance().getBlueprint(bp.id)!.pinned_node_id).toBe(nodeId);
        DatabaseService.getInstance().deleteNode(nodeId);
        expect(DatabaseService.getInstance().getBlueprint(bp.id)!.pinned_node_id).toBeNull();
    });

    it('clearing the pin restores selector behavior on the next tick', () => {
        const nodeA = seedNode();
        const nodeB = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeA] });
        DatabaseService.getInstance().setBlueprintPinnedNode(bp.id, nodeB);
        DatabaseService.getInstance().setBlueprintPinnedNode(bp.id, null);
        const refreshed = DatabaseService.getInstance().getBlueprint(bp.id)!;
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(refreshed, allNodes);
        expect(decision.deploy.map((n: { id: number }) => n.id)).toContain(nodeA);
        expect(decision.deploy.map((n: { id: number }) => n.id)).not.toContain(nodeB);
    });

    it('matches via labels selector and respects label changes', () => {
        const nodeA = seedNode();
        const nodeB = seedNode();
        NodeLabelService.getInstance().addLabel(nodeA, 'prod');
        NodeLabelService.getInstance().addLabel(nodeB, 'staging');
        const bp = DatabaseService.getInstance().createBlueprint({
            name: 'caddy-via-labels',
            description: null,
            compose_content: 'services:\n  caddy:\n    image: caddy\n',
            selector: { type: 'labels', any: ['prod'], all: [] },
            drift_mode: 'suggest',
            classification: 'stateless',
            classification_reasons: [],
            enabled: true,
            created_by: null,
        });
        const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithCompute;
        const allNodes = DatabaseService.getInstance().getNodes();
        const decision = reconciler.computeDecision(bp, allNodes);
        expect(decision.deploy.map((n: { id: number }) => n.id)).toContain(nodeA);
        expect(decision.deploy.map((n: { id: number }) => n.id)).not.toContain(nodeB);
    });
});

describe('BlueprintReconciler developer-mode diagnostics', () => {
    it('does not emit diagnostic logs when developer mode is off', async () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateful', nodeIds: [nodeId] });
        const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);

        await BlueprintReconciler.getInstance().reconcileOne(bp.id);

        expect(infoSpy.mock.calls.some(([message]) => String(message).includes('[BlueprintReconciler:diag]'))).toBe(false);
    });

    it('emits diagnostic decision logs when developer mode is on', async () => {
        DatabaseService.getInstance().updateGlobalSetting('developer_mode', '1');
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateful', nodeIds: [nodeId] });
        const infoSpy = vi.spyOn(console, 'info').mockImplementation(() => undefined);

        await BlueprintReconciler.getInstance().reconcileOne(bp.id);

        expect(infoSpy.mock.calls.some(([message]) => String(message).includes('[BlueprintReconciler:diag]'))).toBe(true);
    });
});

describe('BlueprintService per-stack lock', () => {
    it('deploy under a free lock writes compose then marker, then deploys', async () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
        const node = DatabaseService.getInstance().getNode(nodeId)!;
        const { FileSystemService } = await import('../services/FileSystemService');
        const { ComposeService } = await import('../services/ComposeService');
        // Spy the file/deploy primitives so the locked critical section runs
        // without touching the real filesystem or Docker.
        vi.spyOn(FileSystemService.prototype, 'createStack').mockResolvedValue(undefined);
        const writeSpy = vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockResolvedValue(undefined);
        const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);

        const outcome = await BlueprintService.getInstance().deployToNode(bp, node);

        expect(outcome.status).toBe('active');
        expect(deploySpy).toHaveBeenCalledWith(bp.name, undefined, false);
        // Compose is written first, then the marker, both before the deploy.
        expect(writeSpy).toHaveBeenCalledTimes(2);
        expect(writeSpy.mock.calls[0][2]).toBe(bp.compose_content);
        expect(writeSpy.mock.calls[1][2]).toContain('"blueprintId"');
        const [composeOrder, markerOrder] = writeSpy.mock.invocationCallOrder;
        const [deployOrder] = deploySpy.mock.invocationCallOrder;
        expect(composeOrder).toBeLessThan(markerOrder);
        expect(markerOrder).toBeLessThan(deployOrder);
    });

    it('deploy skips, writes no stack files, and records failed when the stack lock is held', async () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
        const node = DatabaseService.getInstance().getNode(nodeId)!;
        // A manual operation holds the lock; the reconcile deploy must not race
        // it, and must not mutate compose/marker files before owning the lock.
        StackOpLockService.getInstance().tryAcquire(nodeId, bp.name, 'update', 'admin');

        const outcome = await BlueprintService.getInstance().deployToNode(bp, node);

        expect(outcome.status).toBe('failed');
        expect(outcome.error).toContain('already in progress');
        // No marker file was written (the lock guards the file writes too).
        expect(await BlueprintService.getInstance().readMarker(bp.name, node)).toBeNull();
        // The manual op still holds the lock; the deploy never acquired it.
        expect(StackOpLockService.getInstance().get(nodeId, bp.name)?.action).toBe('update');
    });

    it('withdraw skips and records failed when a manual operation holds the stack lock', async () => {
        const nodeId = seedNode();
        const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
        const node = DatabaseService.getInstance().getNode(nodeId)!;
        // A manual operation holds the lock; the withdraw must not race it.
        StackOpLockService.getInstance().tryAcquire(nodeId, bp.name, 'update', 'admin');

        const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node);

        expect(outcome.status).toBe('failed');
        expect(outcome.error).toContain('already in progress');
        // The lock is still held by the manual op (the withdraw never acquired it).
        expect(StackOpLockService.getInstance().get(nodeId, bp.name)?.action).toBe('update');
    });
});

describe('BlueprintService marker parsing + name-conflict guard', () => {
    it('parseMarker accepts a well-formed marker', () => {
        const marker = BlueprintService.parseMarker(JSON.stringify({ blueprintId: 7, revision: 3, lastApplied: 12345 }));
        expect(marker).toEqual({ blueprintId: 7, revision: 3, lastApplied: 12345 });
    });

    it('parseMarker rejects an invalid marker', () => {
        expect(BlueprintService.parseMarker('not json')).toBeNull();
        expect(BlueprintService.parseMarker(JSON.stringify({ revision: 1 }))).toBeNull();
        expect(BlueprintService.parseMarker(JSON.stringify({ blueprintId: 'nope', revision: 1 }))).toBeNull();
    });
});