fix: harden blueprint deployment guardrails (#1027)

* fix: harden blueprint deployment guardrails

* fix: update Docker toolchain to Go 1.26.3

* fix: repair Dockerfile tr argument split across lines

* fix: bump protobufjs to clear npm audit high-severity advisories

backend/src/services/BlueprintAnalyzer.ts

@@ -17,6 +17,7 @@ interface ComposeShape {
 }
 
 interface ComposeService {
+    image?: string | null;
     volumes?: Array<string | ComposeServiceVolume> | null;
     tmpfs?: string | string[] | null;
 }
@@ -208,6 +209,20 @@ export class BlueprintAnalyzer {
         return false;
     }
 
+    static extractImageRefs(composeContent: string): string[] {
+        const doc = (parseYaml(composeContent) ?? {}) as ComposeShape;
+        const services = doc.services ?? {};
+        const seen = new Set<string>();
+        const images: string[] = [];
+        for (const serviceDef of Object.values(services)) {
+            const image = typeof serviceDef?.image === 'string' ? serviceDef.image.trim() : '';
+            if (!image || image.startsWith('sha256:') || seen.has(image)) continue;
+            seen.add(image);
+            images.push(image);
+        }
+        return images;
+    }
+
     private static extractNamedVolumes(composeContent: string): Set<string> {
         try {
             const doc = (parseYaml(composeContent) ?? {}) as ComposeShape;

backend/src/services/BlueprintReconciler.ts

@@ -8,10 +8,27 @@ import { BlueprintService } from './BlueprintService';
 import { BlueprintAnalyzer } from './BlueprintAnalyzer';
 import { NodeLabelService } from './NodeLabelService';
 import { NotificationService } from './NotificationService';
+import { sanitizeForLog } from '../utils/safeLog';
 
 const RECONCILER_INTERVAL_MS = 60_000;
 const RECONCILER_INITIAL_DELAY_MS = 5_000;
 
+function isDeveloperModeEnabled(): boolean {
+    try {
+        return DatabaseService.getInstance().getGlobalSettings().developer_mode === '1';
+    } catch {
+        return false;
+    }
+}
+
+function diagnosticLog(message: string, fields: Record<string, string | number | boolean | null | undefined>): void {
+    if (!isDeveloperModeEnabled()) return;
+    const safeFields = Object.fromEntries(
+        Object.entries(fields).map(([key, value]) => [key, typeof value === 'string' ? sanitizeForLog(value) : value]),
+    );
+    console.info(`[BlueprintReconciler:diag] ${message}`, safeFields);
+}
+
 export interface ReconcileDecision {
     deploy: Node[];
     withdraw: Node[];
@@ -77,17 +94,21 @@ export class BlueprintReconciler {
         const blueprint = DatabaseService.getInstance().getBlueprint(blueprintId);
         if (!blueprint || !blueprint.enabled) return;
         const nodes = DatabaseService.getInstance().getNodes();
+        diagnosticLog('manual reconcile requested', { blueprintId, nodeCount: nodes.length });
         await this.reconcileBlueprint(blueprint, nodes);
     }
 
     private async evaluate(): Promise<void> {
         if (this.running) return; // prevent overlap on slow ticks
         this.running = true;
+        const started = Date.now();
         try {
             const db = DatabaseService.getInstance();
             const blueprints = db.listEnabledBlueprints();
             if (blueprints.length === 0) return;
             const nodes = db.getNodes();
+            console.info('[BlueprintReconciler] tick start blueprints=%s nodes=%s', blueprints.length, nodes.length);
+            diagnosticLog('tick inputs', { blueprintCount: blueprints.length, nodeCount: nodes.length });
             for (const blueprint of blueprints) {
                 try {
                     await this.reconcileBlueprint(blueprint, nodes);
@@ -95,6 +116,7 @@ export class BlueprintReconciler {
                     console.error(`[BlueprintReconciler] failed for blueprint "${blueprint.name}":`, err);
                 }
             }
+            console.info('[BlueprintReconciler] tick complete blueprints=%s durationMs=%s', blueprints.length, Date.now() - started);
         } finally {
             this.running = false;
         }
@@ -102,15 +124,28 @@ export class BlueprintReconciler {
 
     private async reconcileBlueprint(blueprint: Blueprint, allNodes: Node[]): Promise<void> {
         const decision = this.computeDecision(blueprint, allNodes);
+        diagnosticLog('decision computed', {
+            blueprintId: blueprint.id,
+            blueprintName: blueprint.name,
+            revision: blueprint.revision,
+            deploy: decision.deploy.length,
+            withdraw: decision.withdraw.length,
+            check: decision.check.length,
+            stateReview: decision.stateReview.length,
+            evictBlocked: decision.evictBlocked.length,
+        });
 
         // 1. State-review guard for stateful blueprints reaching new nodes.
         for (const node of decision.stateReview) {
+            const existing = DatabaseService.getInstance().getDeployment(blueprint.id, node.id);
             DatabaseService.getInstance().upsertDeployment({
                 blueprint_id: blueprint.id,
                 node_id: node.id,
                 status: 'pending_state_review',
                 last_checked_at: Date.now(),
-                drift_summary: 'Stateful blueprint awaiting operator confirmation before first deploy',
+                drift_summary: existing
+                    ? 'Stateful blueprint revision change awaits operator confirmation'
+                    : 'Stateful blueprint awaiting operator confirmation before first deploy',
             });
         }
 
@@ -213,8 +248,11 @@ export class BlueprintReconciler {
                 continue;
             }
             if (dep.applied_revision !== blueprint.revision) {
-                // revision drift: re-deploy (stateful never auto-redeploys volume-destroying changes; handled in handleDrift)
-                decision.deploy.push(node);
+                if (blueprint.classification === 'stateful' || blueprint.classification === 'unknown') {
+                    decision.stateReview.push(node);
+                } else {
+                    decision.deploy.push(node);
+                }
                 continue;
             }
             if (dep.status === 'failed' || dep.status === 'pending') {

backend/src/services/BlueprintService.ts

@@ -13,11 +13,31 @@ import { FileSystemService } from './FileSystemService';
 import { NodeRegistry } from './NodeRegistry';
 import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './license-headers';
 import { LicenseService } from './LicenseService';
+import { enforcePolicyForImageRefs } from './PolicyEnforcement';
+import { triggerPostDeployScan } from '../helpers/policyGate';
+import { BlueprintAnalyzer } from './BlueprintAnalyzer';
+import { sanitizeForLog } from '../utils/safeLog';
 
 const MARKER_FILENAME = '.blueprint.json';
 const COMPOSE_FILENAME = 'docker-compose.yml';
 const REMOTE_HTTP_TIMEOUT_MS = 30_000;
 
+function isDeveloperModeEnabled(): boolean {
+    try {
+        return DatabaseService.getInstance().getGlobalSettings().developer_mode === '1';
+    } catch {
+        return false;
+    }
+}
+
+function diagnosticLog(message: string, fields: Record<string, string | number | boolean | null | undefined>): void {
+    if (!isDeveloperModeEnabled()) return;
+    const safeFields = Object.fromEntries(
+        Object.entries(fields).map(([key, value]) => [key, typeof value === 'string' ? sanitizeForLog(value) : value]),
+    );
+    console.info(`[BlueprintService:diag] ${message}`, safeFields);
+}
+
 export interface BlueprintMarker {
     blueprintId: number;
     revision: number;
@@ -185,18 +205,34 @@ export class BlueprintService {
         if (!this.acquireLock(blueprint.id, node.id)) {
             return { status: 'pending' };
         }
+        const started = Date.now();
+        console.info('[BlueprintService] deploy start blueprint=%s node=%s type=%s revision=%s',
+            sanitizeForLog(blueprint.name), node.id, node.type, blueprint.revision);
+        diagnosticLog('deploy inputs', {
+            blueprintId: blueprint.id,
+            blueprintName: blueprint.name,
+            nodeId: node.id,
+            nodeType: node.type,
+            revision: blueprint.revision,
+            classification: blueprint.classification,
+            driftMode: blueprint.drift_mode,
+        });
         try {
             this.setStatus(blueprint.id, node.id, 'deploying');
             if (await this.hasNameConflict(blueprint.name, node)) {
                 this.setStatus(blueprint.id, node.id, 'name_conflict', {
                     last_error: `A stack named "${blueprint.name}" already exists on this node and is not managed by Sencho.`,
                 });
+                console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s',
+                    sanitizeForLog(blueprint.name), node.id, Date.now() - started);
                 return { status: 'name_conflict', error: 'name_conflict' };
             }
             const marker = this.buildMarker(blueprint);
             if (node.type === 'local') {
+                diagnosticLog('deploy branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' });
                 await this.deployLocal(blueprint, node, marker);
             } else {
+                diagnosticLog('deploy branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' });
                 await this.deployRemote(blueprint, node, marker);
             }
             this.setStatus(blueprint.id, node.id, 'active', {
@@ -206,10 +242,14 @@ export class BlueprintService {
                 drift_summary: null,
                 last_error: null,
             });
+            console.info('[BlueprintService] deploy complete blueprint=%s node=%s durationMs=%s',
+                sanitizeForLog(blueprint.name), node.id, Date.now() - started);
             return { status: 'active' };
         } catch (err) {
             const message = BlueprintService.formatError(err);
             this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
+            console.error('[BlueprintService] deploy failed blueprint=%s node=%s durationMs=%s error=%s',
+                sanitizeForLog(blueprint.name), node.id, Date.now() - started, sanitizeForLog(message));
             return { status: 'failed', error: message };
         } finally {
             this.releaseLock(blueprint.id, node.id);
@@ -225,6 +265,16 @@ export class BlueprintService {
         if (!this.acquireLock(blueprint.id, node.id)) {
             return { status: 'pending' };
         }
+        const started = Date.now();
+        console.info('[BlueprintService] withdraw start blueprint=%s node=%s type=%s',
+            sanitizeForLog(blueprint.name), node.id, node.type);
+        diagnosticLog('withdraw inputs', {
+            blueprintId: blueprint.id,
+            blueprintName: blueprint.name,
+            nodeId: node.id,
+            nodeType: node.type,
+            classification: blueprint.classification,
+        });
         try {
             this.setStatus(blueprint.id, node.id, 'withdrawing');
             // Refuse to withdraw a directory we do not own
@@ -236,15 +286,21 @@ export class BlueprintService {
                 return { status: 'name_conflict' };
             }
             if (node.type === 'local') {
+                diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' });
                 await this.withdrawLocal(blueprint, node);
             } else {
+                diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' });
                 await this.withdrawRemote(blueprint, node);
             }
             DatabaseService.getInstance().deleteDeployment(blueprint.id, node.id);
+            console.info('[BlueprintService] withdraw complete blueprint=%s node=%s durationMs=%s',
+                sanitizeForLog(blueprint.name), node.id, Date.now() - started);
             return { status: 'withdrawn' };
         } catch (err) {
             const message = BlueprintService.formatError(err);
             this.setStatus(blueprint.id, node.id, 'failed', { last_error: `withdraw failed: ${message}` });
+            console.error('[BlueprintService] withdraw failed blueprint=%s node=%s durationMs=%s error=%s',
+                sanitizeForLog(blueprint.name), node.id, Date.now() - started, sanitizeForLog(message));
             return { status: 'failed', error: message };
         } finally {
             this.releaseLock(blueprint.id, node.id);
@@ -336,6 +392,17 @@ export class BlueprintService {
     }
 
     private async deployLocal(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise<void> {
+        const imageRefs = BlueprintAnalyzer.extractImageRefs(blueprint.compose_content);
+        const gate = await enforcePolicyForImageRefs(blueprint.name, node.id, imageRefs, {
+            bypass: false,
+            actor: 'blueprint-reconciler',
+            auditMethod: 'POST',
+            auditPath: `/api/blueprints/${blueprint.id}/apply`,
+        }, undefined, true);
+        if (!gate.ok) {
+            throw new Error(`Policy "${gate.policy?.name}" blocked deploy: ${gate.violations.length} image(s) exceed ${gate.policy?.max_severity}`);
+        }
+
         const fs = FileSystemService.getInstance(node.id);
         if (!(await this.stackDirExists(node, blueprint.name))) {
             await fs.createStack(blueprint.name);
@@ -343,6 +410,10 @@ export class BlueprintService {
         await fs.writeStackFile(blueprint.name, COMPOSE_FILENAME, blueprint.compose_content);
         await fs.writeStackFile(blueprint.name, MARKER_FILENAME, JSON.stringify(marker, null, 2));
         await ComposeService.getInstance(node.id).deployStack(blueprint.name, undefined, false);
+        triggerPostDeployScan(blueprint.name, node.id).catch(err => {
+            console.error('[BlueprintService] post-deploy scan failed for "%s" on node %s: %s',
+                sanitizeForLog(blueprint.name), node.id, sanitizeForLog(BlueprintService.formatError(err)));
+        });
     }
 
     private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<void> {

backend/src/services/PolicyEnforcement.ts

@@ -50,7 +50,6 @@ export async function enforcePolicyPreDeploy(
     nodeId: number,
     opts: PolicyEnforcementOptions,
 ): Promise<PolicyEnforcementResult> {
-    const svc = TrivyService.getInstance();
     const db = DatabaseService.getInstance();
     const policy = db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
 
@@ -58,6 +57,7 @@ export async function enforcePolicyPreDeploy(
         return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
     }
 
+    const svc = TrivyService.getInstance();
     if (!svc.isTrivyAvailable()) {
         NotificationService.getInstance().dispatchAlert(
             'warning',
@@ -88,9 +88,49 @@ export async function enforcePolicyPreDeploy(
         };
     }
 
+    return enforcePolicyForImageRefs(stackName, nodeId, imageRefs, opts, policy);
+}
+
+export async function enforcePolicyForImageRefs(
+    stackName: string,
+    nodeId: number,
+    imageRefs: string[],
+    opts: PolicyEnforcementOptions,
+    matchedPolicy?: ScanPolicy,
+    failClosedInvalidRefs = false,
+): Promise<PolicyEnforcementResult> {
+    const db = DatabaseService.getInstance();
+    const policy = matchedPolicy ?? db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
+
+    if (!policy || !policy.enabled || !policy.block_on_deploy) {
+        return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
+    }
+
+    const svc = TrivyService.getInstance();
+    if (!svc.isTrivyAvailable()) {
+        NotificationService.getInstance().dispatchAlert(
+            'warning',
+            'scan_finding',
+            `Pre-deploy scan for "${stackName}" skipped: Trivy not installed on this node`,
+            { stackName },
+        );
+        return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
+    }
+
     const violations: PolicyViolation[] = [];
     for (const imageRef of imageRefs) {
-        if (!validateImageRef(imageRef)) continue;
+        if (!validateImageRef(imageRef)) {
+            if (failClosedInvalidRefs) {
+                violations.push({
+                    imageRef,
+                    severity: 'UNKNOWN',
+                    criticalCount: 0,
+                    highCount: 0,
+                    scanId: 0,
+                });
+            }
+            continue;
+        }
         try {
             const scan = await svc.scanImagePreflight(imageRef, nodeId, stackName);
             const severity = scan.highest_severity ?? 'UNKNOWN';