import { Router, type Request, type Response } from 'express'; import { z } from 'zod'; import { CronExpressionParser } from 'cron-parser'; import DockerController from '../services/DockerController'; import { DatabaseService } from '../services/DatabaseService'; import { NodeRegistry } from '../services/NodeRegistry'; import { CacheService } from '../services/CacheService'; import { ImageUpdateService } from '../services/ImageUpdateService'; import { FileSystemService } from '../services/FileSystemService'; import { ComposeService } from '../services/ComposeService'; import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService'; import { NotificationService } from '../services/NotificationService'; import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; import { HealthGateService } from '../services/HealthGateService'; import { authMiddleware } from '../middleware/auth'; import { requireAdmin } from '../middleware/tierGates'; import { buildPolicyGateOptions } from '../helpers/policyGate'; import { summarizeBlockReasons } from '../utils/policy-risk'; import { isValidStackName } from '../utils/validation'; import { sanitizeForLog } from '../utils/safeLog'; import { logDebugTiming } from '../utils/requestTiming'; import { getErrorMessage } from '../utils/errors'; // Fleet aggregation cache: 2-minute TTL, shared across dashboard tabs. const FLEET_UPDATE_CACHE_KEY = 'fleet-updates'; const FLEET_CACHE_TTL = 120_000; const REMOTE_NODE_FETCH_TIMEOUT_MS = 5000; export const imageUpdatesRouter = Router(); imageUpdatesRouter.get('/', authMiddleware, (req: Request, res: Response): void => { try { const updates = DatabaseService.getInstance().getStackUpdateStatus(req.nodeId); res.json(updates); } catch (error) { console.error('Failed to fetch image update status:', error); res.status(500).json({ error: 'Failed to fetch image update status' }); } }); // Rich per-stack status (hasUpdate + check outcome + reason) for the sidebar and // readiness view. Auth-only, matching GET /; the boolean GET / is left intact so // the cross-version fleet aggregation contract is unaffected. imageUpdatesRouter.get('/detail', authMiddleware, (req: Request, res: Response): void => { const startedAt = Date.now(); let outcome: 'ok' | 'error' = 'ok'; let count = 0; try { const nodeId = req.nodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); const detail = DatabaseService.getInstance().getStackUpdateDetail(nodeId); count = Object.keys(detail).length; res.json(detail); } catch (error) { outcome = 'error'; console.error('Failed to fetch image update detail:', error); res.status(500).json({ error: 'Failed to fetch image update detail' }); } finally { logDebugTiming('[ImageUpdates:debug]', { route: 'GET /detail', nodeId: req.nodeId, count, elapsedMs: Date.now() - startedAt, outcome, }); } }); imageUpdatesRouter.post('/refresh', authMiddleware, (req: Request, res: Response): void => { if (!requireAdmin(req, res)) return; try { const triggered = ImageUpdateService.getInstance().triggerManualRefresh(); if (!triggered) { const mins = ImageUpdateService.manualCooldownMinutes; res.status(429).json({ error: `Rate limited. Please wait at least ${mins} minute${mins !== 1 ? 's' : ''} between manual refreshes.` }); return; } res.json({ success: true, message: 'Image update check started in background.' }); } catch (error) { console.error('Failed to trigger image update refresh:', error); res.status(500).json({ error: 'Failed to trigger refresh' }); } }); imageUpdatesRouter.get('/status', authMiddleware, (req: Request, res: Response): void => { const startedAt = Date.now(); let outcome: 'ok' | 'error' = 'ok'; try { res.json(ImageUpdateService.getInstance().getStatus()); } catch (error) { outcome = 'error'; console.error('Failed to fetch image update status:', error); res.status(500).json({ error: 'Failed to fetch image update status' }); } finally { logDebugTiming('[ImageUpdates:debug]', { route: 'GET /status', nodeId: req.nodeId, elapsedMs: Date.now() - startedAt, outcome, }); } }); /** * Validate a cron expression using the same contract as Scheduled Operations: * non-empty, reject 6+ fields, parse with CronExpressionParser, and prove * .next() can produce a future fire time. Nicknames like @daily are accepted. */ function validateImageCheckCron(cron: unknown): string | null { if (typeof cron !== 'string' || !cron.trim()) { return 'Cron expression is required.'; } if (cron.trim().split(/\s+/).length >= 6) { return 'Cron expression must use 5 fields (minute hour day month weekday). The seconds field is not supported.'; } try { const expr = CronExpressionParser.parse(cron); expr.next(); // prove the expression can produce a next fire time } catch { return 'Invalid cron expression.'; } return null; } // Min/max mirror ImageUpdateService's clamp; the service is the authority and // re-clamps on read, so this is the user-facing validation boundary. const IntervalPatchSchema = z.object({ minutes: z.coerce.number().int().min(15).max(1440), mode: z.enum(['interval', 'cron']).optional(), cron: z.string().optional(), }); imageUpdatesRouter.put('/interval', authMiddleware, (req: Request, res: Response): void => { if (!requireAdmin(req, res)) return; const parsed = IntervalPatchSchema.safeParse(req.body); if (!parsed.success) { res.status(400).json({ error: 'minutes must be an integer between 15 and 1440' }); return; } // Validate cron expression when mode is 'cron'. if (parsed.data.mode === 'cron') { const cronError = validateImageCheckCron(parsed.data.cron); if (cronError) { res.status(400).json({ error: cronError }); return; } } try { const db = DatabaseService.getInstance(); const writeSettings = db.getDb().transaction((entries: [string, string][]) => { for (const [k, v] of entries) db.updateGlobalSetting(k, v); }); const entries: [string, string][] = [ ['image_update_check_interval_minutes', String(parsed.data.minutes)], ]; if (parsed.data.mode !== undefined) { entries.push(['image_update_check_mode', parsed.data.mode]); } if (parsed.data.mode === 'cron' && parsed.data.cron !== undefined) { entries.push(['image_update_check_cron', parsed.data.cron]); } else if (parsed.data.mode === 'interval') { entries.push(['image_update_check_cron', '']); // clear stale cron } writeSettings(entries); // Reschedule the live timer so the new cadence takes effect without a restart. ImageUpdateService.getInstance().restartPolling(); res.json(ImageUpdateService.getInstance().getStatus()); } catch (error) { console.error('Failed to update image-update interval:', error); res.status(500).json({ error: 'Failed to update interval' }); } }); imageUpdatesRouter.get('/fleet', authMiddleware, async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; try { const result = await CacheService.getInstance().getOrFetch>>( FLEET_UPDATE_CACHE_KEY, FLEET_CACHE_TTL, async () => { const db = DatabaseService.getInstance(); const nodes = db.getNodes(); const nr = NodeRegistry.getInstance(); const data: Record> = {}; // Local nodes: synchronous DB reads. for (const node of nodes) { if (node.type === 'local') { data[node.id] = db.getStackUpdateStatus(node.id); } } // Remote nodes: parallel fetches with per-request timeouts. // Pilot-agent rows have no api_url; rely on getProxyTarget for the // reachability predicate AND the base URL so pilots with an active // tunnel participate in the fan-out. const remoteCandidates = nodes .filter(n => n.type === 'remote' && n.status === 'online') .map(node => ({ node, proxyTarget: nr.getProxyTarget(node.id) })) .filter((entry): entry is { node: typeof entry.node; proxyTarget: NonNullable } => entry.proxyTarget !== null); const remoteResults = await Promise.allSettled( remoteCandidates.map(async ({ node, proxyTarget }) => { const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS); try { const resp = await fetch(`${baseUrl}/api/image-updates`, { headers: proxyTarget.apiToken ? { Authorization: `Bearer ${proxyTarget.apiToken}` } : {}, signal: controller.signal, }); clearTimeout(timeout); if (resp.ok) return { nodeId: node.id, data: await resp.json() as Record }; } catch { clearTimeout(timeout); } return null; }), ); for (const entry of remoteResults) { if (entry.status === 'fulfilled' && entry.value) { data[entry.value.nodeId] = entry.value.data; } } return data; }, ); res.json(result); } catch (error) { console.error('Failed to aggregate fleet update status:', error); res.status(500).json({ error: 'Failed to aggregate fleet update status' }); } }); imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request, res: Response): Promise => { if (!requireAdmin(_req, res)) return; const db = DatabaseService.getInstance(); const nodes = db.getNodes(); const nr = NodeRegistry.getInstance(); const triggered: number[] = []; const rateLimited: number[] = []; const failed: number[] = []; // ImageUpdateService is a per-instance singleton, so the local node's manual // refresh fires at most once per request regardless of how many local rows // exist in the schema. const localNode = nodes.find(n => n.type === 'local'); if (localNode) { try { if (ImageUpdateService.getInstance().triggerManualRefresh()) { triggered.push(localNode.id); } else { rateLimited.push(localNode.id); } } catch (e) { console.error(`[ImageUpdates] Local fleet refresh failed for node ${localNode.id}:`, e); failed.push(localNode.id); } } // Pilot-agent rows have no api_url; rely on getProxyTarget for the // reachability predicate AND the base URL so pilots with an active // tunnel participate in the fan-out. const remoteCandidates = nodes .filter(n => n.type === 'remote' && n.status === 'online') .map(node => ({ node, proxyTarget: nr.getProxyTarget(node.id) })) .filter((entry): entry is { node: typeof entry.node; proxyTarget: NonNullable } => entry.proxyTarget !== null); const remoteResults = await Promise.allSettled( remoteCandidates.map(async ({ node, proxyTarget }) => { const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS); try { const resp = await fetch(`${baseUrl}/api/image-updates/refresh`, { method: 'POST', headers: proxyTarget.apiToken ? { Authorization: `Bearer ${proxyTarget.apiToken}` } : {}, signal: controller.signal, }); clearTimeout(timeout); return { nodeId: node.id, status: resp.status }; } catch (e) { clearTimeout(timeout); return { nodeId: node.id, status: 0, error: e }; } }), ); for (const entry of remoteResults) { if (entry.status !== 'fulfilled') continue; const { nodeId, status } = entry.value; if (status >= 200 && status < 300) { triggered.push(nodeId); } else if (status === 429) { rateLimited.push(nodeId); } else { failed.push(nodeId); } } CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY); res.json({ triggered, rateLimited, failed }); }); /** * Execute auto-update for a single stack (or for every stack on the local * node when target="*"). This runs on whichever Sencho instance receives * the request; the gateway scheduler proxies to remote nodes via HTTP. */ export const autoUpdateRouter = Router(); autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; try { const { target } = req.body as { target?: string }; console.log(`[AutoUpdate] Execute requested: target="${sanitizeForLog(target || '')}"`); if (!target || typeof target !== 'string') { res.status(400).json({ error: 'Missing "target" (stack name or "*" for all)' }); return; } let stackNames: string[]; if (target === '*') { stackNames = await FileSystemService.getInstance(req.nodeId).getStacks(); if (stackNames.length === 0) { res.json({ result: 'No stacks found on node; skipped.' }); return; } } else { if (!isValidStackName(target)) { res.status(400).json({ error: 'Invalid stack name' }); return; } stackNames = [target]; } const docker = DockerController.getInstance(req.nodeId); const imageUpdateService = ImageUpdateService.getInstance(); const compose = ComposeService.getInstance(req.nodeId); const db = DatabaseService.getInstance(); const atomic = true; const results: string[] = []; for (const stackName of stackNames) { try { const containers = await docker.getContainersByStack(stackName); if (!containers || containers.length === 0) { results.push(`Stack "${stackName}": no containers found; skipped.`); continue; } const imageRefs = [...new Set( containers .map((c: { Image?: string }) => c.Image) .filter((img): img is string => !!img && !img.startsWith('sha256:')), )]; if (imageRefs.length === 0) { results.push(`Stack "${stackName}": no pullable images; skipped.`); continue; } let hasUpdate = false; const updatedImages: string[] = []; const checkErrors: string[] = []; for (const imageRef of imageRefs) { try { const result = await imageUpdateService.checkImage(docker, imageRef); if (result.error) { checkErrors.push(result.error); } else if (result.hasUpdate) { hasUpdate = true; updatedImages.push(imageRef); } } catch (e) { const errMsg = getErrorMessage(e, String(e)); checkErrors.push(errMsg); console.warn('[AutoUpdate] Failed to check image %s:', sanitizeForLog(imageRef), sanitizeForLog((e as Error)?.message ?? String(e))); } } if (!hasUpdate) { if (checkErrors.length > 0 && checkErrors.length === imageRefs.length) { results.push(`Stack "${stackName}": WARNING - all image checks failed (${checkErrors.join('; ')}). Unable to determine update status.`); } else if (checkErrors.length > 0) { results.push(`Stack "${stackName}": all reachable images up to date (${checkErrors.length} check(s) failed).`); } else { results.push(`Stack "${stackName}": all images up to date.`); } continue; } // Auto-update runs from the scheduler: a policy bypass is never // appropriate. If updated images fail the gate, skip the stack and // raise a notification so an operator can review before a manual retry. const autoUpdateGate = await enforcePolicyPreDeploy( stackName, req.nodeId, buildPolicyGateOptions(req, { bypass: false, actor: `auto-update:${req.user?.username ?? 'scheduler'}`, }), ); if (!autoUpdateGate.ok) { const blockedImages = autoUpdateGate.violations.map((v) => v.imageRef).join(', '); const blockedMsg = `Policy "${autoUpdateGate.policy?.name}" blocked auto-update: ${autoUpdateGate.violations.length} image(s) matched ${summarizeBlockReasons(autoUpdateGate.violations)}${blockedImages ? ` (${blockedImages})` : ''}`; NotificationService.getInstance().dispatchAlert('warning', 'scan_finding', blockedMsg, { stackName, actor: 'system:image-update' }); results.push(`Stack "${stackName}": ${blockedMsg}`); continue; } const lock = await StackOpLockService.getInstance().runExclusive( req.nodeId, stackName, 'update', 'system', () => compose.updateStack(stackName, undefined, atomic), ); if (!lock.ran) { results.push(stackOpSkipMessage(stackName, lock.existing.action)); continue; } db.clearStackUpdateStatus(req.nodeId, stackName); HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`); NotificationService.getInstance().broadcastEvent({ type: 'state-invalidate', scope: 'image-updates', nodeId: req.nodeId, stackName, action: 'stack-updated', ts: Date.now(), }); NotificationService.getInstance().dispatchAlert( 'info', 'image_update_applied', `Auto-update: stack "${stackName}" updated with new images`, { stackName, actor: 'system:image-update' }, ); results.push(`Stack "${stackName}": updated (${updatedImages.join(', ')}).`); } catch (e) { const msg = getErrorMessage(e, String(e)); results.push(`Stack "${stackName}" failed: ${msg}`); console.error(`[AutoUpdate] Failed for stack "${stackName}":`, e); } } CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY); res.json({ result: results.join('\n') }); } catch (error) { const msg = getErrorMessage(error, 'Auto-update execution failed'); console.error('[AutoUpdate] Execute error:', msg); res.status(500).json({ error: msg }); } });