Files
sencho/backend/src/routes/imageUpdates.ts
T
Anso b70a529656 feat: add developer-mode startup and stack hydration timing (#1619)
* feat: add developer-mode startup and stack hydration timing

Instrument boot-to-list and detail hydration with commit-aligned milestones, truthful request stages, and destination/gateway debug duration logs so performance work is guided by measurements.

* fix: redact stack names and complete hydration request stages

Stop logging stack identifiers in containers debug timing, and record state_dispatch (plus detail fetch spans) so copied reports match the advertised stage breakdown.
2026-07-14 17:24:25 -04:00

461 lines
18 KiB
TypeScript

import { Router, type Request, type Response } from 'express';
import { z } from 'zod';
import { CronExpressionParser } from 'cron-parser';
import DockerController from '../services/DockerController';
import { DatabaseService } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry';
import { CacheService } from '../services/CacheService';
import { ImageUpdateService } from '../services/ImageUpdateService';
import { FileSystemService } from '../services/FileSystemService';
import { ComposeService } from '../services/ComposeService';
import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService';
import { NotificationService } from '../services/NotificationService';
import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
import { HealthGateService } from '../services/HealthGateService';
import { authMiddleware } from '../middleware/auth';
import { requireAdmin } from '../middleware/tierGates';
import { buildPolicyGateOptions } from '../helpers/policyGate';
import { summarizeBlockReasons } from '../utils/policy-risk';
import { isValidStackName } from '../utils/validation';
import { sanitizeForLog } from '../utils/safeLog';
import { logDebugTiming } from '../utils/requestTiming';
import { getErrorMessage } from '../utils/errors';
// Fleet aggregation cache: 2-minute TTL, shared across dashboard tabs.
const FLEET_UPDATE_CACHE_KEY = 'fleet-updates';
const FLEET_CACHE_TTL = 120_000;
const REMOTE_NODE_FETCH_TIMEOUT_MS = 5000;
export const imageUpdatesRouter = Router();
imageUpdatesRouter.get('/', authMiddleware, (req: Request, res: Response): void => {
try {
const updates = DatabaseService.getInstance().getStackUpdateStatus(req.nodeId);
res.json(updates);
} catch (error) {
console.error('Failed to fetch image update status:', error);
res.status(500).json({ error: 'Failed to fetch image update status' });
}
});
// Rich per-stack status (hasUpdate + check outcome + reason) for the sidebar and
// readiness view. Auth-only, matching GET /; the boolean GET / is left intact so
// the cross-version fleet aggregation contract is unaffected.
imageUpdatesRouter.get('/detail', authMiddleware, (req: Request, res: Response): void => {
const startedAt = Date.now();
let outcome: 'ok' | 'error' = 'ok';
let count = 0;
try {
const nodeId = req.nodeId ?? NodeRegistry.getInstance().getDefaultNodeId();
const detail = DatabaseService.getInstance().getStackUpdateDetail(nodeId);
count = Object.keys(detail).length;
res.json(detail);
} catch (error) {
outcome = 'error';
console.error('Failed to fetch image update detail:', error);
res.status(500).json({ error: 'Failed to fetch image update detail' });
} finally {
logDebugTiming('[ImageUpdates:debug]', {
route: 'GET /detail',
nodeId: req.nodeId,
count,
elapsedMs: Date.now() - startedAt,
outcome,
});
}
});
imageUpdatesRouter.post('/refresh', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
try {
const triggered = ImageUpdateService.getInstance().triggerManualRefresh();
if (!triggered) {
const mins = ImageUpdateService.manualCooldownMinutes;
res.status(429).json({ error: `Rate limited. Please wait at least ${mins} minute${mins !== 1 ? 's' : ''} between manual refreshes.` });
return;
}
res.json({ success: true, message: 'Image update check started in background.' });
} catch (error) {
console.error('Failed to trigger image update refresh:', error);
res.status(500).json({ error: 'Failed to trigger refresh' });
}
});
imageUpdatesRouter.get('/status', authMiddleware, (req: Request, res: Response): void => {
const startedAt = Date.now();
let outcome: 'ok' | 'error' = 'ok';
try {
res.json(ImageUpdateService.getInstance().getStatus());
} catch (error) {
outcome = 'error';
console.error('Failed to fetch image update status:', error);
res.status(500).json({ error: 'Failed to fetch image update status' });
} finally {
logDebugTiming('[ImageUpdates:debug]', {
route: 'GET /status',
nodeId: req.nodeId,
elapsedMs: Date.now() - startedAt,
outcome,
});
}
});
/**
* Validate a cron expression using the same contract as Scheduled Operations:
* non-empty, reject 6+ fields, parse with CronExpressionParser, and prove
* .next() can produce a future fire time. Nicknames like @daily are accepted.
*/
function validateImageCheckCron(cron: unknown): string | null {
if (typeof cron !== 'string' || !cron.trim()) {
return 'Cron expression is required.';
}
if (cron.trim().split(/\s+/).length >= 6) {
return 'Cron expression must use 5 fields (minute hour day month weekday). The seconds field is not supported.';
}
try {
const expr = CronExpressionParser.parse(cron);
expr.next(); // prove the expression can produce a next fire time
} catch {
return 'Invalid cron expression.';
}
return null;
}
// Min/max mirror ImageUpdateService's clamp; the service is the authority and
// re-clamps on read, so this is the user-facing validation boundary.
const IntervalPatchSchema = z.object({
minutes: z.coerce.number().int().min(15).max(1440),
mode: z.enum(['interval', 'cron']).optional(),
cron: z.string().optional(),
});
imageUpdatesRouter.put('/interval', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
const parsed = IntervalPatchSchema.safeParse(req.body);
if (!parsed.success) {
res.status(400).json({ error: 'minutes must be an integer between 15 and 1440' });
return;
}
// Validate cron expression when mode is 'cron'.
if (parsed.data.mode === 'cron') {
const cronError = validateImageCheckCron(parsed.data.cron);
if (cronError) {
res.status(400).json({ error: cronError });
return;
}
}
try {
const db = DatabaseService.getInstance();
const writeSettings = db.getDb().transaction((entries: [string, string][]) => {
for (const [k, v] of entries) db.updateGlobalSetting(k, v);
});
const entries: [string, string][] = [
['image_update_check_interval_minutes', String(parsed.data.minutes)],
];
if (parsed.data.mode !== undefined) {
entries.push(['image_update_check_mode', parsed.data.mode]);
}
if (parsed.data.mode === 'cron' && parsed.data.cron !== undefined) {
entries.push(['image_update_check_cron', parsed.data.cron]);
} else if (parsed.data.mode === 'interval') {
entries.push(['image_update_check_cron', '']); // clear stale cron
}
writeSettings(entries);
// Reschedule the live timer so the new cadence takes effect without a restart.
ImageUpdateService.getInstance().restartPolling();
res.json(ImageUpdateService.getInstance().getStatus());
} catch (error) {
console.error('Failed to update image-update interval:', error);
res.status(500).json({ error: 'Failed to update interval' });
}
});
imageUpdatesRouter.get('/fleet', authMiddleware, async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
try {
const result = await CacheService.getInstance().getOrFetch<Record<number, Record<string, boolean>>>(
FLEET_UPDATE_CACHE_KEY,
FLEET_CACHE_TTL,
async () => {
const db = DatabaseService.getInstance();
const nodes = db.getNodes();
const nr = NodeRegistry.getInstance();
const data: Record<number, Record<string, boolean>> = {};
// Local nodes: synchronous DB reads.
for (const node of nodes) {
if (node.type === 'local') {
data[node.id] = db.getStackUpdateStatus(node.id);
}
}
// Remote nodes: parallel fetches with per-request timeouts.
// Pilot-agent rows have no api_url; rely on getProxyTarget for the
// reachability predicate AND the base URL so pilots with an active
// tunnel participate in the fan-out.
const remoteCandidates = nodes
.filter(n => n.type === 'remote' && n.status === 'online')
.map(node => ({ node, proxyTarget: nr.getProxyTarget(node.id) }))
.filter((entry): entry is { node: typeof entry.node; proxyTarget: NonNullable<typeof entry.proxyTarget> } => entry.proxyTarget !== null);
const remoteResults = await Promise.allSettled(
remoteCandidates.map(async ({ node, proxyTarget }) => {
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS);
try {
const resp = await fetch(`${baseUrl}/api/image-updates`, {
headers: proxyTarget.apiToken
? { Authorization: `Bearer ${proxyTarget.apiToken}` }
: {},
signal: controller.signal,
});
clearTimeout(timeout);
if (resp.ok) return { nodeId: node.id, data: await resp.json() as Record<string, boolean> };
} catch {
clearTimeout(timeout);
}
return null;
}),
);
for (const entry of remoteResults) {
if (entry.status === 'fulfilled' && entry.value) {
data[entry.value.nodeId] = entry.value.data;
}
}
return data;
},
);
res.json(result);
} catch (error) {
console.error('Failed to aggregate fleet update status:', error);
res.status(500).json({ error: 'Failed to aggregate fleet update status' });
}
});
imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request, res: Response): Promise<void> => {
if (!requireAdmin(_req, res)) return;
const db = DatabaseService.getInstance();
const nodes = db.getNodes();
const nr = NodeRegistry.getInstance();
const triggered: number[] = [];
const rateLimited: number[] = [];
const failed: number[] = [];
// ImageUpdateService is a per-instance singleton, so the local node's manual
// refresh fires at most once per request regardless of how many local rows
// exist in the schema.
const localNode = nodes.find(n => n.type === 'local');
if (localNode) {
try {
if (ImageUpdateService.getInstance().triggerManualRefresh()) {
triggered.push(localNode.id);
} else {
rateLimited.push(localNode.id);
}
} catch (e) {
console.error(`[ImageUpdates] Local fleet refresh failed for node ${localNode.id}:`, e);
failed.push(localNode.id);
}
}
// Pilot-agent rows have no api_url; rely on getProxyTarget for the
// reachability predicate AND the base URL so pilots with an active
// tunnel participate in the fan-out.
const remoteCandidates = nodes
.filter(n => n.type === 'remote' && n.status === 'online')
.map(node => ({ node, proxyTarget: nr.getProxyTarget(node.id) }))
.filter((entry): entry is { node: typeof entry.node; proxyTarget: NonNullable<typeof entry.proxyTarget> } => entry.proxyTarget !== null);
const remoteResults = await Promise.allSettled(
remoteCandidates.map(async ({ node, proxyTarget }) => {
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS);
try {
const resp = await fetch(`${baseUrl}/api/image-updates/refresh`, {
method: 'POST',
headers: proxyTarget.apiToken
? { Authorization: `Bearer ${proxyTarget.apiToken}` }
: {},
signal: controller.signal,
});
clearTimeout(timeout);
return { nodeId: node.id, status: resp.status };
} catch (e) {
clearTimeout(timeout);
return { nodeId: node.id, status: 0, error: e };
}
}),
);
for (const entry of remoteResults) {
if (entry.status !== 'fulfilled') continue;
const { nodeId, status } = entry.value;
if (status >= 200 && status < 300) {
triggered.push(nodeId);
} else if (status === 429) {
rateLimited.push(nodeId);
} else {
failed.push(nodeId);
}
}
CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY);
res.json({ triggered, rateLimited, failed });
});
/**
* Execute auto-update for a single stack (or for every stack on the local
* node when target="*"). This runs on whichever Sencho instance receives
* the request; the gateway scheduler proxies to remote nodes via HTTP.
*/
export const autoUpdateRouter = Router();
autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
try {
const { target } = req.body as { target?: string };
console.log(`[AutoUpdate] Execute requested: target="${sanitizeForLog(target || '')}"`);
if (!target || typeof target !== 'string') {
res.status(400).json({ error: 'Missing "target" (stack name or "*" for all)' });
return;
}
let stackNames: string[];
if (target === '*') {
stackNames = await FileSystemService.getInstance(req.nodeId).getStacks();
if (stackNames.length === 0) {
res.json({ result: 'No stacks found on node; skipped.' });
return;
}
} else {
if (!isValidStackName(target)) {
res.status(400).json({ error: 'Invalid stack name' });
return;
}
stackNames = [target];
}
const docker = DockerController.getInstance(req.nodeId);
const imageUpdateService = ImageUpdateService.getInstance();
const compose = ComposeService.getInstance(req.nodeId);
const db = DatabaseService.getInstance();
const atomic = true;
const results: string[] = [];
for (const stackName of stackNames) {
try {
const containers = await docker.getContainersByStack(stackName);
if (!containers || containers.length === 0) {
results.push(`Stack "${stackName}": no containers found; skipped.`);
continue;
}
const imageRefs = [...new Set(
containers
.map((c: { Image?: string }) => c.Image)
.filter((img): img is string => !!img && !img.startsWith('sha256:')),
)];
if (imageRefs.length === 0) {
results.push(`Stack "${stackName}": no pullable images; skipped.`);
continue;
}
let hasUpdate = false;
const updatedImages: string[] = [];
const checkErrors: string[] = [];
for (const imageRef of imageRefs) {
try {
const result = await imageUpdateService.checkImage(docker, imageRef);
if (result.error) {
checkErrors.push(result.error);
} else if (result.hasUpdate) {
hasUpdate = true;
updatedImages.push(imageRef);
}
} catch (e) {
const errMsg = getErrorMessage(e, String(e));
checkErrors.push(errMsg);
console.warn('[AutoUpdate] Failed to check image %s:', sanitizeForLog(imageRef), sanitizeForLog((e as Error)?.message ?? String(e)));
}
}
if (!hasUpdate) {
if (checkErrors.length > 0 && checkErrors.length === imageRefs.length) {
results.push(`Stack "${stackName}": WARNING - all image checks failed (${checkErrors.join('; ')}). Unable to determine update status.`);
} else if (checkErrors.length > 0) {
results.push(`Stack "${stackName}": all reachable images up to date (${checkErrors.length} check(s) failed).`);
} else {
results.push(`Stack "${stackName}": all images up to date.`);
}
continue;
}
// Auto-update runs from the scheduler: a policy bypass is never
// appropriate. If updated images fail the gate, skip the stack and
// raise a notification so an operator can review before a manual retry.
const autoUpdateGate = await enforcePolicyPreDeploy(
stackName,
req.nodeId,
buildPolicyGateOptions(req, {
bypass: false,
actor: `auto-update:${req.user?.username ?? 'scheduler'}`,
}),
);
if (!autoUpdateGate.ok) {
const blockedImages = autoUpdateGate.violations.map((v) => v.imageRef).join(', ');
const blockedMsg = `Policy "${autoUpdateGate.policy?.name}" blocked auto-update: ${autoUpdateGate.violations.length} image(s) matched ${summarizeBlockReasons(autoUpdateGate.violations)}${blockedImages ? ` (${blockedImages})` : ''}`;
NotificationService.getInstance().dispatchAlert('warning', 'scan_finding', blockedMsg, { stackName, actor: 'system:image-update' });
results.push(`Stack "${stackName}": ${blockedMsg}`);
continue;
}
const lock = await StackOpLockService.getInstance().runExclusive(
req.nodeId, stackName, 'update', 'system',
() => compose.updateStack(stackName, undefined, atomic),
);
if (!lock.ran) {
results.push(stackOpSkipMessage(stackName, lock.existing.action));
continue;
}
db.clearStackUpdateStatus(req.nodeId, stackName);
HealthGateService.getInstance().begin(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
scope: 'image-updates',
nodeId: req.nodeId,
stackName,
action: 'stack-updated',
ts: Date.now(),
});
NotificationService.getInstance().dispatchAlert(
'info',
'image_update_applied',
`Auto-update: stack "${stackName}" updated with new images`,
{ stackName, actor: 'system:image-update' },
);
results.push(`Stack "${stackName}": updated (${updatedImages.join(', ')}).`);
} catch (e) {
const msg = getErrorMessage(e, String(e));
results.push(`Stack "${stackName}" failed: ${msg}`);
console.error(`[AutoUpdate] Failed for stack "${stackName}":`, e);
}
}
CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY);
res.json({ result: results.join('\n') });
} catch (error) {
const msg = getErrorMessage(error, 'Auto-update execution failed');
console.error('[AutoUpdate] Execute error:', msg);
res.status(500).json({ error: msg });
}
});