refactor(backend): add tests then extract metrics and image-updates routers (phase 4b follow-up) (#738)

Wraps up Phase 4 Round B by tackling the two deferred groups. 25 new
integration tests land first and run green against the inline monolith,
then each group is extracted byte-for-byte.

index.ts drops from ~3,678 to ~3,231 lines; test count rises 1,320 → 1,345.

New coverage:
- metrics-routes.test.ts (11) — auth + shape checks for /api/stats,
  /api/metrics/historical, /api/system/stats, /api/system/cache-stats
  (admin-only), and SSE headers for /api/logs/global/stream
- image-updates-routes.test.ts (14) — auth, admin gating, rate-limit
  tolerance on /refresh, fleet aggregation, /auto-update/execute input
  validation and no-stacks short-circuit

New route files:
- routes/metrics.ts — /stats, /metrics/historical, /logs/global (+ SSE
  /stream), /system/stats, /system/cache-stats. Mounted at /api so the
  mixed sub-paths line up.
- routes/imageUpdates.ts — /api/image-updates CRUD + fleet aggregation,
  plus a separate autoUpdateRouter mounted at /api/auto-update that
  owns the /execute handler. Same split pattern as license.ts +
  systemUpdateRouter.

index.ts trims unused imports left behind by the extraction:
globalDockerNetwork, si, STATS_CACHE_TTL_MS, SYSTEM_STATS_CACHE_TTL_MS,
GlobalLogEntry + log-parsing helpers.

backend/src/routes/imageUpdates.ts

@@ -0,0 +1,242 @@
+import { Router, type Request, type Response } from 'express';
+import DockerController from '../services/DockerController';
+import { DatabaseService } from '../services/DatabaseService';
+import { NodeRegistry } from '../services/NodeRegistry';
+import { CacheService } from '../services/CacheService';
+import { ImageUpdateService } from '../services/ImageUpdateService';
+import { FileSystemService } from '../services/FileSystemService';
+import { ComposeService } from '../services/ComposeService';
+import { LicenseService } from '../services/LicenseService';
+import { NotificationService } from '../services/NotificationService';
+import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
+import { authMiddleware } from '../middleware/auth';
+import { requireAdmin } from '../middleware/tierGates';
+import { buildPolicyGateOptions } from '../helpers/policyGate';
+import { isValidStackName } from '../utils/validation';
+import { getErrorMessage } from '../utils/errors';
+
+// Fleet aggregation cache: 2-minute TTL, shared across dashboard tabs.
+const FLEET_UPDATE_CACHE_KEY = 'fleet-updates';
+const FLEET_CACHE_TTL = 120_000;
+const REMOTE_NODE_FETCH_TIMEOUT_MS = 5000;
+
+export const imageUpdatesRouter = Router();
+
+imageUpdatesRouter.get('/', authMiddleware, (req: Request, res: Response): void => {
+  try {
+    const updates = DatabaseService.getInstance().getStackUpdateStatus(req.nodeId);
+    res.json(updates);
+  } catch (error) {
+    console.error('Failed to fetch image update status:', error);
+    res.status(500).json({ error: 'Failed to fetch image update status' });
+  }
+});
+
+imageUpdatesRouter.post('/refresh', authMiddleware, (req: Request, res: Response): void => {
+  if (!requireAdmin(req, res)) return;
+  try {
+    const triggered = ImageUpdateService.getInstance().triggerManualRefresh();
+    if (!triggered) {
+      const mins = ImageUpdateService.manualCooldownMinutes;
+      res.status(429).json({ error: `Rate limited. Please wait at least ${mins} minute${mins !== 1 ? 's' : ''} between manual refreshes.` });
+      return;
+    }
+    res.json({ success: true, message: 'Image update check started in background.' });
+  } catch (error) {
+    console.error('Failed to trigger image update refresh:', error);
+    res.status(500).json({ error: 'Failed to trigger refresh' });
+  }
+});
+
+imageUpdatesRouter.get('/status', authMiddleware, (_req: Request, res: Response): void => {
+  res.json({ checking: ImageUpdateService.getInstance().isChecking() });
+});
+
+imageUpdatesRouter.get('/fleet', authMiddleware, async (_req: Request, res: Response): Promise<void> => {
+  try {
+    const result = await CacheService.getInstance().getOrFetch<Record<number, Record<string, boolean>>>(
+      FLEET_UPDATE_CACHE_KEY,
+      FLEET_CACHE_TTL,
+      async () => {
+        const db = DatabaseService.getInstance();
+        const nodes = db.getNodes();
+        const nr = NodeRegistry.getInstance();
+        const data: Record<number, Record<string, boolean>> = {};
+
+        // Local nodes: synchronous DB reads.
+        for (const node of nodes) {
+          if (node.type === 'local') {
+            data[node.id] = db.getStackUpdateStatus(node.id);
+          }
+        }
+
+        // Remote nodes: parallel fetches with per-request timeouts.
+        const remoteNodes = nodes.filter(n => n.type === 'remote' && n.status === 'online' && n.api_url);
+        const remoteResults = await Promise.allSettled(
+          remoteNodes.map(async (node) => {
+            const proxyTarget = nr.getProxyTarget(node.id);
+            const baseUrl = node.api_url!.replace(/\/$/, '');
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS);
+            try {
+              const resp = await fetch(`${baseUrl}/api/image-updates`, {
+                headers: proxyTarget?.apiToken
+                  ? { Authorization: `Bearer ${proxyTarget.apiToken}` }
+                  : {},
+                signal: controller.signal,
+              });
+              clearTimeout(timeout);
+              if (resp.ok) return { nodeId: node.id, data: await resp.json() as Record<string, boolean> };
+            } catch {
+              clearTimeout(timeout);
+            }
+            return null;
+          }),
+        );
+
+        for (const entry of remoteResults) {
+          if (entry.status === 'fulfilled' && entry.value) {
+            data[entry.value.nodeId] = entry.value.data;
+          }
+        }
+
+        return data;
+      },
+    );
+    res.json(result);
+  } catch (error) {
+    console.error('Failed to aggregate fleet update status:', error);
+    res.status(500).json({ error: 'Failed to aggregate fleet update status' });
+  }
+});
+
+/**
+ * Execute auto-update for a single stack (or for every stack on the local
+ * node when target="*"). This runs on whichever Sencho instance receives
+ * the request; the gateway scheduler proxies to remote nodes via HTTP.
+ */
+export const autoUpdateRouter = Router();
+
+autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Response): Promise<void> => {
+  if (!requireAdmin(req, res)) return;
+  try {
+    const { target } = req.body as { target?: string };
+    console.log(`[AutoUpdate] Execute requested: target="${target || ''}"`);
+    if (!target || typeof target !== 'string') {
+      res.status(400).json({ error: 'Missing "target" (stack name or "*" for all)' });
+      return;
+    }
+
+    let stackNames: string[];
+    if (target === '*') {
+      stackNames = await FileSystemService.getInstance(req.nodeId).getStacks();
+      if (stackNames.length === 0) {
+        res.json({ result: 'No stacks found on node; skipped.' });
+        return;
+      }
+    } else {
+      if (!isValidStackName(target)) {
+        res.status(400).json({ error: 'Invalid stack name' });
+        return;
+      }
+      stackNames = [target];
+    }
+
+    const docker = DockerController.getInstance(req.nodeId);
+    const imageUpdateService = ImageUpdateService.getInstance();
+    const compose = ComposeService.getInstance(req.nodeId);
+    const db = DatabaseService.getInstance();
+    const atomic = LicenseService.getInstance().getTier() === 'paid';
+    const results: string[] = [];
+
+    for (const stackName of stackNames) {
+      try {
+        const containers = await docker.getContainersByStack(stackName);
+        if (!containers || containers.length === 0) {
+          results.push(`Stack "${stackName}": no containers found; skipped.`);
+          continue;
+        }
+
+        const imageRefs = [...new Set(
+          containers
+            .map((c: { Image?: string }) => c.Image)
+            .filter((img): img is string => !!img && !img.startsWith('sha256:')),
+        )];
+
+        if (imageRefs.length === 0) {
+          results.push(`Stack "${stackName}": no pullable images; skipped.`);
+          continue;
+        }
+
+        let hasUpdate = false;
+        const updatedImages: string[] = [];
+        const checkErrors: string[] = [];
+        for (const imageRef of imageRefs) {
+          try {
+            const result = await imageUpdateService.checkImage(docker, imageRef);
+            if (result.error) {
+              checkErrors.push(result.error);
+            } else if (result.hasUpdate) {
+              hasUpdate = true;
+              updatedImages.push(imageRef);
+            }
+          } catch (e) {
+            const errMsg = getErrorMessage(e, String(e));
+            checkErrors.push(errMsg);
+            console.warn(`[AutoUpdate] Failed to check image ${imageRef}:`, e);
+          }
+        }
+
+        if (!hasUpdate) {
+          if (checkErrors.length > 0 && checkErrors.length === imageRefs.length) {
+            results.push(`Stack "${stackName}": WARNING - all image checks failed (${checkErrors.join('; ')}). Unable to determine update status.`);
+          } else if (checkErrors.length > 0) {
+            results.push(`Stack "${stackName}": all reachable images up to date (${checkErrors.length} check(s) failed).`);
+          } else {
+            results.push(`Stack "${stackName}": all images up to date.`);
+          }
+          continue;
+        }
+
+        // Auto-update runs from the scheduler: a policy bypass is never
+        // appropriate. If updated images fail the gate, skip the stack and
+        // raise a notification so an operator can review before a manual retry.
+        const autoUpdateGate = await enforcePolicyPreDeploy(
+          stackName,
+          req.nodeId,
+          buildPolicyGateOptions(req, {
+            bypass: false,
+            actor: `auto-update:${req.user?.username ?? 'scheduler'}`,
+          }),
+        );
+        if (!autoUpdateGate.ok) {
+          const blockedMsg = `Policy "${autoUpdateGate.policy?.name}" blocked auto-update: ${autoUpdateGate.violations.length} image(s) exceed ${autoUpdateGate.policy?.max_severity}`;
+          NotificationService.getInstance().dispatchAlert('warning', blockedMsg, stackName);
+          results.push(`Stack "${stackName}": ${blockedMsg}`);
+          continue;
+        }
+
+        await compose.updateStack(stackName, undefined, atomic);
+        db.clearStackUpdateStatus(req.nodeId, stackName);
+
+        NotificationService.getInstance().dispatchAlert(
+          'info',
+          `Auto-update: stack "${stackName}" updated with new images`,
+          stackName,
+        );
+
+        results.push(`Stack "${stackName}": updated (${updatedImages.join(', ')}).`);
+      } catch (e) {
+        const msg = getErrorMessage(e, String(e));
+        results.push(`Stack "${stackName}" failed: ${msg}`);
+        console.error(`[AutoUpdate] Failed for stack "${stackName}":`, e);
+      }
+    }
+
+    res.json({ result: results.join('\n') });
+  } catch (error) {
+    const msg = getErrorMessage(error, 'Auto-update execution failed');
+    console.error('[AutoUpdate] Execute error:', msg);
+    res.status(500).json({ error: msg });
+  }
+});

backend/src/routes/metrics.ts

@@ -0,0 +1,251 @@
+import { Router, type Request, type Response } from 'express';
+import path from 'path';
+import si from 'systeminformation';
+import DockerController, { globalDockerNetwork } from '../services/DockerController';
+import { DatabaseService } from '../services/DatabaseService';
+import { CacheService } from '../services/CacheService';
+import { NodeRegistry } from '../services/NodeRegistry';
+import { authMiddleware } from '../middleware/auth';
+import { requireAdmin } from '../middleware/tierGates';
+import { STATS_CACHE_TTL_MS, SYSTEM_STATS_CACHE_TTL_MS } from '../helpers/constants';
+import { isDebugEnabled } from '../utils/debug';
+import { getErrorMessage } from '../utils/errors';
+import {
+  type GlobalLogEntry,
+  normalizeContainerName,
+  parseLogTimestamp,
+  detectLogLevel,
+  demuxDockerLog,
+} from '../utils/log-parsing';
+
+export const metricsRouter = Router();
+
+/**
+ * Container stats aggregated for the dashboard. Cached per-node for 2s to
+ * collapse multi-tab polling pressure. Write-path endpoints (deploy, down,
+ * start, stop, restart) invalidate this key via `invalidateNodeCaches`.
+ */
+metricsRouter.get('/stats', authMiddleware, async (req: Request, res: Response): Promise<void> => {
+  try {
+    const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(req.nodeId));
+    const result = await CacheService.getInstance().getOrFetch(
+      `stats:${req.nodeId}`,
+      STATS_CACHE_TTL_MS,
+      async () => {
+        const allContainers = await DockerController.getInstance(req.nodeId).getAllContainers();
+
+        // "Managed" means Docker started the container from within COMPOSE_DIR.
+        // We key on `com.docker.compose.project.working_dir` rather than the
+        // project name so stacks launched from the COMPOSE_DIR root (not a
+        // subdirectory) aren't all mis-classified as external.
+        const isManagedByComposeDir = (c: { Labels?: Record<string, string> }): boolean => {
+          const workingDir: string | undefined = c.Labels?.['com.docker.compose.project.working_dir'];
+          if (!workingDir) return false;
+          const resolved = path.resolve(workingDir);
+          return resolved === composeDir || resolved.startsWith(composeDir + path.sep);
+        };
+
+        type ContainerInfo = { State?: string; Labels?: Record<string, string> };
+        const cs = allContainers as ContainerInfo[];
+        const active = cs.filter(c => c.State === 'running').length;
+        const exited = cs.filter(c => c.State === 'exited').length;
+        const total = cs.length;
+        const managed = cs.filter(c => c.State === 'running' && isManagedByComposeDir(c)).length;
+        const unmanaged = cs.filter(c => c.State === 'running' && !isManagedByComposeDir(c)).length;
+
+        return { active, managed, unmanaged, exited, total };
+      },
+    );
+    res.json(result);
+  } catch {
+    res.status(500).json({ error: 'Failed to fetch stats' });
+  }
+});
+
+metricsRouter.get('/metrics/historical', authMiddleware, async (_req: Request, res: Response): Promise<void> => {
+  try {
+    const metrics = DatabaseService.getInstance().getContainerMetrics(24);
+    res.json(metrics);
+  } catch {
+    res.status(500).json({ error: 'Failed to fetch metrics' });
+  }
+});
+
+metricsRouter.get('/logs/global', authMiddleware, async (req: Request, res: Response): Promise<void> => {
+  try {
+    const debug = isDebugEnabled();
+    const dockerController = DockerController.getInstance(req.nodeId);
+    const containers = await dockerController.getRunningContainers();
+    const allLogs: GlobalLogEntry[] = [];
+    if (debug) console.debug('[GlobalLogs:debug] Polling snapshot starting', { containerCount: containers.length, nodeId: req.nodeId });
+
+    await Promise.all(containers.map(async (c) => {
+      const stackName = c.Labels?.['com.docker.compose.project'] || 'system';
+      const rawName = c.Names?.[0]?.replace(/^\//, '') || c.Id.substring(0, 12);
+      const containerName = normalizeContainerName(rawName, stackName);
+
+      try {
+        const container = dockerController.getDocker().getContainer(c.Id);
+        const inspect = await container.inspect();
+        const isTty = inspect.Config.Tty;
+        const logsBuffer = await container.logs({ stdout: true, stderr: true, tail: 100, timestamps: true }) as Buffer;
+
+        demuxDockerLog(logsBuffer, isTty, (line, source) => {
+          if (!line.trim()) return;
+          const { timestampMs, cleanMessage } = parseLogTimestamp(line);
+          const level = detectLogLevel(cleanMessage, source);
+          allLogs.push({ stackName, containerName, source, level, message: cleanMessage, timestampMs });
+        });
+      } catch (err) {
+        console.warn(`[GlobalLogs] Failed to fetch/parse logs for container ${containerName} (${c.Id.substring(0, 12)}):`, getErrorMessage(err, 'unknown'));
+      }
+    }));
+
+    // Sort ascending by timestamp (newest bottom). Limit to 500 lines; the
+    // client only renders ~300 at a time.
+    allLogs.sort((a, b) => a.timestampMs - b.timestampMs);
+    if (debug) console.debug('[GlobalLogs:debug] Polling snapshot complete', { totalLines: allLogs.length });
+    res.json(allLogs.slice(-500));
+  } catch (error) {
+    console.error('[GlobalLogs] Snapshot fetch failed:', getErrorMessage(error, 'unknown'));
+    res.status(500).json({ error: 'Failed to fetch global logs' });
+  }
+});
+
+metricsRouter.get('/logs/global/stream', authMiddleware, async (req: Request, res: Response): Promise<void> => {
+  res.setHeader('Content-Type', 'text/event-stream');
+  res.setHeader('Cache-Control', 'no-cache');
+  res.setHeader('Connection', 'keep-alive');
+  // Prevent nginx from buffering SSE events (would cause burst delivery).
+  res.setHeader('X-Accel-Buffering', 'no');
+  res.flushHeaders();
+
+  const debug = isDebugEnabled();
+  const dockerController = DockerController.getInstance(req.nodeId);
+  const streams: NodeJS.ReadableStream[] = [];
+
+  // SSE heartbeat (: prefix is a comment, silently dropped by EventSource)
+  // every 30s keeps reverse proxies from closing idle connections.
+  const heartbeat = setInterval(() => {
+    if (!res.writableEnded) res.write(':heartbeat\n\n');
+  }, 30_000);
+
+  try {
+    const containers = await dockerController.getRunningContainers();
+    if (debug) console.debug('[GlobalLogs:debug] SSE stream opened', { containerCount: containers.length, nodeId: req.nodeId });
+
+    await Promise.all(containers.map(async (c) => {
+      const stackName = c.Labels?.['com.docker.compose.project'] || 'system';
+      const rawName = c.Names?.[0]?.replace(/^\//, '') || c.Id.substring(0, 12);
+      const containerName = normalizeContainerName(rawName, stackName);
+
+      try {
+        const container = dockerController.getDocker().getContainer(c.Id);
+        const inspect = await container.inspect();
+        const isTty = inspect.Config.Tty;
+
+        const stream = await container.logs({ follow: true, stdout: true, stderr: true, tail: 500, timestamps: true });
+        streams.push(stream);
+
+        stream.on('data', (chunk: Buffer) => {
+          demuxDockerLog(chunk, isTty, (line, source) => {
+            if (!line.trim()) return;
+            const { timestampMs, cleanMessage } = parseLogTimestamp(line);
+            const level = detectLogLevel(cleanMessage, source);
+            if (!res.writableEnded) {
+              res.write(`data: ${JSON.stringify({ stackName, containerName, source, level, message: cleanMessage, timestampMs })}\n\n`);
+            }
+          });
+        });
+      } catch (err) {
+        console.warn(`[GlobalLogs] Failed to attach stream for container ${containerName} (${c.Id.substring(0, 12)}):`, getErrorMessage(err, 'unknown'));
+      }
+    }));
+
+    req.on('close', () => {
+      clearInterval(heartbeat);
+      if (debug) console.debug('[GlobalLogs:debug] SSE stream closed, cleaning up', { streamCount: streams.length });
+      streams.forEach(s => {
+        try { (s as NodeJS.ReadableStream & { destroy(): void }).destroy(); } catch { /* stream already ended */ }
+      });
+    });
+
+  } catch (error) {
+    clearInterval(heartbeat);
+    console.error('[GlobalLogs] SSE stream attachment failed:', getErrorMessage(error, 'unknown'));
+    res.write(`data: ${JSON.stringify({ level: 'ERROR', message: '[Sencho] Failed to attach global log stream.', timestampMs: Date.now(), stackName: 'system', containerName: 'backend', source: 'STDERR' })}\n\n`);
+    res.end();
+  }
+});
+
+/**
+ * Host-level CPU / memory / disk / network sample. Cached for 3s to collapse
+ * overlapping samplers (dashboard polls every 5s, MonitorService samples
+ * every 30s, `si.currentLoad()` blocks ~200ms per call). No write-path
+ * invalidation: these are pure host metrics.
+ */
+metricsRouter.get('/system/stats', authMiddleware, async (req: Request, res: Response): Promise<void> => {
+  try {
+    // Network is read outside the cache because it is cheap and per-request.
+    const rxSec = Math.max(0, globalDockerNetwork.rxSec);
+    const txSec = Math.max(0, globalDockerNetwork.txSec);
+
+    const sample = await CacheService.getInstance().getOrFetch(
+      `system-stats:${req.nodeId}`,
+      SYSTEM_STATS_CACHE_TTL_MS,
+      async () => {
+        // Remote-node requests are intercepted and proxied upstream before
+        // reaching here; this fetcher only runs for local nodes.
+        const [currentLoad, mem, fsSize] = await Promise.all([
+          si.currentLoad(),
+          si.mem(),
+          si.fsSize(),
+        ]);
+
+        const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0];
+
+        return {
+          cpu: {
+            usage: currentLoad.currentLoad.toFixed(1),
+            cores: currentLoad.cpus.length,
+          },
+          memory: {
+            total: mem.total,
+            used: mem.used,
+            free: mem.free,
+            usagePercent: ((mem.used / mem.total) * 100).toFixed(1),
+          },
+          disk: mainDisk ? {
+            fs: mainDisk.fs,
+            mount: mainDisk.mount,
+            total: mainDisk.size,
+            used: mainDisk.used,
+            free: mainDisk.available,
+            usagePercent: mainDisk.use ? mainDisk.use.toFixed(1) : '0',
+          } : null,
+        };
+      },
+    );
+
+    res.json({ ...sample, network: { rxBytes: 0, txBytes: 0, rxSec, txSec } });
+  } catch (error) {
+    console.error('Failed to fetch system stats:', error);
+    res.status(500).json({ error: 'Failed to fetch system stats' });
+  }
+});
+
+/**
+ * Admin-only cache observability. Surfaces per-namespace hit/miss/stale
+ * counters and live entry counts for the unified CacheService. Used by
+ * Settings → About and for post-deploy verification that cache hit rates
+ * look healthy.
+ */
+metricsRouter.get('/system/cache-stats', authMiddleware, async (req: Request, res: Response): Promise<void> => {
+  if (!requireAdmin(req, res)) return;
+  try {
+    res.json(CacheService.getInstance().getStats());
+  } catch (error) {
+    console.error('Failed to fetch cache stats:', error);
+    res.status(500).json({ error: 'Failed to fetch cache stats' });
+  }
+});