From 90eae0392229892de1431f10f1e98919cb1a2593 Mon Sep 17 00:00:00 2001 From: Anso Date: Thu, 23 Apr 2026 21:05:04 -0400 Subject: [PATCH] refactor(backend): extract webhooks, users, git-sources, and fleet routers (phase 4a-3) (#736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final slice of Phase 4 Round A. Pulls the four remaining well-tested route groups out of index.ts. index.ts drops from ~5,930 to ~4,206 lines. New route files: - routes/webhooks.ts: /api/webhooks CRUD + HMAC-authenticated trigger. Uses shared webhookTriggerLimiter. Trigger preserves the raw-body path established by the conditional JSON parser for HMAC validation. - routes/users.ts: /api/users CRUD + /:id/mfa/reset + /:id/roles scoped-assignment surface. Uses rejectApiTokenScope across every handler, validateUsername helper, BCRYPT_SALT_ROUNDS, and isSqliteUniqueViolation for the role-assignment UNIQUE guard. - routes/gitSources.ts: /api/git-sources + /api/stacks/:name/git-source/*. Exports two routers (gitSourcesRouter + stackGitSourceRouter) because the per-stack paths need to mount at /api/stacks alongside the label routes extracted in phase 4a-1. String length limits are now named constants so the 400 responses stay truthful if the bounds change. - routes/fleet.ts: /api/fleet role, sync, overview, node drill-down, update-status + trigger (single + fleet-wide), and snapshot CRUD + restore. Local parseIdParam helper collapses seven copies of the parseInt/isNaN route-param pattern. Bugs fixed during review: - users.ts :id/roles POST — replace the fragile (err as Error).message?.includes('UNIQUE constraint') check with isSqliteUniqueViolation from utils/errors.ts. index.ts carries forward three symbols (updateTracker alias, CVE_ID_RE, parseScannersInput) until the corresponding security / nodes / scan routes get extracted in a later slice. --- backend/src/index.ts | 1756 +----------------------------- backend/src/routes/fleet.ts | 1035 ++++++++++++++++++ backend/src/routes/gitSources.ts | 239 ++++ backend/src/routes/users.ts | 346 ++++++ backend/src/routes/webhooks.ts | 158 +++ 5 files changed, 1794 insertions(+), 1740 deletions(-) create mode 100644 backend/src/routes/fleet.ts create mode 100644 backend/src/routes/gitSources.ts create mode 100644 backend/src/routes/users.ts create mode 100644 backend/src/routes/webhooks.ts diff --git a/backend/src/index.ts b/backend/src/index.ts index ccf26c9d..fc7c220e 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,14 +1,12 @@ import express, { Request, Response } from 'express'; import jwt from 'jsonwebtoken'; import DockerController, { globalDockerNetwork, type CreateNetworkOptions, type NetworkDriver } from './services/DockerController'; -import type Dockerode from 'dockerode'; import { FileSystemService } from './services/FileSystemService'; import { ComposeService } from './services/ComposeService'; -import bcrypt from 'bcrypt'; import crypto from 'crypto'; import si from 'systeminformation'; import path from 'path'; -import { DatabaseService, Node, ScheduledTask, UserRole, ResourceType, parsePolicyEvaluation, type VulnerabilityScan } from './services/DatabaseService'; +import { DatabaseService, ScheduledTask, parsePolicyEvaluation, type VulnerabilityScan } from './services/DatabaseService'; import { NotificationService } from './services/NotificationService'; import { MonitorService } from './services/MonitorService'; import { AutoHealService } from './services/AutoHealService'; @@ -22,33 +20,27 @@ import { PilotTunnelManager } from './services/PilotTunnelManager'; import { PilotCloseCode } from './pilot/protocol'; import { FleetSyncService } from './services/FleetSyncService'; import { LicenseService } from './services/LicenseService'; -import { WebhookService } from './services/WebhookService'; import { SSOService } from './services/SSOService'; import { SchedulerService } from './services/SchedulerService'; import { RegistryService } from './services/RegistryService'; import { CacheService } from './services/CacheService'; -import { CAPABILITIES, getSenchoVersion, isValidVersion, fetchRemoteMeta, type RemoteMeta } from './services/CapabilityRegistry'; +import { CAPABILITIES, getSenchoVersion, fetchRemoteMeta, type RemoteMeta } from './services/CapabilityRegistry'; import { GitSourceService, GitSourceError, sweepStaleTempDirs as sweepStaleGitTempDirs, repoHost as gitRepoHost } from './services/GitSourceService'; import { sendGitSourceError } from './utils/gitSourceHttp'; import './types/express'; import { PORT, - MIN_PASSWORD_LENGTH, MFA_REPLAY_TTL_MS, MFA_REPLAY_PURGE_INTERVAL_MS, STATS_CACHE_TTL_MS, SYSTEM_STATS_CACHE_TTL_MS, STACK_STATUSES_CACHE_TTL_MS, } from './helpers/constants'; -import { - checkPermission, - requirePermission, -} from './middleware/permissions'; +import { requirePermission } from './middleware/permissions'; import { requirePaid, requireAdmiral, requireAdmin, - requireNodeProxy, requireScheduledTaskTier, } from './middleware/tierGates'; import { @@ -56,10 +48,7 @@ import { runPolicyGate, triggerPostDeployScan, } from './helpers/policyGate'; -import { - webhookTriggerLimiter, - trivyInstallLimiter, -} from './middleware/rateLimiters'; +import { trivyInstallLimiter } from './middleware/rateLimiters'; import { authGate, auditLog } from './middleware/authGate'; import { enforceApiTokenScope } from './middleware/apiTokenScope'; import { errorHandler } from './middleware/errorHandler'; @@ -76,7 +65,11 @@ import { metaRouter } from './routes/meta'; import { authRouter } from './routes/auth'; import { mfaRouter } from './routes/mfa'; import { ssoRouter } from './routes/sso'; -import { licenseRouter, systemUpdateRouter, scheduleLocalUpdate } from './routes/license'; +import { licenseRouter, systemUpdateRouter } from './routes/license'; +import { webhooksRouter } from './routes/webhooks'; +import { usersRouter } from './routes/users'; +import { gitSourcesRouter, stackGitSourceRouter } from './routes/gitSources'; +import { fleetRouter } from './routes/fleet'; import { permissionsRouter } from './routes/permissions'; import { convertRouter } from './routes/convert'; import { alertsRouter } from './routes/alerts'; @@ -85,9 +78,7 @@ import { apiTokensRouter } from './routes/apiTokens'; import { auditLogRouter } from './routes/auditLog'; import { isDebugEnabled } from './utils/debug'; -import { getLatestVersion } from './utils/version-check'; import { getErrorMessage } from './utils/errors'; -import { captureLocalNodeFiles, captureRemoteNodeFiles, SnapshotNodeData } from './utils/snapshot-capture'; import { GlobalLogEntry, normalizeContainerName, parseLogTimestamp, detectLogLevel, demuxDockerLog } from './utils/log-parsing'; import SelfUpdateService from './services/SelfUpdateService'; import TrivyService, { SbomFormat } from './services/TrivyService'; @@ -96,7 +87,6 @@ import { enforcePolicyPreDeploy } from './services/PolicyEnforcement'; import { validateImageRef } from './utils/image-ref'; import { applySuppressions } from './utils/suppression-filter'; import { generateSarif } from './services/SarifExporter'; -import semver from 'semver'; import { CronExpressionParser } from 'cron-parser'; import { isValidStackName, isValidRemoteUrl, isPathWithinBase, isValidCidr, isValidIPv4, isValidDockerResourceId } from './utils/validation'; import YAML from 'yaml'; @@ -154,84 +144,17 @@ app.use('/api/labels', labelsRouter); app.use('/api/stacks', stackLabelsRouter); app.use('/api/api-tokens', apiTokensRouter); app.use('/api/audit-log', auditLogRouter); +app.use('/api/fleet', fleetRouter); +app.use('/api/webhooks', webhooksRouter); +app.use('/api/users', usersRouter); +app.use('/api/git-sources', gitSourcesRouter); +app.use('/api/stacks', stackGitSourceRouter); -// --- Fleet Overview (local-only, aggregates all nodes) --- - -// In-memory tracker for remote node updates (transient; lost on gateway restart). +// Symbols still consumed by inline security and node routes still living in +// index.ts. These will move with their route groups in a later slice. const updateTracker = FleetUpdateTrackerService.getInstance(); -const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes -const UPDATE_TIMEOUT_MSG = 'Node did not come back online within 5 minutes.'; -const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure - -// Latest Sencho version lookup and caching live in utils/version-check.ts -// (shared with MonitorService). Fleet compares the gateway version against -// whatever getLatestVersion() returns from GitHub or Docker Hub. - -/** Resolve the version to compare nodes against (latest from GitHub, or gateway fallback). */ -async function getCompareTarget(gatewayVersion: string | null) { - const latestVersion = await getLatestVersion(); - const latestValid = latestVersion !== null && isValidVersion(latestVersion); - const result = { - latestVersion, - latestValid, - compareVersion: latestValid ? latestVersion : gatewayVersion, - compareValid: latestValid || isValidVersion(gatewayVersion), - }; - if (isDebugEnabled()) { - console.debug('[Fleet:debug] Compare target resolved:', { gatewayVersion, latestVersion, using: result.compareVersion, valid: result.compareValid }); - } - return result; -} - -interface FleetNodeOverview { - id: number; - name: string; - type: 'local' | 'remote'; - status: 'online' | 'offline' | 'unknown'; - stats: { - active: number; - managed: number; - unmanaged: number; - exited: number; - total: number; - } | null; - systemStats: { - cpu: { usage: string; cores: number }; - memory: { total: number; used: number; free: number; usagePercent: string }; - disk: { total: number; used: number; free: number; usagePercent: string } | null; - } | null; - stacks: string[] | null; -} - -// Fleet role: tells the frontend whether this Sencho is the control or a replica. -// The control serves read+write for security rules. Replicas are read-only and managed upstream. -app.get('/api/fleet/role', authMiddleware, (req: Request, res: Response): void => { - if (!requireAdmin(req, res)) return; - res.json({ role: FleetSyncService.getRole() }); -}); - -const MAX_SYNC_ROWS = 5000; -const VALID_SEVERITY = new Set(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']); -const isIntFlag = (v: unknown): v is 0 | 1 => v === 0 || v === 1; - -function validateScanPolicyRow(row: unknown): string | null { - if (!row || typeof row !== 'object') return 'row must be an object'; - const r = row as Record; - if (typeof r.name !== 'string' || r.name.length === 0 || r.name.length > 200) return 'name must be a non-empty string'; - if (typeof r.max_severity !== 'string' || !VALID_SEVERITY.has(r.max_severity)) return 'max_severity must be CRITICAL, HIGH, MEDIUM, or LOW'; - if (r.stack_pattern !== null && typeof r.stack_pattern !== 'string') return 'stack_pattern must be a string or null'; - if (typeof r.stack_pattern === 'string' && r.stack_pattern.length > 200) return 'stack_pattern is too long'; - if (typeof r.node_identity !== 'string') return 'node_identity must be a string'; - if (r.node_identity.length > 500) return 'node_identity is too long'; - if (!isIntFlag(r.block_on_deploy)) return 'block_on_deploy must be 0 or 1'; - if (!isIntFlag(r.enabled)) return 'enabled must be 0 or 1'; - return null; -} - const CVE_ID_RE = /^(CVE-\d{4}-\d{4,}|GHSA-[\w-]{14,})$/; -// Returns a normalized scanners array, undefined when no input was provided, -// or null when the input is present but invalid. function parseScannersInput(raw: unknown): readonly ('vuln' | 'secret')[] | undefined | null { if (raw === undefined || raw === null) return undefined; if (!Array.isArray(raw) || raw.length === 0) return null; @@ -242,1455 +165,6 @@ function parseScannersInput(raw: unknown): readonly ('vuln' | 'secret')[] | unde } return Array.from(out) as readonly ('vuln' | 'secret')[]; } -function validateCveSuppressionRow(row: unknown): string | null { - if (!row || typeof row !== 'object') return 'row must be an object'; - const r = row as Record; - if (typeof r.cve_id !== 'string' || !CVE_ID_RE.test(r.cve_id)) return 'cve_id must be a valid CVE or GHSA identifier'; - if (r.pkg_name !== null && typeof r.pkg_name !== 'string') return 'pkg_name must be a string or null'; - if (typeof r.pkg_name === 'string' && r.pkg_name.length > 200) return 'pkg_name is too long'; - if (r.image_pattern !== null && typeof r.image_pattern !== 'string') return 'image_pattern must be a string or null'; - if (typeof r.image_pattern === 'string' && r.image_pattern.length > 300) return 'image_pattern is too long'; - if (typeof r.reason !== 'string') return 'reason must be a string'; - if (r.reason.length > 2000) return 'reason is too long'; - if (typeof r.created_by !== 'string' || r.created_by.length > 200) return 'created_by must be a string'; - if (typeof r.created_at !== 'number') return 'created_at must be a number'; - if (r.expires_at !== null && typeof r.expires_at !== 'number') return 'expires_at must be a number or null'; - return null; -} - -// Fleet sync: receive a full replacement of a replicated resource from the control. -// Restricted to node_proxy Bearer tokens so only a sibling Sencho can push. -app.post('/api/fleet/sync/:resource', authMiddleware, (req: Request, res: Response): void => { - if (!requireNodeProxy(req, res)) return; - const resource = req.params.resource; - if (resource !== 'scan_policies' && resource !== 'cve_suppressions') { - res.status(400).json({ error: `Unsupported sync resource: ${resource}` }); - return; - } - const body = req.body ?? {}; - const rows = Array.isArray(body.rows) ? body.rows : null; - const targetIdentity = typeof body.targetIdentity === 'string' ? body.targetIdentity : ''; - if (!rows) { - res.status(400).json({ error: 'rows array is required' }); - return; - } - if (rows.length > MAX_SYNC_ROWS) { - res.status(413).json({ error: `Too many rows (max ${MAX_SYNC_ROWS})` }); - return; - } - const validator = resource === 'scan_policies' ? validateScanPolicyRow : validateCveSuppressionRow; - for (let i = 0; i < rows.length; i++) { - const err = validator(rows[i]); - if (err) { - res.status(400).json({ error: `Invalid row at index ${i}: ${err}` }); - return; - } - } - try { - FleetSyncService.getInstance().applyIncomingSync(resource, rows, targetIdentity); - res.json({ success: true, applied: rows.length }); - } catch (error) { - console.error('[FleetSync] Failed to apply incoming sync:', error); - res.status(500).json({ error: 'Failed to apply sync' }); - } -}); - -// Fleet sync status: surfaces per-node replication results so operators can spot stale replicas. -app.get('/api/fleet/sync-status', authMiddleware, (req: Request, res: Response): void => { - if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; - res.json(DatabaseService.getInstance().getFleetSyncStatuses()); -}); - -app.get('/api/fleet/overview', authMiddleware, async (_req: Request, res: Response): Promise => { - try { - const debug = isDebugEnabled(); - const db = DatabaseService.getInstance(); - const nodes = db.getNodes(); - if (debug) console.debug('[Fleet:debug] Overview requested, fetching', nodes.length, 'nodes'); - - const results = await Promise.allSettled( - nodes.map(async (node): Promise => { - if (node.type === 'remote') { - return fetchRemoteNodeOverview(node); - } - return fetchLocalNodeOverview(node); - }) - ); - - const overview: FleetNodeOverview[] = results.map((result, i) => { - if (result.status === 'fulfilled') return result.value; - console.error(`[Fleet] Failed to fetch node ${nodes[i].name}:`, result.reason); - return { - id: nodes[i].id, - name: nodes[i].name, - type: nodes[i].type, - status: 'offline' as const, - stats: null, - systemStats: null, - stacks: null, - }; - }); - - if (debug) { - const online = overview.filter(n => n.status === 'online').length; - console.debug('[Fleet:debug] Overview complete:', online, 'online,', overview.length - online, 'offline'); - } - res.json(overview); - } catch (error) { - console.error('[Fleet] Overview error:', error); - res.status(500).json({ error: 'Failed to fetch fleet overview' }); - } -}); - -// Paid-gated: detailed stack info per node -app.get('/api/fleet/node/:nodeId/stacks', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requirePaid(req, res)) return; - - try { - const nodeId = parseInt(req.params.nodeId as string, 10); - if (isNaN(nodeId)) { res.status(400).json({ error: 'Invalid node ID' }); return; } - const node = DatabaseService.getInstance().getNode(nodeId); - if (!node) { - res.status(404).json({ error: 'Node not found' }); - return; - } - - if (node.type === 'remote') { - if (!node.api_url || !node.api_token) { - res.status(503).json({ error: 'Remote node not configured' }); - return; - } - const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks`, { - headers: { Authorization: `Bearer ${node.api_token}` }, - signal: AbortSignal.timeout(10000), - }); - if (!response.ok) { - res.status(502).json({ error: 'Failed to fetch stacks from remote node' }); - return; - } - const stacks = await response.json(); - if (isDebugEnabled()) console.debug('[Fleet:debug] Node stacks:', nodeId, node.type, Array.isArray(stacks) ? stacks.length : 0, 'stacks'); - res.json(stacks); - return; - } - - const stacks = await FileSystemService.getInstance(nodeId).getStacks(); - if (isDebugEnabled()) console.debug('[Fleet:debug] Node stacks:', nodeId, node.type, stacks.length, 'stacks'); - res.json(stacks); - } catch (error) { - console.error('[Fleet] Node stacks error:', error); - res.status(500).json({ error: 'Failed to fetch node stacks' }); - } -}); - -// Paid-gated: container details for a specific stack on a specific node -app.get('/api/fleet/node/:nodeId/stacks/:stackName/containers', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requirePaid(req, res)) return; - - try { - const nodeId = parseInt(req.params.nodeId as string, 10); - if (isNaN(nodeId)) { res.status(400).json({ error: 'Invalid node ID' }); return; } - const stackName = req.params.stackName as string; - if (!isValidStackName(stackName)) { - res.status(400).json({ error: 'Invalid stack name' }); - return; - } - const node = DatabaseService.getInstance().getNode(nodeId); - if (!node) { - res.status(404).json({ error: 'Node not found' }); - return; - } - - if (node.type === 'remote') { - if (!node.api_url || !node.api_token) { - res.status(503).json({ error: 'Remote node not configured' }); - return; - } - const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, { - headers: { Authorization: `Bearer ${node.api_token}` }, - signal: AbortSignal.timeout(10000), - }); - if (!response.ok) { - res.status(502).json({ error: 'Failed to fetch containers from remote node' }); - return; - } - const containers = await response.json(); - res.json(containers); - return; - } - - const dockerController = DockerController.getInstance(nodeId); - const containers = await dockerController.getContainersByStack(stackName); - if (isDebugEnabled()) console.debug('[Fleet:debug] Stack containers:', nodeId, stackName, containers.length, 'containers'); - res.json(containers); - } catch (error) { - console.error('[Fleet] Node stack containers error:', error); - res.status(500).json({ error: 'Failed to fetch stack containers' }); - } -}); - -// Fleet Update Status: returns version comparison and active update status for all nodes. -app.get('/api/fleet/update-status', authMiddleware, async (_req: Request, res: Response): Promise => { - if (!requirePaid(_req, res)) return; - try { - const db = DatabaseService.getInstance(); - const nodes = db.getNodes(); - const gatewayVersion = getSenchoVersion(); - const gatewayValid = isValidVersion(gatewayVersion); - - const { latestVersion, latestValid, compareVersion, compareValid } = await getCompareTarget(gatewayVersion); - const debug = isDebugEnabled(); - - const results = await Promise.allSettled( - nodes.map(async (node) => { - const tracker = updateTracker.get(node.id); - - let version: string | null = null; - let remoteStartedAt: number | null = null; - let remoteUpdateError: string | null = null; - let remoteOnline = false; - if (node.type === 'local') { - version = gatewayVersion; - } else if (node.api_url && node.api_token) { - const meta = await fetchRemoteMeta(node.api_url, node.api_token); - version = meta.version; - remoteStartedAt = meta.startedAt; - remoteUpdateError = meta.updateError; - remoteOnline = meta.online; - } - - // For nodes actively updating, check if they've come back - if (tracker?.status === 'updating') { - const elapsed = Date.now() - tracker.startedAt; - - if (debug) { - console.debug('[Fleet:debug] Polling update status for node', node.id, node.name, '- elapsed:', Math.round(elapsed / 1000) + 's', 'version:', version, 'wasOffline:', tracker.wasOffline, 'remoteOnline:', remoteOnline); - } - - if (elapsed > UPDATE_TIMEOUT_MS) { - // Final timeout (5 min) - if (debug) console.debug('[Fleet:debug] Node', node.id, 'timed out after', Math.round(elapsed / 1000) + 's'); - updateTracker.set(node.id, updateTracker.resolve(tracker, 'timeout', UPDATE_TIMEOUT_MSG)); - } else if (node.type === 'remote') { - if (remoteUpdateError) { - // Remote reported a pull failure via /api/meta - if (debug) console.debug('[Fleet:debug] Node', node.id, 'reported pull failure:', remoteUpdateError); - updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', remoteUpdateError)); - } else if (!remoteOnline) { - // Node is unreachable (restarting); record that it went offline - if (!tracker.wasOffline) { - if (debug) console.debug('[Fleet:debug] Node', node.id, 'went offline (restarting)'); - updateTracker.set(node.id, { ...tracker, wasOffline: true }); - } - } else if (version !== tracker.previousVersion) { - // Signal 1: Version changed (or version now resolvable after being unknown) - if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 1 (version changed):', tracker.previousVersion, '->', version); - updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); - } else if ( - remoteStartedAt !== null && - tracker.previousProcessStart !== null && - remoteStartedAt !== tracker.previousProcessStart - ) { - // Signal 2: Process restarted (startedAt changed) - if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 2 (process restarted):', tracker.previousProcessStart, '->', remoteStartedAt); - updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); - } else if (tracker.wasOffline && remoteOnline) { - // Signal 3: Node went offline and is back online (container was recreated) - if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 3 (offline then online)'); - updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); - } else if ( - elapsed > 15_000 && - isValidVersion(version) && - gatewayValid && - !semver.lt(version, compareVersion!) - ) { - // Signal 4: Remote is now at or above gateway version (after minimum processing time). - // Catches fast restarts where the 5s polling interval misses the offline window - // and startedAt hasn't been observed to change yet. - if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 4 (version >= compare target):', version, '>=', compareVersion); - updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); - } else if (elapsed > EARLY_FAIL_MS) { - // Heuristic: node never went offline and nothing changed after 3 min - if (debug) console.debug('[Fleet:debug] Node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's - no signals detected'); - updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', 'Update may have failed. The node is still running and its version has not changed.')); - } - } else if (node.type === 'local') { - // Local node has only two failure signals: an explicit pull/spawn error, - // or the early-fail heuristic. Success is observed by the frontend overlay - // (it reloads the page when /api/health reports a new startedAt), at which - // point the new process starts with an empty tracker map. - const selfUpdate = SelfUpdateService.getInstance(); - const localError = selfUpdate.getLastError(); - if (localError) { - if (debug) console.debug('[Fleet:debug] Local node', node.id, 'update failed:', localError); - updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', localError)); - selfUpdate.clearLastError(); - } else if (elapsed > EARLY_FAIL_MS) { - // Helper container likely failed silently. Surface failure before the 5 min timeout. - if (debug) console.debug('[Fleet:debug] Local node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's'); - updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', 'Local update did not complete. The container may not have restarted; check Docker logs on the host.')); - } - } - } - - // Auto-expire completed entries 60s after they resolved so the badge is visible - if (tracker?.status === 'completed' && tracker.resolvedAt && Date.now() - tracker.resolvedAt > 60_000) { - updateTracker.delete(node.id); - } - - // Assume remote nodes are outdated when their version is unresolvable - let updateAvailable = false; - if (!isValidVersion(version)) { - updateAvailable = node.type === 'remote'; - } else if (compareValid) { - updateAvailable = semver.lt(version, compareVersion!); - } - - const currentTracker = updateTracker.get(node.id); - return { - nodeId: node.id, - name: node.name, - type: node.type, - version, - latestVersion: latestValid ? latestVersion : gatewayVersion, - updateAvailable, - updateStatus: currentTracker?.status ?? null, - error: currentTracker?.error ?? null, - }; - }) - ); - - const nodeStatuses = results.map((r, i) => { - if (r.status === 'fulfilled') return r.value; - return { - nodeId: nodes[i].id, - name: nodes[i].name, - type: nodes[i].type, - version: null, - latestVersion: latestValid ? latestVersion : gatewayVersion, - updateAvailable: false, - updateStatus: null, - error: null, - }; - }); - - if (isDebugEnabled()) { - const trackerStates = Array.from(updateTracker.entries()).map(([nid, t]) => `${nid}:${t.status}`); - console.debug('[Fleet:debug] Update status:', nodeStatuses.length, 'nodes, trackers:', trackerStates.join(', ') || 'none'); - } - res.json({ nodes: nodeStatuses }); - } catch (error) { - console.error('[Fleet] Update status error:', error); - res.status(500).json({ error: 'Failed to fetch update status' }); - } -}); - -// Trigger update on a specific node -app.post('/api/fleet/nodes/:nodeId/update', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requirePaid(req, res)) return; - if (!requireAdmin(req, res)) return; - try { - const nodeId = parseInt(req.params.nodeId as string, 10); - if (isNaN(nodeId)) { res.status(400).json({ error: 'Invalid node ID' }); return; } - const db = DatabaseService.getInstance(); - const node = db.getNode(nodeId); - if (!node) { - res.status(404).json({ error: 'Node not found' }); - return; - } - - const existing = updateTracker.get(nodeId); - if (existing?.status === 'updating') { - if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) { - updateTracker.set(nodeId, updateTracker.resolve(existing, 'timeout', UPDATE_TIMEOUT_MSG)); - } else { - res.status(409).json({ error: 'Update already in progress for this node.' }); - return; - } - } - // Clear terminal states to allow retry - if (existing && (existing.status === 'timeout' || existing.status === 'failed' || existing.status === 'completed')) { - updateTracker.delete(nodeId); - } - - console.log('[Fleet] Update triggered for node', node.name, node.type); - if (isDebugEnabled()) { - console.debug('[Fleet:debug] Update trigger details:', { nodeId, name: node.name, type: node.type, hasUrl: !!node.api_url, hasToken: !!node.api_token }); - } - - if (node.type === 'local') { - if (!SelfUpdateService.getInstance().isAvailable()) { - res.status(503).json({ error: 'Self-update unavailable on the local node.' }); - return; - } - updateTracker.set(nodeId, updateTracker.create('updating', getSenchoVersion(), null)); - scheduleLocalUpdate(res, 'Update initiated on local node. The server will restart shortly.'); - return; - } - - // Remote node - if (!node.api_url || !node.api_token) { - res.status(503).json({ error: 'Remote node not configured.' }); - return; - } - - // Check remote availability and capabilities - const meta = await fetchRemoteMeta(node.api_url, node.api_token); - if (isDebugEnabled()) { - console.debug('[Fleet:debug] Remote meta for update:', { nodeId, online: meta.online, version: meta.version, capabilities: meta.capabilities, startedAt: meta.startedAt }); - } - if (!meta.online) { - res.status(503).json({ error: 'Remote node is unreachable. Verify the node is running and the API URL is correct.' }); - return; - } - if (!meta.capabilities.includes('self-update')) { - res.status(503).json({ error: 'Remote node does not support self-update. It may need to be updated manually first.' }); - return; - } - - // Trigger remote update - const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/system/update`, { - method: 'POST', - headers: { - Authorization: `Bearer ${node.api_token}`, - 'Content-Type': 'application/json', - }, - signal: AbortSignal.timeout(10000), - }); - - if (!response.ok) { - const err = await response.json().catch(() => ({})); - const errorMsg = (err as Record)?.error || 'Remote node rejected update request.'; - updateTracker.set(nodeId, updateTracker.create('failed', meta.version, meta.startedAt, errorMsg)); - res.status(502).json({ error: errorMsg }); - return; - } - - updateTracker.set(nodeId, updateTracker.create('updating', meta.version, meta.startedAt)); - res.status(202).json({ message: `Update initiated on ${node.name}.` }); - } catch (error) { - console.error('[Fleet] Node update error:', error); - const errorMsg = (error as Error)?.message || 'Failed to trigger node update.'; - const failedNodeId = parseInt(req.params.nodeId as string, 10); - if (!isNaN(failedNodeId)) { - updateTracker.set(failedNodeId, updateTracker.create('failed', null, null, errorMsg)); - } - res.status(500).json({ error: 'Failed to trigger node update.' }); - } -}); - -// Trigger update on all outdated nodes -app.post('/api/fleet/update-all', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requirePaid(req, res)) return; - if (!requireAdmin(req, res)) return; - try { - const db = DatabaseService.getInstance(); - const nodes = db.getNodes(); - const gatewayVersion = getSenchoVersion(); - const { compareVersion, compareValid } = await getCompareTarget(gatewayVersion); - - const debug = isDebugEnabled(); - console.log('[Fleet] Update-all triggered,', nodes.length, 'nodes registered'); - if (debug) console.debug('[Fleet:debug] Update-all compare target:', { gatewayVersion, compareVersion, compareValid }); - - // Filter to eligible candidates, then trigger all in parallel - const candidates = nodes.filter(node => { - if (node.type === 'local') return false; - const tracker = updateTracker.get(node.id); - if (tracker?.status === 'updating') return false; - if (!node.api_url || !node.api_token) return false; - // Clear terminal states so they can be re-triggered - if (tracker && (tracker.status === 'timeout' || tracker.status === 'failed' || tracker.status === 'completed')) { - updateTracker.delete(node.id); - } - return true; - }); - - const results = await Promise.allSettled(candidates.map(async (node) => { - const meta = await fetchRemoteMeta(node.api_url!, node.api_token!); - if (!meta.online) { - return { name: node.name, triggered: false }; - } - if (!meta.capabilities.includes('self-update')) { - return { name: node.name, triggered: false }; - } - if (isValidVersion(meta.version) && compareValid && !semver.lt(meta.version, compareVersion!)) { - return { name: node.name, triggered: false }; - } - const response = await fetch(`${node.api_url!.replace(/\/$/, '')}/api/system/update`, { - method: 'POST', - headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' }, - signal: AbortSignal.timeout(10000), - }); - if (response.ok) { - updateTracker.set(node.id, updateTracker.create('updating', meta.version, meta.startedAt)); - return { name: node.name, triggered: true }; - } - return { name: node.name, triggered: false }; - })); - - const updating: string[] = []; - const skipped = nodes.filter(n => !candidates.includes(n)).map(n => n.name); - for (let i = 0; i < results.length; i++) { - const r = results[i]; - const val = r.status === 'fulfilled' ? r.value : { name: candidates[i].name, triggered: false }; - (val.triggered ? updating : skipped).push(val.name); - } - - if (debug) console.debug('[Fleet:debug] Update-all results:', { updating, skippedCount: skipped.length, candidateCount: candidates.length }); - res.status(202).json({ updating, skipped }); - } catch (error) { - console.error('[Fleet] Update all error:', error); - res.status(500).json({ error: 'Failed to trigger fleet update.' }); - } -}); - -// Clear update tracker entry for a specific node (dismiss or before retry) -app.delete('/api/fleet/nodes/:nodeId/update-status', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requirePaid(req, res)) return; - try { - const nodeId = parseInt(req.params.nodeId as string, 10); - if (isNaN(nodeId)) { res.status(400).json({ error: 'Invalid node ID' }); return; } - const node = DatabaseService.getInstance().getNode(nodeId); - if (!node) { - res.status(404).json({ error: 'Node not found' }); - return; - } - updateTracker.delete(nodeId); - res.status(204).send(); - } catch (error) { - console.error('[Fleet] Clear update status error:', error); - res.status(500).json({ error: 'Failed to clear update status.' }); - } -}); - -// Clear all terminal (timed-out, failed, completed) tracker entries at once -app.delete('/api/fleet/update-status', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requirePaid(req, res)) return; - // Pre-fetch fresh latest version so the next GET has up-to-date data - if (req.query.recheck === 'true') { - await getLatestVersion(true); - } - for (const [nodeId, tracker] of updateTracker.entries()) { - if (tracker.status === 'timeout' || tracker.status === 'failed' || tracker.status === 'completed') { - updateTracker.delete(nodeId); - } - } - res.status(204).send(); -}); - -async function fetchLocalNodeOverview(node: Node): Promise { - try { - const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(node.id)); - const [allContainers, stacks, currentLoad, mem, fsSize] = await Promise.all([ - DockerController.getInstance(node.id).getAllContainers(), - FileSystemService.getInstance(node.id).getStacks(), - si.currentLoad(), - si.mem(), - si.fsSize(), - ]); - - const isManagedByComposeDir = (c: Dockerode.ContainerInfo): boolean => { - const workingDir: string | undefined = c.Labels?.['com.docker.compose.project.working_dir']; - if (!workingDir) return false; - const resolved = path.resolve(workingDir); - return resolved === composeDir || resolved.startsWith(composeDir + path.sep); - }; - - const containers = allContainers as Dockerode.ContainerInfo[]; - const active = containers.filter(c => c.State === 'running').length; - const exited = containers.filter(c => c.State === 'exited').length; - const total = containers.length; - const managed = containers.filter(c => c.State === 'running' && isManagedByComposeDir(c)).length; - const unmanaged = containers.filter(c => c.State === 'running' && !isManagedByComposeDir(c)).length; - - const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0]; - - return { - id: node.id, - name: node.name, - type: node.type, - status: 'online', - stats: { active, managed, unmanaged, exited, total }, - systemStats: { - cpu: { usage: currentLoad.currentLoad.toFixed(1), cores: currentLoad.cpus.length }, - memory: { - total: mem.total, - used: mem.used, - free: mem.free, - usagePercent: ((mem.used / mem.total) * 100).toFixed(1), - }, - disk: mainDisk ? { - total: mainDisk.size, - used: mainDisk.used, - free: mainDisk.available, - usagePercent: mainDisk.use ? mainDisk.use.toFixed(1) : '0', - } : null, - }, - stacks, - }; - } catch (error) { - console.error(`[Fleet] Local node ${node.name} error:`, error); - return { - id: node.id, name: node.name, type: node.type, status: 'offline', - stats: null, systemStats: null, stacks: null, - }; - } -} - -async function fetchRemoteNodeOverview(node: Node): Promise { - if (!node.api_url || !node.api_token) { - return { - id: node.id, name: node.name, type: node.type, status: 'offline', - stats: null, systemStats: null, stacks: null, - }; - } - - const baseUrl = node.api_url.replace(/\/$/, ''); - const headers = { Authorization: `Bearer ${node.api_token}` }; - - try { - const [statsRes, systemStatsRes, stacksRes] = await Promise.allSettled([ - fetch(`${baseUrl}/api/stats`, { headers, signal: AbortSignal.timeout(10000) }), - fetch(`${baseUrl}/api/system/stats`, { headers, signal: AbortSignal.timeout(10000) }), - fetch(`${baseUrl}/api/stacks`, { headers, signal: AbortSignal.timeout(10000) }), - ]); - - interface RemoteSystemStats { - cpu: { usage: string; cores: number }; - memory: { total: number; used: number; free: number; usagePercent: string }; - disk?: { total: number; used: number; free: number; usagePercent: string } | null; - } - - const stats: FleetNodeOverview['stats'] | null = statsRes.status === 'fulfilled' && statsRes.value.ok - ? await statsRes.value.json() as FleetNodeOverview['stats'] : null; - const systemStatsRaw: RemoteSystemStats | null = systemStatsRes.status === 'fulfilled' && systemStatsRes.value.ok - ? await systemStatsRes.value.json() as RemoteSystemStats : null; - const stacks: string[] | null = stacksRes.status === 'fulfilled' && stacksRes.value.ok - ? await stacksRes.value.json() as string[] : null; - - const systemStats: FleetNodeOverview['systemStats'] | null = systemStatsRaw ? { - cpu: systemStatsRaw.cpu, - memory: systemStatsRaw.memory, - disk: systemStatsRaw.disk ? { - total: systemStatsRaw.disk.total, - used: systemStatsRaw.disk.used, - free: systemStatsRaw.disk.free, - usagePercent: systemStatsRaw.disk.usagePercent, - } : null, - } : null; - - return { - id: node.id, - name: node.name, - type: node.type, - status: stats || systemStats ? 'online' : 'offline', - stats, - systemStats, - stacks, - }; - } catch (error) { - console.error(`[Fleet] Remote node ${node.name} error:`, error); - return { - id: node.id, name: node.name, type: node.type, status: 'offline', - stats: null, systemStats: null, stacks: null, - }; - } -} - -// ─── Fleet Snapshots (Skipper+) ─── - -// Create fleet snapshot -app.post('/api/fleet/snapshots', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; - - try { - const { description = '' } = req.body; - if (typeof description === 'string' && description.length > 500) { - res.status(400).json({ error: 'Description must be 500 characters or less' }); - return; - } - const db = DatabaseService.getInstance(); - const nodes = db.getNodes(); - const username = req.user?.username || 'admin'; - - const captureStart = Date.now(); - const results = await Promise.allSettled( - nodes.map(async (node) => { - if (node.type === 'remote') { - return captureRemoteNodeFiles(node); - } - return captureLocalNodeFiles(node); - }) - ); - - const capturedNodes: SnapshotNodeData[] = []; - const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = []; - - results.forEach((result, i) => { - if (result.status === 'fulfilled') { - capturedNodes.push(result.value); - } else { - console.error(`[Fleet Snapshot] Failed to capture node ${nodes[i].name}:`, result.reason); - skippedNodes.push({ - nodeId: nodes[i].id, - nodeName: nodes[i].name, - reason: result.reason instanceof Error ? result.reason.message : 'Unknown error', - }); - } - }); - - let totalStacks = 0; - const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = []; - - for (const nodeData of capturedNodes) { - totalStacks += nodeData.stacks.length; - for (const stack of nodeData.stacks) { - for (const file of stack.files) { - allFiles.push({ - nodeId: nodeData.nodeId, - nodeName: nodeData.nodeName, - stackName: stack.stackName, - filename: file.filename, - content: file.content, - }); - } - } - } - - const snapshotId = db.createSnapshot( - description, - username, - capturedNodes.length, - totalStacks, - JSON.stringify(skippedNodes), - ); - - if (allFiles.length > 0) { - db.insertSnapshotFiles(snapshotId, allFiles); - } - - console.log('[Fleet] Snapshot created:', capturedNodes.length, 'nodes,', totalStacks, 'stacks'); - if (isDebugEnabled()) { - console.debug(`[Fleet:debug] Snapshot ${snapshotId} capture completed in ${Date.now() - captureStart}ms, ${allFiles.length} file(s) stored`); - for (const skip of skippedNodes) { - console.debug(`[Fleet:debug] Skipped node "${skip.nodeName}" (id=${skip.nodeId}): ${skip.reason}`); - } - } - const snapshot = db.getSnapshot(snapshotId); - res.status(201).json(snapshot); - } catch (error) { - console.error('[Fleet Snapshot] Create error:', error); - res.status(500).json({ error: 'Failed to create fleet snapshot' }); - } -}); - -// List fleet snapshots -app.get('/api/fleet/snapshots', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requirePaid(req, res)) return; - - try { - const limit = Math.min(parseInt(req.query.limit as string, 10) || 50, 100); - const offset = parseInt(req.query.offset as string, 10) || 0; - const db = DatabaseService.getInstance(); - const snapshots = db.getSnapshots(limit, offset); - const total = db.getSnapshotCount(); - if (isDebugEnabled()) console.debug('[Fleet:debug] Snapshots list: limit=', limit, 'offset=', offset, 'total=', total); - res.json({ snapshots, total }); - } catch (error) { - console.error('[Fleet Snapshot] List error:', error); - res.status(500).json({ error: 'Failed to list fleet snapshots' }); - } -}); - -// Get snapshot detail -app.get('/api/fleet/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requirePaid(req, res)) return; - - try { - const id = parseInt(req.params.id as string, 10); - if (isNaN(id)) { res.status(400).json({ error: 'Invalid snapshot ID' }); return; } - const db = DatabaseService.getInstance(); - const snapshot = db.getSnapshot(id); - if (!snapshot) { - res.status(404).json({ error: 'Snapshot not found' }); - return; - } - - const files = db.getSnapshotFiles(id); - - // Group files by node and stack - const nodesMap = new Map> }>(); - for (const file of files) { - if (!nodesMap.has(file.node_id)) { - nodesMap.set(file.node_id, { nodeId: file.node_id, nodeName: file.node_name, stacks: new Map() }); - } - const nodeEntry = nodesMap.get(file.node_id)!; - if (!nodeEntry.stacks.has(file.stack_name)) { - nodeEntry.stacks.set(file.stack_name, []); - } - nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, content: file.content }); - } - - const nodes = Array.from(nodesMap.values()).map(n => ({ - nodeId: n.nodeId, - nodeName: n.nodeName, - stacks: Array.from(n.stacks.entries()).map(([stackName, stackFiles]) => ({ - stackName, - files: stackFiles, - })), - })); - - if (isDebugEnabled()) console.debug('[Fleet:debug] Snapshot detail:', id, files.length, 'files'); - res.json({ ...snapshot, nodes }); - } catch (error) { - console.error('[Fleet Snapshot] Detail error:', error); - res.status(500).json({ error: 'Failed to fetch snapshot details' }); - } -}); - -// Restore a stack from snapshot -app.post('/api/fleet/snapshots/:id/restore', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; - - try { - const snapshotId = parseInt(req.params.id as string, 10); - if (isNaN(snapshotId)) { res.status(400).json({ error: 'Invalid snapshot ID' }); return; } - const { nodeId, stackName, redeploy = false } = req.body; - - if (!nodeId || !stackName) { - res.status(400).json({ error: 'nodeId and stackName are required' }); - return; - } - if (!isValidStackName(stackName)) { - res.status(400).json({ error: 'Invalid stack name' }); - return; - } - - const db = DatabaseService.getInstance(); - const snapshot = db.getSnapshot(snapshotId); - if (!snapshot) { - res.status(404).json({ error: 'Snapshot not found' }); - return; - } - - const files = db.getSnapshotStackFiles(snapshotId, nodeId, stackName); - if (files.length === 0) { - res.status(404).json({ error: 'No files found for this stack in the snapshot' }); - return; - } - - if (isDebugEnabled()) { - const fileNames = files.map(f => f.filename).join(', '); - console.debug(`[Fleet:debug] Restore: snapshot=${snapshotId}, node=${nodeId}, stack="${stackName}", files=[${fileNames}], redeploy=${redeploy}`); - } - - const node = db.getNode(nodeId); - if (!node) { - res.status(404).json({ error: 'Target node no longer exists' }); - return; - } - - if (node.type === 'local') { - const fsService = FileSystemService.getInstance(node.id); - - // Backup current files before restore - try { - await fsService.backupStackFiles(stackName); - } catch (e) { - // Stack may not exist yet before first restore; that is ok. - console.warn(`[Fleet Snapshot] Pre-restore backup failed for stack "${stackName}" (may not exist yet):`, (e as Error).message); - } - - for (const file of files) { - if (file.filename === 'compose.yaml') { - await fsService.saveStackContent(stackName, file.content); - } else if (file.filename === '.env') { - await fsService.saveEnvContent(stackName, file.content); - } - } - - if (redeploy) { - const composeService = ComposeService.getInstance(node.id); - await composeService.deployStack(stackName); - } - } else { - // Remote node - if (!node.api_url || !node.api_token) { - res.status(503).json({ error: 'Remote node not configured' }); - return; - } - - const baseUrl = node.api_url.replace(/\/$/, ''); - const headers: Record = { - Authorization: `Bearer ${node.api_token}`, - 'Content-Type': 'application/json', - }; - - for (const file of files) { - if (file.filename === 'compose.yaml') { - const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, { - method: 'PUT', - headers, - body: JSON.stringify({ content: file.content }), - signal: AbortSignal.timeout(15000), - }); - if (!putRes.ok) throw new Error('Failed to restore compose file on remote node'); - } else if (file.filename === '.env') { - const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, { - method: 'PUT', - headers, - body: JSON.stringify({ content: file.content }), - signal: AbortSignal.timeout(15000), - }); - if (!putRes.ok) throw new Error('Failed to restore env file on remote node'); - } - } - - if (redeploy) { - await fetch(`${baseUrl}/api/compose/${encodeURIComponent(stackName)}/up`, { - method: 'POST', - headers, - signal: AbortSignal.timeout(30000), - }); - } - } - - console.log('[Fleet] Snapshot restore:', snapshotId, 'node=', nodeId, 'stack=', stackName); - res.json({ message: 'Stack restored successfully', redeployed: redeploy }); - } catch (error) { - console.error('[Fleet Snapshot] Restore error:', error); - res.status(500).json({ error: 'Failed to restore stack from snapshot' }); - } -}); - -// Delete snapshot -app.delete('/api/fleet/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; - - try { - const id = parseInt(req.params.id as string, 10); - if (isNaN(id)) { res.status(400).json({ error: 'Invalid snapshot ID' }); return; } - const db = DatabaseService.getInstance(); - const snapshot = db.getSnapshot(id); - if (!snapshot) { - res.status(404).json({ error: 'Snapshot not found' }); - return; - } - if (isDebugEnabled()) { - console.debug(`[Fleet:debug] Deleting snapshot ${id} (${snapshot.node_count} node(s), ${snapshot.stack_count} stack(s))`); - } - db.deleteSnapshot(id); - console.log('[Fleet] Snapshot deleted:', id); - res.json({ message: 'Snapshot deleted' }); - } catch (error) { - console.error('[Fleet Snapshot] Delete error:', error); - res.status(500).json({ error: 'Failed to delete snapshot' }); - } -}); - -// ─── Webhooks (Skipper+) ─── CRUD requires auth + paid tier, trigger is public with HMAC ─── - -// Webhook CRUD (auth + paid tier required) -app.get('/api/webhooks', authMiddleware, async (_req: Request, res: Response): Promise => { - if (!requirePaid(_req, res)) return; - try { - const webhooks = DatabaseService.getInstance().getWebhooks(); - const svc = WebhookService.getInstance(); - res.json(webhooks.map(w => ({ ...w, secret: svc.maskSecret(w.secret) }))); - } catch (error) { - console.error('[Webhooks] List error:', error); - res.status(500).json({ error: 'Failed to list webhooks' }); - } -}); - -app.post('/api/webhooks', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; - try { - const { name, stack_name, action, enabled } = req.body; - if (!name || !stack_name || !action) { - res.status(400).json({ error: 'name, stack_name, and action are required' }); - return; - } - const validActions = ['deploy', 'restart', 'stop', 'start', 'pull', 'git-pull']; - if (!validActions.includes(action)) { - res.status(400).json({ error: `action must be one of: ${validActions.join(', ')}` }); - return; - } - if (action === 'git-pull' && !GitSourceService.getInstance().get(stack_name)) { - res.status(400).json({ error: 'Configure a Git source for this stack before creating a git-pull webhook' }); - return; - } - - const svc = WebhookService.getInstance(); - const secret = svc.generateSecret(); - const id = DatabaseService.getInstance().addWebhook({ - name, stack_name, action, secret, enabled: enabled !== false, - }); - - // Return the full secret only on creation - res.status(201).json({ id, secret }); - } catch (error) { - console.error('[Webhooks] Create error:', error); - res.status(500).json({ error: 'Failed to create webhook' }); - } -}); - -app.put('/api/webhooks/:id', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; - try { - const id = parseInt(req.params.id as string, 10); - const webhook = DatabaseService.getInstance().getWebhook(id); - if (!webhook) { res.status(404).json({ error: 'Webhook not found' }); return; } - - const { name, stack_name, action, enabled } = req.body; - const validActions = ['deploy', 'restart', 'stop', 'start', 'pull', 'git-pull']; - if (action && !validActions.includes(action)) { - res.status(400).json({ error: `action must be one of: ${validActions.join(', ')}` }); - return; - } - if (action === 'git-pull') { - const targetStack = stack_name || webhook.stack_name; - if (!GitSourceService.getInstance().get(targetStack)) { - res.status(400).json({ error: 'Configure a Git source for this stack before enabling a git-pull webhook' }); - return; - } - } - - DatabaseService.getInstance().updateWebhook(id, { name, stack_name, action, enabled }); - res.json({ success: true }); - } catch (error) { - console.error('[Webhooks] Update error:', error); - res.status(500).json({ error: 'Failed to update webhook' }); - } -}); - -app.delete('/api/webhooks/:id', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; - try { - const id = parseInt(req.params.id as string, 10); - DatabaseService.getInstance().deleteWebhook(id); - res.json({ success: true }); - } catch (error) { - console.error('[Webhooks] Delete error:', error); - res.status(500).json({ error: 'Failed to delete webhook' }); - } -}); - -app.get('/api/webhooks/:id/history', authMiddleware, async (req: Request, res: Response): Promise => { - if (!requirePaid(req, res)) return; - try { - const id = parseInt(req.params.id as string, 10); - const executions = DatabaseService.getInstance().getWebhookExecutions(id); - res.json(executions); - } catch (error) { - console.error('[Webhooks] History error:', error); - res.status(500).json({ error: 'Failed to fetch webhook history' }); - } -}); - -// Webhook trigger - public endpoint, authenticated via HMAC signature -app.post('/api/webhooks/:id/trigger', webhookTriggerLimiter, async (req: Request, res: Response): Promise => { - try { - const id = parseInt(req.params.id as string, 10); - const db = DatabaseService.getInstance(); - const webhook = db.getWebhook(id); - - if (!webhook || !webhook.enabled) { - res.status(404).json({ error: 'Webhook not found or disabled' }); - return; - } - - // Paid tier gate - trigger only works with an active Skipper or Admiral license - if (LicenseService.getInstance().getTier() !== 'paid') { - res.status(403).json({ error: 'This feature requires a Skipper or Admiral license.', code: 'PAID_REQUIRED' }); - return; - } - - // Validate HMAC signature - const signature = req.headers['x-webhook-signature'] as string; - if (!signature) { - res.status(401).json({ error: 'Missing X-Webhook-Signature header' }); - return; - } - - const rawBody = req.rawBody?.toString('utf-8') ?? JSON.stringify(req.body ?? {}); - const svc = WebhookService.getInstance(); - if (!svc.validateSignature(rawBody, webhook.secret, signature)) { - res.status(401).json({ error: 'Invalid signature' }); - return; - } - - // Use action from body if provided, otherwise use webhook default - const action = req.body?.action || webhook.action; - const triggerSource = req.headers['user-agent'] || req.ip || null; - - // Execute asynchronously - return 202 immediately - res.status(202).json({ message: 'Webhook accepted', action }); - - const atomic = LicenseService.getInstance().getTier() === 'paid'; - svc.execute(id, action, triggerSource, atomic).catch(err => { - console.error(`[Webhooks] Execution error for webhook ${id}:`, err); - }); - } catch (error) { - console.error('[Webhooks] Trigger error:', error); - res.status(500).json({ error: 'Failed to process webhook' }); - } -}); - -// --- User Management (local-only, admin + paid tier gated for creation) --- - -app.get('/api/users', authMiddleware, async (req: Request, res: Response): Promise => { - if (req.apiTokenScope) { - res.status(403).json({ error: 'API tokens cannot access user management.', code: 'SCOPE_DENIED' }); - return; - } - if (!requireAdmin(req, res)) return; - try { - const db = DatabaseService.getInstance(); - const users = db.getUsers(); - const mfaUserIds = db.getUsersWithMfaEnabled(); - const enriched = users.map((u) => ({ - ...u, - mfaEnabled: mfaUserIds.has(u.id), - })); - res.json(enriched); - } catch (error) { - console.error('[Users] List error:', error); - res.status(500).json({ error: 'Failed to fetch users' }); - } -}); - -app.post('/api/users', authMiddleware, async (req: Request, res: Response): Promise => { - if (req.apiTokenScope) { - res.status(403).json({ error: 'API tokens cannot access user management.', code: 'SCOPE_DENIED' }); - return; - } - if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; - try { - const { username, password, role } = req.body; - - if (!username || !password || !role) { - res.status(400).json({ error: 'Username, password, and role are required' }); - return; - } - if (typeof username !== 'string' || username.length < 3 || !/^[a-zA-Z0-9_-]+$/.test(username)) { - res.status(400).json({ error: 'Username must be at least 3 characters (letters, numbers, underscore, hyphen)' }); - return; - } - if (typeof password !== 'string' || password.length < MIN_PASSWORD_LENGTH) { - res.status(400).json({ error: `Password must be at least ${MIN_PASSWORD_LENGTH} characters` }); - return; - } - const validRoles: UserRole[] = ['admin', 'viewer', 'deployer', 'node-admin', 'auditor']; - if (!validRoles.includes(role)) { - res.status(400).json({ error: 'Role must be "admin", "viewer", "deployer", "node-admin", or "auditor"' }); - return; - } - if ((role === 'deployer' || role === 'node-admin' || role === 'auditor') && !requireAdmiral(req, res)) return; - - const db = DatabaseService.getInstance(); - const existing = db.getUserByUsername(username); - if (existing) { - res.status(409).json({ error: 'A user with this username already exists' }); - return; - } - - // Enforce seat limits based on license variant - const seatLimits = LicenseService.getInstance().getSeatLimits(); - if (role === 'admin' && seatLimits.maxAdmins !== null && db.getAdminCount() >= seatLimits.maxAdmins) { - res.status(403).json({ error: `Your license allows a maximum of ${seatLimits.maxAdmins} admin account${seatLimits.maxAdmins === 1 ? '' : 's'}. Upgrade to Admiral for unlimited accounts.` }); - return; - } - if (role !== 'admin' && seatLimits.maxViewers !== null && db.getNonAdminCount() >= seatLimits.maxViewers) { - res.status(403).json({ error: `Your license allows a maximum of ${seatLimits.maxViewers} viewer account${seatLimits.maxViewers === 1 ? '' : 's'}. Upgrade to Admiral for unlimited accounts.` }); - return; - } - - - const passwordHash = await bcrypt.hash(password, 10); - const id = db.addUser({ username, password_hash: passwordHash, role }); - console.log('[Users] Created:', username, 'role:', role, 'by:', req.user!.username); - res.status(201).json({ id, username, role }); - } catch (error) { - console.error('[Users] Create error:', error); - res.status(500).json({ error: 'Failed to create user' }); - } -}); - -// Note: requirePaid is intentionally not enforced on PUT/DELETE user endpoints. -// Admins must be able to manage existing users even if their license lapses (security consideration). -app.put('/api/users/:id', authMiddleware, async (req: Request, res: Response): Promise => { - if (req.apiTokenScope) { - res.status(403).json({ error: 'API tokens cannot access user management.', code: 'SCOPE_DENIED' }); - return; - } - if (!requireAdmin(req, res)) return; - try { - const id = parseInt(req.params.id as string, 10); - const db = DatabaseService.getInstance(); - const user = db.getUser(id); - if (!user) { - res.status(404).json({ error: 'User not found' }); - return; - } - - const { username, password, role } = req.body; - const updates: Partial<{ username: string; password_hash: string; role: string }> = {}; - - if (username !== undefined) { - if (typeof username !== 'string' || username.length < 3 || !/^[a-zA-Z0-9_-]+$/.test(username)) { - res.status(400).json({ error: 'Username must be at least 3 characters (letters, numbers, underscore, hyphen)' }); - return; - } - const existing = db.getUserByUsername(username); - if (existing && existing.id !== id) { - res.status(409).json({ error: 'A user with this username already exists' }); - return; - } - updates.username = username; - } - - if (role !== undefined) { - const validRoles: UserRole[] = ['admin', 'viewer', 'deployer', 'node-admin', 'auditor']; - if (!validRoles.includes(role)) { - res.status(400).json({ error: 'Role must be "admin", "viewer", "deployer", "node-admin", or "auditor"' }); - return; - } - if ((role === 'deployer' || role === 'node-admin' || role === 'auditor') && !requireAdmiral(req, res)) return; - // Prevent demoting yourself - if (user.username === req.user!.username && role !== user.role) { - res.status(400).json({ error: 'Cannot change your own role' }); - return; - } - // Prevent removing the last admin - if (user.role === 'admin' && role !== 'admin' && db.getAdminCount() <= 1) { - res.status(400).json({ error: 'Cannot demote the only admin user' }); - return; - } - updates.role = role; - } - - if (password !== undefined) { - // Prevent setting passwords on SSO-provisioned users (would enable local login bypass) - if (user.auth_provider !== 'local') { - res.status(400).json({ error: 'Cannot set a password on an SSO-provisioned user.' }); - return; - } - if (typeof password !== 'string' || password.length < MIN_PASSWORD_LENGTH) { - res.status(400).json({ error: `Password must be at least ${MIN_PASSWORD_LENGTH} characters` }); - return; - } - updates.password_hash = await bcrypt.hash(password, 10); - } - - db.updateUser(id, updates); - // Invalidate the user's active sessions when their role or password changes - if (updates.role || updates.password_hash) { - db.bumpTokenVersion(id); - } - console.log('[Users] Updated user', id, 'fields:', Object.keys(updates).join(', '), 'by:', req.user!.username); - res.json({ success: true }); - } catch (error) { - console.error('[Users] Update error:', error); - res.status(500).json({ error: 'Failed to update user' }); - } -}); - -app.delete('/api/users/:id', authMiddleware, async (req: Request, res: Response): Promise => { - if (req.apiTokenScope) { - res.status(403).json({ error: 'API tokens cannot access user management.', code: 'SCOPE_DENIED' }); - return; - } - if (!requireAdmin(req, res)) return; - try { - const id = parseInt(req.params.id as string, 10); - const db = DatabaseService.getInstance(); - const user = db.getUser(id); - if (!user) { - res.status(404).json({ error: 'User not found' }); - return; - } - - // Cannot delete yourself - if (user.username === req.user!.username) { - res.status(400).json({ error: 'Cannot delete your own account' }); - return; - } - - // Cannot delete the last admin - if (user.role === 'admin' && db.getAdminCount() <= 1) { - res.status(400).json({ error: 'Cannot delete the only admin user' }); - return; - } - - db.deleteUser(id); - console.log('[Users] Deleted:', user.username, '(id:', id, ') by:', req.user!.username); - res.json({ success: true }); - } catch (error) { - console.error('[Users] Delete error:', error); - res.status(500).json({ error: 'Failed to delete user' }); - } -}); - -/** - * Admin reset: clear a target user's MFA enrolment and force re-auth. Used - * when a user has lost their authenticator AND exhausted their backup codes, - * and another admin is available. For total lockout (including sole admin), - * see the CLI `reset-mfa` command. - */ -app.post('/api/users/:id/mfa/reset', authMiddleware, (req: Request, res: Response): void => { - if (req.apiTokenScope) { - res.status(403).json({ error: 'API tokens cannot access user management.', code: 'SCOPE_DENIED' }); - return; - } - if (!requireAdmin(req, res)) return; - try { - const id = parseInt(req.params.id as string, 10); - if (!Number.isFinite(id)) { - res.status(400).json({ error: 'Invalid user id' }); - return; - } - const db = DatabaseService.getInstance(); - const target = db.getUser(id); - if (!target) { - res.status(404).json({ error: 'User not found' }); - return; - } - db.deleteUserMfa(id); - db.bumpTokenVersion(id); - try { - db.insertAuditLog({ - timestamp: Date.now(), - username: req.user!.username, - method: 'POST', - path: req.originalUrl, - status_code: 200, - node_id: null, - ip_address: req.ip || 'unknown', - summary: `Admin reset two-factor authentication for ${target.username}`, - }); - } catch (err) { - console.warn('[MFA] Admin reset audit log write failed:', (err as Error).message); - } - console.log('[MFA] Admin reset: target=', target.username, 'by=', req.user!.username); - if (isDebugEnabled()) { - console.log('[MFA:diag] admin-reset target=', target.username, 'actor=', req.user!.username); - } - res.json({ success: true }); - } catch (error: unknown) { - console.error('[MFA] Admin reset error:', (error as Error).message); - res.status(500).json({ error: 'Failed to reset two-factor authentication' }); - } -}); - -// --- Scoped Role Assignments (Admiral) --- - -app.get('/api/users/:id/roles', authMiddleware, (req: Request, res: Response): void => { - if (req.apiTokenScope) { - res.status(403).json({ error: 'API tokens cannot access user management.', code: 'SCOPE_DENIED' }); - return; - } - if (!requireAdmin(req, res)) return; - if (!requireAdmiral(req, res)) return; - try { - const userId = parseInt(req.params.id as string, 10); - const db = DatabaseService.getInstance(); - if (!db.getUser(userId)) { - res.status(404).json({ error: 'User not found' }); - return; - } - const assignments = db.getAllRoleAssignments(userId); - res.json(assignments); - } catch (error) { - console.error('[Roles] List error:', error); - res.status(500).json({ error: 'Failed to fetch role assignments' }); - } -}); - -app.post('/api/users/:id/roles', authMiddleware, (req: Request, res: Response): void => { - if (req.apiTokenScope) { - res.status(403).json({ error: 'API tokens cannot access user management.', code: 'SCOPE_DENIED' }); - return; - } - if (!requireAdmin(req, res)) return; - if (!requireAdmiral(req, res)) return; - try { - const userId = parseInt(req.params.id as string, 10); - const { role, resource_type, resource_id } = req.body; - - const validRoles: UserRole[] = ['admin', 'viewer', 'deployer', 'node-admin']; - if (!validRoles.includes(role)) { - res.status(400).json({ error: 'Invalid role' }); - return; - } - const validResourceTypes: ResourceType[] = ['stack', 'node']; - if (!validResourceTypes.includes(resource_type)) { - res.status(400).json({ error: 'Invalid resource type' }); - return; - } - if (!resource_id || typeof resource_id !== 'string') { - res.status(400).json({ error: 'resource_id is required' }); - return; - } - - const db = DatabaseService.getInstance(); - if (!db.getUser(userId)) { - res.status(404).json({ error: 'User not found' }); - return; - } - - try { - const id = db.addRoleAssignment({ user_id: userId, role, resource_type, resource_id }); - console.log('[Roles] Assigned', role, 'on', resource_type, resource_id, 'to user', userId, 'by:', req.user!.username); - res.status(201).json({ id, user_id: userId, role, resource_type, resource_id }); - } catch (err: unknown) { - if ((err as Error).message?.includes('UNIQUE constraint')) { - res.status(409).json({ error: 'This role assignment already exists' }); - return; - } - throw err; - } - } catch (error) { - console.error('[Roles] Create error:', error); - res.status(500).json({ error: 'Failed to add role assignment' }); - } -}); - -app.delete('/api/users/:id/roles/:assignId', authMiddleware, (req: Request, res: Response): void => { - if (req.apiTokenScope) { - res.status(403).json({ error: 'API tokens cannot access user management.', code: 'SCOPE_DENIED' }); - return; - } - if (!requireAdmin(req, res)) return; - if (!requireAdmiral(req, res)) return; - try { - const userId = parseInt(req.params.id as string, 10); - const assignId = parseInt(req.params.assignId as string, 10); - const db = DatabaseService.getInstance(); - - const assignment = db.getRoleAssignmentById(assignId); - if (!assignment || assignment.user_id !== userId) { - res.status(404).json({ error: 'Role assignment not found' }); - return; - } - - db.deleteRoleAssignment(assignId); - console.log('[Roles] Removed assignment', assignId, 'from user', userId, 'by:', req.user!.username); - res.json({ success: true }); - } catch (error) { - console.error('[Roles] Delete error:', error); - res.status(500).json({ error: 'Failed to delete role assignment' }); - } -}); // Remote Node HTTP Proxy (see proxy/remoteNodeProxy.ts). Mounted here after // authGate + auditLog + apiTokenScope so local Sencho enforces auth first; @@ -1970,204 +444,6 @@ app.put('/api/stacks/:stackName/env', async (req: Request, res: Response) => { } }); -// ── Git sources ──────────────────────────────────────────────────────── -// Status mapping and error helper live in utils/gitSourceHttp so the -// mapping can be unit-tested without spinning up the full app. - -app.get('/api/git-sources', async (req: Request, res: Response) => { - try { - const all = GitSourceService.getInstance().list(); - // Filter to the subset of stacks the caller can read. Keeps scoped - // Admiral roles from discovering git config for stacks outside their grant. - const visible = all.filter(src => checkPermission(req, 'stack:read', 'stack', src.stack_name)); - res.json(visible); - } catch (error) { - sendGitSourceError(res, error); - } -}); - -app.get('/api/stacks/:stackName/git-source', async (req: Request, res: Response) => { - const stackName = req.params.stackName as string; - if (!isValidStackName(stackName)) { - return res.status(400).json({ error: 'Invalid stack name' }); - } - if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return; - try { - const source = GitSourceService.getInstance().get(stackName); - if (!source) return res.status(404).json({ error: 'No Git source configured for this stack' }); - res.json(source); - } catch (error) { - sendGitSourceError(res, error); - } -}); - -app.put('/api/stacks/:stackName/git-source', async (req: Request, res: Response) => { - const stackName = req.params.stackName as string; - if (!isValidStackName(stackName)) { - return res.status(400).json({ error: 'Invalid stack name' }); - } - if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; - try { - const { - repo_url, - branch, - compose_path, - sync_env, - env_path, - auth_type, - token, - auto_apply_on_webhook, - auto_deploy_on_apply, - } = req.body ?? {}; - - if (typeof repo_url !== 'string' || !repo_url.trim()) { - return res.status(400).json({ error: 'repo_url is required' }); - } - if (typeof branch !== 'string' || !branch.trim()) { - return res.status(400).json({ error: 'branch is required' }); - } - if (typeof compose_path !== 'string' || !compose_path.trim()) { - return res.status(400).json({ error: 'compose_path is required' }); - } - if (auth_type !== 'none' && auth_type !== 'token') { - return res.status(400).json({ error: 'auth_type must be "none" or "token"' }); - } - if (!/^https:\/\//i.test(repo_url)) { - return res.status(400).json({ error: 'Only HTTPS repository URLs are supported' }); - } - // Bound each field so a caller cannot flood the service with huge - // payloads. These limits are generous compared to anything a real Git - // provider would produce. - if (repo_url.length > 2048) { - return res.status(400).json({ error: 'repo_url is too long' }); - } - if (branch.length > 256) { - return res.status(400).json({ error: 'branch is too long' }); - } - if (compose_path.length > 1024) { - return res.status(400).json({ error: 'compose_path is too long' }); - } - if (typeof env_path === 'string' && env_path.length > 1024) { - return res.status(400).json({ error: 'env_path is too long' }); - } - if (typeof token === 'string' && token.length > 8192) { - return res.status(400).json({ error: 'token is too long' }); - } - - // Confirm the stack actually exists on the active node. Without this - // guard, a caller can stash a git-source row for a name that does not - // exist yet and have it auto-link when a stack with that name is - // later created. - const stacks = await FileSystemService.getInstance(req.nodeId).getStacks(); - if (!stacks.includes(stackName)) { - return res.status(404).json({ error: 'Stack not found' }); - } - - const syncEnv = Boolean(sync_env); - const resolvedEnvPath = syncEnv - ? (typeof env_path === 'string' && env_path.trim() - ? env_path - : path.posix.join(path.posix.dirname(compose_path.replace(/\\/g, '/')) || '.', '.env')) - : null; - - const source = await GitSourceService.getInstance().upsert({ - stackName, - repoUrl: repo_url.trim(), - branch: branch.trim(), - composePath: compose_path.trim(), - syncEnv, - envPath: resolvedEnvPath, - authType: auth_type, - token: typeof token === 'string' ? token : undefined, - autoApplyOnWebhook: Boolean(auto_apply_on_webhook), - autoDeployOnApply: Boolean(auto_deploy_on_apply), - }); - - console.log(`[GitSource] Configured git source for ${stackName}`); - res.json(source); - } catch (error) { - sendGitSourceError(res, error); - } -}); - -app.delete('/api/stacks/:stackName/git-source', async (req: Request, res: Response) => { - const stackName = req.params.stackName as string; - if (!isValidStackName(stackName)) { - return res.status(400).json({ error: 'Invalid stack name' }); - } - if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; - try { - GitSourceService.getInstance().delete(stackName); - console.log(`[GitSource] Removed git source for ${stackName}`); - res.json({ success: true }); - } catch (error) { - sendGitSourceError(res, error); - } -}); - -app.post('/api/stacks/:stackName/git-source/pull', async (req: Request, res: Response) => { - const stackName = req.params.stackName as string; - if (!isValidStackName(stackName)) { - return res.status(400).json({ error: 'Invalid stack name' }); - } - if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; - try { - const result = await GitSourceService.getInstance().pull(stackName); - res.json(result); - } catch (error) { - sendGitSourceError(res, error); - } -}); - -app.post('/api/stacks/:stackName/git-source/apply', async (req: Request, res: Response) => { - const stackName = req.params.stackName as string; - if (!isValidStackName(stackName)) { - return res.status(400).json({ error: 'Invalid stack name' }); - } - if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; - try { - const { commitSha, deploy } = req.body ?? {}; - if (typeof commitSha !== 'string' || !commitSha.trim()) { - return res.status(400).json({ error: 'commitSha is required' }); - } - const result = await GitSourceService.getInstance().apply( - stackName, - commitSha.trim(), - { deploy: typeof deploy === 'boolean' ? deploy : undefined } - ); - invalidateNodeCaches(req.nodeId); - const shortSha = commitSha.trim().slice(0, 7); - if (result.deployed) { - console.log(`[GitSource] Applied commit ${shortSha} to ${stackName} (deployed)`); - } else if (result.deployError) { - console.warn(`[GitSource] Applied commit ${shortSha} to ${stackName}, deploy failed: ${result.deployError}`); - } else { - console.log(`[GitSource] Applied commit ${shortSha} to ${stackName}`); - } - res.json(result); - if (result.deployed) { - triggerPostDeployScan(stackName, req.nodeId).catch(err => - console.error(`[Security] Post-deploy scan failed for ${stackName}:`, err), - ); - } - } catch (error) { - sendGitSourceError(res, error); - } -}); - -app.post('/api/stacks/:stackName/git-source/dismiss-pending', async (req: Request, res: Response) => { - const stackName = req.params.stackName as string; - if (!isValidStackName(stackName)) { - return res.status(400).json({ error: 'Invalid stack name' }); - } - if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; - try { - GitSourceService.getInstance().dismissPending(stackName); - res.json({ success: true }); - } catch (error) { - sendGitSourceError(res, error); - } -}); app.post('/api/stacks', async (req: Request, res: Response) => { if (!requirePermission(req, res, 'stack:create')) return; diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts new file mode 100644 index 00000000..88e308ef --- /dev/null +++ b/backend/src/routes/fleet.ts @@ -0,0 +1,1035 @@ +import { Router, type Request, type Response } from 'express'; +import path from 'path'; +import semver from 'semver'; +import si from 'systeminformation'; +import type Dockerode from 'dockerode'; +import { DatabaseService, type Node } from '../services/DatabaseService'; +import { FleetSyncService } from '../services/FleetSyncService'; +import { FleetUpdateTrackerService } from '../services/FleetUpdateTrackerService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import DockerController from '../services/DockerController'; +import { FileSystemService } from '../services/FileSystemService'; +import { ComposeService } from '../services/ComposeService'; +import SelfUpdateService from '../services/SelfUpdateService'; +import { fetchRemoteMeta, getSenchoVersion, isValidVersion } from '../services/CapabilityRegistry'; +import { authMiddleware } from '../middleware/auth'; +import { requirePaid, requireAdmin, requireNodeProxy } from '../middleware/tierGates'; +import { scheduleLocalUpdate } from './license'; +import { captureLocalNodeFiles, captureRemoteNodeFiles, type SnapshotNodeData } from '../utils/snapshot-capture'; +import { getLatestVersion } from '../utils/version-check'; +import { isValidStackName } from '../utils/validation'; +import { isDebugEnabled } from '../utils/debug'; +import { getErrorMessage } from '../utils/errors'; + +const updateTracker = FleetUpdateTrackerService.getInstance(); +const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes + +/** + * Parse a numeric route param. Writes a 400 response and returns null when + * the value isn't a valid integer; callers early-return on null. Collapses + * the 7 copies of the `parseInt ... isNaN ... 400 'Invalid X ID'` shape + * across the fleet router. + */ +function parseIdParam(req: Request, res: Response, paramName: string, label: string): number | null { + const raw = req.params[paramName] as string | undefined; + const parsed = parseInt(raw ?? '', 10); + if (isNaN(parsed)) { + res.status(400).json({ error: `Invalid ${label}` }); + return null; + } + return parsed; +} +const UPDATE_TIMEOUT_MSG = 'Node did not come back online within 5 minutes.'; +const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure + +const MAX_SYNC_ROWS = 5000; +const VALID_SEVERITY = new Set(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']); +const CVE_ID_RE = /^(CVE-\d{4}-\d{4,}|GHSA-[\w-]{14,})$/; + +const isIntFlag = (v: unknown): v is 0 | 1 => v === 0 || v === 1; + +function validateScanPolicyRow(row: unknown): string | null { + if (!row || typeof row !== 'object') return 'row must be an object'; + const r = row as Record; + if (typeof r.name !== 'string' || r.name.length === 0 || r.name.length > 200) return 'name must be a non-empty string'; + if (typeof r.max_severity !== 'string' || !VALID_SEVERITY.has(r.max_severity)) return 'max_severity must be CRITICAL, HIGH, MEDIUM, or LOW'; + if (r.stack_pattern !== null && typeof r.stack_pattern !== 'string') return 'stack_pattern must be a string or null'; + if (typeof r.stack_pattern === 'string' && r.stack_pattern.length > 200) return 'stack_pattern is too long'; + if (typeof r.node_identity !== 'string') return 'node_identity must be a string'; + if (r.node_identity.length > 500) return 'node_identity is too long'; + if (!isIntFlag(r.block_on_deploy)) return 'block_on_deploy must be 0 or 1'; + if (!isIntFlag(r.enabled)) return 'enabled must be 0 or 1'; + return null; +} + +function validateCveSuppressionRow(row: unknown): string | null { + if (!row || typeof row !== 'object') return 'row must be an object'; + const r = row as Record; + if (typeof r.cve_id !== 'string' || !CVE_ID_RE.test(r.cve_id)) return 'cve_id must be a valid CVE or GHSA identifier'; + if (r.pkg_name !== null && typeof r.pkg_name !== 'string') return 'pkg_name must be a string or null'; + if (typeof r.pkg_name === 'string' && r.pkg_name.length > 200) return 'pkg_name is too long'; + if (r.image_pattern !== null && typeof r.image_pattern !== 'string') return 'image_pattern must be a string or null'; + if (typeof r.image_pattern === 'string' && r.image_pattern.length > 300) return 'image_pattern is too long'; + if (typeof r.reason !== 'string') return 'reason must be a string'; + if (r.reason.length > 2000) return 'reason is too long'; + if (typeof r.created_by !== 'string' || r.created_by.length > 200) return 'created_by must be a string'; + if (typeof r.created_at !== 'number') return 'created_at must be a number'; + if (r.expires_at !== null && typeof r.expires_at !== 'number') return 'expires_at must be a number or null'; + return null; +} + +interface FleetNodeOverview { + id: number; + name: string; + type: 'local' | 'remote'; + status: 'online' | 'offline' | 'unknown'; + stats: { + active: number; + managed: number; + unmanaged: number; + exited: number; + total: number; + } | null; + systemStats: { + cpu: { usage: string; cores: number }; + memory: { total: number; used: number; free: number; usagePercent: string }; + disk: { total: number; used: number; free: number; usagePercent: string } | null; + } | null; + stacks: string[] | null; +} + +/** Resolve the version to compare nodes against (latest from GitHub, or gateway fallback). */ +async function getCompareTarget(gatewayVersion: string | null) { + const latestVersion = await getLatestVersion(); + const latestValid = latestVersion !== null && isValidVersion(latestVersion); + const result = { + latestVersion, + latestValid, + compareVersion: latestValid ? latestVersion : gatewayVersion, + compareValid: latestValid || isValidVersion(gatewayVersion), + }; + if (isDebugEnabled()) { + console.debug('[Fleet:debug] Compare target resolved:', { gatewayVersion, latestVersion, using: result.compareVersion, valid: result.compareValid }); + } + return result; +} + +async function fetchLocalNodeOverview(node: Node): Promise { + try { + const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(node.id)); + const [allContainers, stacks, currentLoad, mem, fsSize] = await Promise.all([ + DockerController.getInstance(node.id).getAllContainers(), + FileSystemService.getInstance(node.id).getStacks(), + si.currentLoad(), + si.mem(), + si.fsSize(), + ]); + + const isManagedByComposeDir = (c: Dockerode.ContainerInfo): boolean => { + const workingDir: string | undefined = c.Labels?.['com.docker.compose.project.working_dir']; + if (!workingDir) return false; + const resolved = path.resolve(workingDir); + return resolved === composeDir || resolved.startsWith(composeDir + path.sep); + }; + + const containers = allContainers as Dockerode.ContainerInfo[]; + const active = containers.filter(c => c.State === 'running').length; + const exited = containers.filter(c => c.State === 'exited').length; + const total = containers.length; + const managed = containers.filter(c => c.State === 'running' && isManagedByComposeDir(c)).length; + const unmanaged = containers.filter(c => c.State === 'running' && !isManagedByComposeDir(c)).length; + + const mainDisk = fsSize.find(fs => fs.mount === '/' || fs.mount === 'C:') || fsSize[0]; + + return { + id: node.id, + name: node.name, + type: node.type, + status: 'online', + stats: { active, managed, unmanaged, exited, total }, + systemStats: { + cpu: { usage: currentLoad.currentLoad.toFixed(1), cores: currentLoad.cpus.length }, + memory: { + total: mem.total, + used: mem.used, + free: mem.free, + usagePercent: ((mem.used / mem.total) * 100).toFixed(1), + }, + disk: mainDisk ? { + total: mainDisk.size, + used: mainDisk.used, + free: mainDisk.available, + usagePercent: mainDisk.use ? mainDisk.use.toFixed(1) : '0', + } : null, + }, + stacks, + }; + } catch (error) { + console.error(`[Fleet] Local node ${node.name} error:`, error); + return { + id: node.id, name: node.name, type: node.type, status: 'offline', + stats: null, systemStats: null, stacks: null, + }; + } +} + +async function fetchRemoteNodeOverview(node: Node): Promise { + if (!node.api_url || !node.api_token) { + return { + id: node.id, name: node.name, type: node.type, status: 'offline', + stats: null, systemStats: null, stacks: null, + }; + } + + const baseUrl = node.api_url.replace(/\/$/, ''); + const headers = { Authorization: `Bearer ${node.api_token}` }; + + try { + const [statsRes, systemStatsRes, stacksRes] = await Promise.allSettled([ + fetch(`${baseUrl}/api/stats`, { headers, signal: AbortSignal.timeout(10000) }), + fetch(`${baseUrl}/api/system/stats`, { headers, signal: AbortSignal.timeout(10000) }), + fetch(`${baseUrl}/api/stacks`, { headers, signal: AbortSignal.timeout(10000) }), + ]); + + interface RemoteSystemStats { + cpu: { usage: string; cores: number }; + memory: { total: number; used: number; free: number; usagePercent: string }; + disk?: { total: number; used: number; free: number; usagePercent: string } | null; + } + + const stats: FleetNodeOverview['stats'] | null = statsRes.status === 'fulfilled' && statsRes.value.ok + ? await statsRes.value.json() as FleetNodeOverview['stats'] : null; + const systemStatsRaw: RemoteSystemStats | null = systemStatsRes.status === 'fulfilled' && systemStatsRes.value.ok + ? await systemStatsRes.value.json() as RemoteSystemStats : null; + const stacks: string[] | null = stacksRes.status === 'fulfilled' && stacksRes.value.ok + ? await stacksRes.value.json() as string[] : null; + + const systemStats: FleetNodeOverview['systemStats'] | null = systemStatsRaw ? { + cpu: systemStatsRaw.cpu, + memory: systemStatsRaw.memory, + disk: systemStatsRaw.disk ? { + total: systemStatsRaw.disk.total, + used: systemStatsRaw.disk.used, + free: systemStatsRaw.disk.free, + usagePercent: systemStatsRaw.disk.usagePercent, + } : null, + } : null; + + return { + id: node.id, + name: node.name, + type: node.type, + status: stats || systemStats ? 'online' : 'offline', + stats, + systemStats, + stacks, + }; + } catch (error) { + console.error(`[Fleet] Remote node ${node.name} error:`, error); + return { + id: node.id, name: node.name, type: node.type, status: 'offline', + stats: null, systemStats: null, stacks: null, + }; + } +} + +export const fleetRouter = Router(); + +// Fleet role: tells the frontend whether this Sencho is the control or a +// replica. The control serves read+write for security rules. Replicas are +// read-only and managed upstream. +fleetRouter.get('/role', authMiddleware, (req: Request, res: Response): void => { + if (!requireAdmin(req, res)) return; + res.json({ role: FleetSyncService.getRole() }); +}); + +// Receive a full replacement of a replicated resource from the control. +// Restricted to node_proxy Bearer tokens so only a sibling Sencho can push. +fleetRouter.post('/sync/:resource', authMiddleware, (req: Request, res: Response): void => { + if (!requireNodeProxy(req, res)) return; + const resource = req.params.resource; + if (resource !== 'scan_policies' && resource !== 'cve_suppressions') { + res.status(400).json({ error: `Unsupported sync resource: ${resource}` }); + return; + } + const body = req.body ?? {}; + const rows = Array.isArray(body.rows) ? body.rows : null; + const targetIdentity = typeof body.targetIdentity === 'string' ? body.targetIdentity : ''; + if (!rows) { + res.status(400).json({ error: 'rows array is required' }); + return; + } + if (rows.length > MAX_SYNC_ROWS) { + res.status(413).json({ error: `Too many rows (max ${MAX_SYNC_ROWS})` }); + return; + } + const validator = resource === 'scan_policies' ? validateScanPolicyRow : validateCveSuppressionRow; + for (let i = 0; i < rows.length; i++) { + const err = validator(rows[i]); + if (err) { + res.status(400).json({ error: `Invalid row at index ${i}: ${err}` }); + return; + } + } + try { + FleetSyncService.getInstance().applyIncomingSync(resource, rows, targetIdentity); + res.json({ success: true, applied: rows.length }); + } catch (error) { + console.error('[FleetSync] Failed to apply incoming sync:', error); + res.status(500).json({ error: 'Failed to apply sync' }); + } +}); + +fleetRouter.get('/sync-status', authMiddleware, (req: Request, res: Response): void => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + res.json(DatabaseService.getInstance().getFleetSyncStatuses()); +}); + +fleetRouter.get('/overview', authMiddleware, async (_req: Request, res: Response): Promise => { + try { + const debug = isDebugEnabled(); + const db = DatabaseService.getInstance(); + const nodes = db.getNodes(); + if (debug) console.debug('[Fleet:debug] Overview requested, fetching', nodes.length, 'nodes'); + + const results = await Promise.allSettled( + nodes.map(async (node): Promise => { + if (node.type === 'remote') { + return fetchRemoteNodeOverview(node); + } + return fetchLocalNodeOverview(node); + }), + ); + + const overview: FleetNodeOverview[] = results.map((result, i) => { + if (result.status === 'fulfilled') return result.value; + console.error(`[Fleet] Failed to fetch node ${nodes[i].name}:`, result.reason); + return { + id: nodes[i].id, + name: nodes[i].name, + type: nodes[i].type, + status: 'offline' as const, + stats: null, + systemStats: null, + stacks: null, + }; + }); + + if (debug) { + const online = overview.filter(n => n.status === 'online').length; + console.debug('[Fleet:debug] Overview complete:', online, 'online,', overview.length - online, 'offline'); + } + res.json(overview); + } catch (error) { + console.error('[Fleet] Overview error:', error); + res.status(500).json({ error: 'Failed to fetch fleet overview' }); + } +}); + +fleetRouter.get('/node/:nodeId/stacks', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + + try { + const nodeId = parseIdParam(req, res, 'nodeId', 'node ID'); + if (nodeId === null) return; + const node = DatabaseService.getInstance().getNode(nodeId); + if (!node) { + res.status(404).json({ error: 'Node not found' }); + return; + } + + if (node.type === 'remote') { + if (!node.api_url || !node.api_token) { + res.status(503).json({ error: 'Remote node not configured' }); + return; + } + const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks`, { + headers: { Authorization: `Bearer ${node.api_token}` }, + signal: AbortSignal.timeout(10000), + }); + if (!response.ok) { + res.status(502).json({ error: 'Failed to fetch stacks from remote node' }); + return; + } + const stacks = await response.json(); + if (isDebugEnabled()) console.debug('[Fleet:debug] Node stacks:', nodeId, node.type, Array.isArray(stacks) ? stacks.length : 0, 'stacks'); + res.json(stacks); + return; + } + + const stacks = await FileSystemService.getInstance(nodeId).getStacks(); + if (isDebugEnabled()) console.debug('[Fleet:debug] Node stacks:', nodeId, node.type, stacks.length, 'stacks'); + res.json(stacks); + } catch (error) { + console.error('[Fleet] Node stacks error:', error); + res.status(500).json({ error: 'Failed to fetch node stacks' }); + } +}); + +fleetRouter.get('/node/:nodeId/stacks/:stackName/containers', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + + try { + const nodeId = parseIdParam(req, res, 'nodeId', 'node ID'); + if (nodeId === null) return; + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + const node = DatabaseService.getInstance().getNode(nodeId); + if (!node) { + res.status(404).json({ error: 'Node not found' }); + return; + } + + if (node.type === 'remote') { + if (!node.api_url || !node.api_token) { + res.status(503).json({ error: 'Remote node not configured' }); + return; + } + const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, { + headers: { Authorization: `Bearer ${node.api_token}` }, + signal: AbortSignal.timeout(10000), + }); + if (!response.ok) { + res.status(502).json({ error: 'Failed to fetch containers from remote node' }); + return; + } + const containers = await response.json(); + res.json(containers); + return; + } + + const dockerController = DockerController.getInstance(nodeId); + const containers = await dockerController.getContainersByStack(stackName); + if (isDebugEnabled()) console.debug('[Fleet:debug] Stack containers:', nodeId, stackName, containers.length, 'containers'); + res.json(containers); + } catch (error) { + console.error('[Fleet] Node stack containers error:', error); + res.status(500).json({ error: 'Failed to fetch stack containers' }); + } +}); + +fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + try { + const db = DatabaseService.getInstance(); + const nodes = db.getNodes(); + const gatewayVersion = getSenchoVersion(); + const gatewayValid = isValidVersion(gatewayVersion); + + const { latestVersion, latestValid, compareVersion, compareValid } = await getCompareTarget(gatewayVersion); + const debug = isDebugEnabled(); + + const results = await Promise.allSettled( + nodes.map(async (node) => { + const tracker = updateTracker.get(node.id); + + let version: string | null = null; + let remoteStartedAt: number | null = null; + let remoteUpdateError: string | null = null; + let remoteOnline = false; + if (node.type === 'local') { + version = gatewayVersion; + } else if (node.api_url && node.api_token) { + const meta = await fetchRemoteMeta(node.api_url, node.api_token); + version = meta.version; + remoteStartedAt = meta.startedAt; + remoteUpdateError = meta.updateError; + remoteOnline = meta.online; + } + + if (tracker?.status === 'updating') { + const elapsed = Date.now() - tracker.startedAt; + + if (debug) { + console.debug('[Fleet:debug] Polling update status for node', node.id, node.name, '- elapsed:', Math.round(elapsed / 1000) + 's', 'version:', version, 'wasOffline:', tracker.wasOffline, 'remoteOnline:', remoteOnline); + } + + if (elapsed > UPDATE_TIMEOUT_MS) { + if (debug) console.debug('[Fleet:debug] Node', node.id, 'timed out after', Math.round(elapsed / 1000) + 's'); + updateTracker.set(node.id, updateTracker.resolve(tracker, 'timeout', UPDATE_TIMEOUT_MSG)); + } else if (node.type === 'remote') { + if (remoteUpdateError) { + if (debug) console.debug('[Fleet:debug] Node', node.id, 'reported pull failure:', remoteUpdateError); + updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', remoteUpdateError)); + } else if (!remoteOnline) { + if (!tracker.wasOffline) { + if (debug) console.debug('[Fleet:debug] Node', node.id, 'went offline (restarting)'); + updateTracker.set(node.id, { ...tracker, wasOffline: true }); + } + } else if (version !== tracker.previousVersion) { + if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 1 (version changed):', tracker.previousVersion, '->', version); + updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); + } else if ( + remoteStartedAt !== null && + tracker.previousProcessStart !== null && + remoteStartedAt !== tracker.previousProcessStart + ) { + if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 2 (process restarted):', tracker.previousProcessStart, '->', remoteStartedAt); + updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); + } else if (tracker.wasOffline && remoteOnline) { + if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 3 (offline then online)'); + updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); + } else if ( + elapsed > 15_000 && + isValidVersion(version) && + gatewayValid && + !semver.lt(version, compareVersion!) + ) { + // Signal 4: remote is now at or above gateway version (after + // minimum processing time). Catches fast restarts where the 5s + // polling interval misses the offline window and startedAt + // hasn't been observed to change yet. + if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 4 (version >= compare target):', version, '>=', compareVersion); + updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); + } else if (elapsed > EARLY_FAIL_MS) { + if (debug) console.debug('[Fleet:debug] Node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's - no signals detected'); + updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', 'Update may have failed. The node is still running and its version has not changed.')); + } + } else if (node.type === 'local') { + // Local node has only two failure signals: an explicit pull/spawn + // error, or the early-fail heuristic. Success is observed by the + // frontend overlay (it reloads when /api/health reports a new + // startedAt), at which point the new process starts with an empty + // tracker map. + const selfUpdate = SelfUpdateService.getInstance(); + const localError = selfUpdate.getLastError(); + if (localError) { + if (debug) console.debug('[Fleet:debug] Local node', node.id, 'update failed:', localError); + updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', localError)); + selfUpdate.clearLastError(); + } else if (elapsed > EARLY_FAIL_MS) { + if (debug) console.debug('[Fleet:debug] Local node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's'); + updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', 'Local update did not complete. The container may not have restarted; check Docker logs on the host.')); + } + } + } + + // Auto-expire completed entries 60s after they resolved so the badge + // is visible briefly after completion. + if (tracker?.status === 'completed' && tracker.resolvedAt && Date.now() - tracker.resolvedAt > 60_000) { + updateTracker.delete(node.id); + } + + let updateAvailable = false; + if (!isValidVersion(version)) { + // Assume remote nodes are outdated when their version is unresolvable. + updateAvailable = node.type === 'remote'; + } else if (compareValid) { + updateAvailable = semver.lt(version, compareVersion!); + } + + const currentTracker = updateTracker.get(node.id); + return { + nodeId: node.id, + name: node.name, + type: node.type, + version, + latestVersion: latestValid ? latestVersion : gatewayVersion, + updateAvailable, + updateStatus: currentTracker?.status ?? null, + error: currentTracker?.error ?? null, + }; + }), + ); + + const nodeStatuses = results.map((r, i) => { + if (r.status === 'fulfilled') return r.value; + return { + nodeId: nodes[i].id, + name: nodes[i].name, + type: nodes[i].type, + version: null, + latestVersion: latestValid ? latestVersion : gatewayVersion, + updateAvailable: false, + updateStatus: null, + error: null, + }; + }); + + if (isDebugEnabled()) { + const trackerStates = Array.from(updateTracker.entries()).map(([nid, t]) => `${nid}:${t.status}`); + console.debug('[Fleet:debug] Update status:', nodeStatuses.length, 'nodes, trackers:', trackerStates.join(', ') || 'none'); + } + res.json({ nodes: nodeStatuses }); + } catch (error) { + console.error('[Fleet] Update status error:', error); + res.status(500).json({ error: 'Failed to fetch update status' }); + } +}); + +fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + if (!requireAdmin(req, res)) return; + try { + const nodeId = parseIdParam(req, res, 'nodeId', 'node ID'); + if (nodeId === null) return; + const db = DatabaseService.getInstance(); + const node = db.getNode(nodeId); + if (!node) { + res.status(404).json({ error: 'Node not found' }); + return; + } + + const existing = updateTracker.get(nodeId); + if (existing?.status === 'updating') { + if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) { + updateTracker.set(nodeId, updateTracker.resolve(existing, 'timeout', UPDATE_TIMEOUT_MSG)); + } else { + res.status(409).json({ error: 'Update already in progress for this node.' }); + return; + } + } + // Clear terminal states to allow retry. + if (existing && (existing.status === 'timeout' || existing.status === 'failed' || existing.status === 'completed')) { + updateTracker.delete(nodeId); + } + + console.log('[Fleet] Update triggered for node', node.name, node.type); + if (isDebugEnabled()) { + console.debug('[Fleet:debug] Update trigger details:', { nodeId, name: node.name, type: node.type, hasUrl: !!node.api_url, hasToken: !!node.api_token }); + } + + if (node.type === 'local') { + if (!SelfUpdateService.getInstance().isAvailable()) { + res.status(503).json({ error: 'Self-update unavailable on the local node.' }); + return; + } + updateTracker.set(nodeId, updateTracker.create('updating', getSenchoVersion(), null)); + scheduleLocalUpdate(res, 'Update initiated on local node. The server will restart shortly.'); + return; + } + + if (!node.api_url || !node.api_token) { + res.status(503).json({ error: 'Remote node not configured.' }); + return; + } + + const meta = await fetchRemoteMeta(node.api_url, node.api_token); + if (isDebugEnabled()) { + console.debug('[Fleet:debug] Remote meta for update:', { nodeId, online: meta.online, version: meta.version, capabilities: meta.capabilities, startedAt: meta.startedAt }); + } + if (!meta.online) { + res.status(503).json({ error: 'Remote node is unreachable. Verify the node is running and the API URL is correct.' }); + return; + } + if (!meta.capabilities.includes('self-update')) { + res.status(503).json({ error: 'Remote node does not support self-update. It may need to be updated manually first.' }); + return; + } + + const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/system/update`, { + method: 'POST', + headers: { + Authorization: `Bearer ${node.api_token}`, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(10000), + }); + + if (!response.ok) { + const err = await response.json().catch(() => ({})); + const errorMsg = (err as Record)?.error || 'Remote node rejected update request.'; + updateTracker.set(nodeId, updateTracker.create('failed', meta.version, meta.startedAt, errorMsg)); + res.status(502).json({ error: errorMsg }); + return; + } + + updateTracker.set(nodeId, updateTracker.create('updating', meta.version, meta.startedAt)); + res.status(202).json({ message: `Update initiated on ${node.name}.` }); + } catch (error) { + console.error('[Fleet] Node update error:', error); + const errorMsg = getErrorMessage(error, 'Failed to trigger node update.'); + const failedNodeId = parseInt(req.params.nodeId as string, 10); + if (!isNaN(failedNodeId)) { + updateTracker.set(failedNodeId, updateTracker.create('failed', null, null, errorMsg)); + } + res.status(500).json({ error: 'Failed to trigger node update.' }); + } +}); + +fleetRouter.post('/update-all', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + if (!requireAdmin(req, res)) return; + try { + const db = DatabaseService.getInstance(); + const nodes = db.getNodes(); + const gatewayVersion = getSenchoVersion(); + const { compareVersion, compareValid } = await getCompareTarget(gatewayVersion); + + const debug = isDebugEnabled(); + console.log('[Fleet] Update-all triggered,', nodes.length, 'nodes registered'); + if (debug) console.debug('[Fleet:debug] Update-all compare target:', { gatewayVersion, compareVersion, compareValid }); + + const candidates = nodes.filter(node => { + if (node.type === 'local') return false; + const tracker = updateTracker.get(node.id); + if (tracker?.status === 'updating') return false; + if (!node.api_url || !node.api_token) return false; + // Clear terminal states so they can be re-triggered. + if (tracker && (tracker.status === 'timeout' || tracker.status === 'failed' || tracker.status === 'completed')) { + updateTracker.delete(node.id); + } + return true; + }); + + const results = await Promise.allSettled(candidates.map(async (node) => { + const meta = await fetchRemoteMeta(node.api_url!, node.api_token!); + if (!meta.online) { + return { name: node.name, triggered: false }; + } + if (!meta.capabilities.includes('self-update')) { + return { name: node.name, triggered: false }; + } + if (isValidVersion(meta.version) && compareValid && !semver.lt(meta.version, compareVersion!)) { + return { name: node.name, triggered: false }; + } + const response = await fetch(`${node.api_url!.replace(/\/$/, '')}/api/system/update`, { + method: 'POST', + headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' }, + signal: AbortSignal.timeout(10000), + }); + if (response.ok) { + updateTracker.set(node.id, updateTracker.create('updating', meta.version, meta.startedAt)); + return { name: node.name, triggered: true }; + } + return { name: node.name, triggered: false }; + })); + + const updating: string[] = []; + const skipped = nodes.filter(n => !candidates.includes(n)).map(n => n.name); + for (let i = 0; i < results.length; i++) { + const r = results[i]; + const val = r.status === 'fulfilled' ? r.value : { name: candidates[i].name, triggered: false }; + (val.triggered ? updating : skipped).push(val.name); + } + + if (debug) console.debug('[Fleet:debug] Update-all results:', { updating, skippedCount: skipped.length, candidateCount: candidates.length }); + res.status(202).json({ updating, skipped }); + } catch (error) { + console.error('[Fleet] Update all error:', error); + res.status(500).json({ error: 'Failed to trigger fleet update.' }); + } +}); + +fleetRouter.delete('/nodes/:nodeId/update-status', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + try { + const nodeId = parseIdParam(req, res, 'nodeId', 'node ID'); + if (nodeId === null) return; + const node = DatabaseService.getInstance().getNode(nodeId); + if (!node) { + res.status(404).json({ error: 'Node not found' }); + return; + } + updateTracker.delete(nodeId); + res.status(204).send(); + } catch (error) { + console.error('[Fleet] Clear update status error:', error); + res.status(500).json({ error: 'Failed to clear update status.' }); + } +}); + +fleetRouter.delete('/update-status', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + // Pre-fetch fresh latest version so the next GET has up-to-date data. + if (req.query.recheck === 'true') { + await getLatestVersion(true); + } + for (const [nodeId, tracker] of updateTracker.entries()) { + if (tracker.status === 'timeout' || tracker.status === 'failed' || tracker.status === 'completed') { + updateTracker.delete(nodeId); + } + } + res.status(204).send(); +}); + +// ─── Fleet Snapshots (Skipper+) ─── + +fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + + try { + const { description = '' } = req.body; + if (typeof description === 'string' && description.length > 500) { + res.status(400).json({ error: 'Description must be 500 characters or less' }); + return; + } + const db = DatabaseService.getInstance(); + const nodes = db.getNodes(); + const username = req.user?.username || 'admin'; + + const captureStart = Date.now(); + const results = await Promise.allSettled( + nodes.map(async (node) => { + if (node.type === 'remote') { + return captureRemoteNodeFiles(node); + } + return captureLocalNodeFiles(node); + }), + ); + + const capturedNodes: SnapshotNodeData[] = []; + const skippedNodes: Array<{ nodeId: number; nodeName: string; reason: string }> = []; + + results.forEach((result, i) => { + if (result.status === 'fulfilled') { + capturedNodes.push(result.value); + } else { + console.error(`[Fleet Snapshot] Failed to capture node ${nodes[i].name}:`, result.reason); + skippedNodes.push({ + nodeId: nodes[i].id, + nodeName: nodes[i].name, + reason: result.reason instanceof Error ? result.reason.message : 'Unknown error', + }); + } + }); + + let totalStacks = 0; + const allFiles: Array<{ nodeId: number; nodeName: string; stackName: string; filename: string; content: string }> = []; + + for (const nodeData of capturedNodes) { + totalStacks += nodeData.stacks.length; + for (const stack of nodeData.stacks) { + for (const file of stack.files) { + allFiles.push({ + nodeId: nodeData.nodeId, + nodeName: nodeData.nodeName, + stackName: stack.stackName, + filename: file.filename, + content: file.content, + }); + } + } + } + + const snapshotId = db.createSnapshot( + description, + username, + capturedNodes.length, + totalStacks, + JSON.stringify(skippedNodes), + ); + + if (allFiles.length > 0) { + db.insertSnapshotFiles(snapshotId, allFiles); + } + + console.log('[Fleet] Snapshot created:', capturedNodes.length, 'nodes,', totalStacks, 'stacks'); + if (isDebugEnabled()) { + console.debug(`[Fleet:debug] Snapshot ${snapshotId} capture completed in ${Date.now() - captureStart}ms, ${allFiles.length} file(s) stored`); + for (const skip of skippedNodes) { + console.debug(`[Fleet:debug] Skipped node "${skip.nodeName}" (id=${skip.nodeId}): ${skip.reason}`); + } + } + const snapshot = db.getSnapshot(snapshotId); + res.status(201).json(snapshot); + } catch (error) { + console.error('[Fleet Snapshot] Create error:', error); + res.status(500).json({ error: 'Failed to create fleet snapshot' }); + } +}); + +fleetRouter.get('/snapshots', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + + try { + const limit = Math.min(parseInt(req.query.limit as string, 10) || 50, 100); + const offset = parseInt(req.query.offset as string, 10) || 0; + const db = DatabaseService.getInstance(); + const snapshots = db.getSnapshots(limit, offset); + const total = db.getSnapshotCount(); + if (isDebugEnabled()) console.debug('[Fleet:debug] Snapshots list: limit=', limit, 'offset=', offset, 'total=', total); + res.json({ snapshots, total }); + } catch (error) { + console.error('[Fleet Snapshot] List error:', error); + res.status(500).json({ error: 'Failed to list fleet snapshots' }); + } +}); + +fleetRouter.get('/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + + try { + const id = parseIdParam(req, res, 'id', 'snapshot ID'); + if (id === null) return; + const db = DatabaseService.getInstance(); + const snapshot = db.getSnapshot(id); + if (!snapshot) { + res.status(404).json({ error: 'Snapshot not found' }); + return; + } + + const files = db.getSnapshotFiles(id); + + // Group files by node and stack. + const nodesMap = new Map> }>(); + for (const file of files) { + if (!nodesMap.has(file.node_id)) { + nodesMap.set(file.node_id, { nodeId: file.node_id, nodeName: file.node_name, stacks: new Map() }); + } + const nodeEntry = nodesMap.get(file.node_id)!; + if (!nodeEntry.stacks.has(file.stack_name)) { + nodeEntry.stacks.set(file.stack_name, []); + } + nodeEntry.stacks.get(file.stack_name)!.push({ filename: file.filename, content: file.content }); + } + + const nodes = Array.from(nodesMap.values()).map(n => ({ + nodeId: n.nodeId, + nodeName: n.nodeName, + stacks: Array.from(n.stacks.entries()).map(([stackName, stackFiles]) => ({ + stackName, + files: stackFiles, + })), + })); + + if (isDebugEnabled()) console.debug('[Fleet:debug] Snapshot detail:', id, files.length, 'files'); + res.json({ ...snapshot, nodes }); + } catch (error) { + console.error('[Fleet Snapshot] Detail error:', error); + res.status(500).json({ error: 'Failed to fetch snapshot details' }); + } +}); + +fleetRouter.post('/snapshots/:id/restore', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + + try { + const snapshotId = parseIdParam(req, res, 'id', 'snapshot ID'); + if (snapshotId === null) return; + const { nodeId, stackName, redeploy = false } = req.body; + + if (!nodeId || !stackName) { + res.status(400).json({ error: 'nodeId and stackName are required' }); + return; + } + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + + const db = DatabaseService.getInstance(); + const snapshot = db.getSnapshot(snapshotId); + if (!snapshot) { + res.status(404).json({ error: 'Snapshot not found' }); + return; + } + + const files = db.getSnapshotStackFiles(snapshotId, nodeId, stackName); + if (files.length === 0) { + res.status(404).json({ error: 'No files found for this stack in the snapshot' }); + return; + } + + if (isDebugEnabled()) { + const fileNames = files.map(f => f.filename).join(', '); + console.debug(`[Fleet:debug] Restore: snapshot=${snapshotId}, node=${nodeId}, stack="${stackName}", files=[${fileNames}], redeploy=${redeploy}`); + } + + const node = db.getNode(nodeId); + if (!node) { + res.status(404).json({ error: 'Target node no longer exists' }); + return; + } + + if (node.type === 'local') { + const fsService = FileSystemService.getInstance(node.id); + + try { + await fsService.backupStackFiles(stackName); + } catch (e) { + // Stack may not exist yet before first restore; that is ok. + console.warn(`[Fleet Snapshot] Pre-restore backup failed for stack "${stackName}" (may not exist yet):`, getErrorMessage(e, 'unknown')); + } + + for (const file of files) { + if (file.filename === 'compose.yaml') { + await fsService.saveStackContent(stackName, file.content); + } else if (file.filename === '.env') { + await fsService.saveEnvContent(stackName, file.content); + } + } + + if (redeploy) { + const composeService = ComposeService.getInstance(node.id); + await composeService.deployStack(stackName); + } + } else { + if (!node.api_url || !node.api_token) { + res.status(503).json({ error: 'Remote node not configured' }); + return; + } + + const baseUrl = node.api_url.replace(/\/$/, ''); + const headers: Record = { + Authorization: `Bearer ${node.api_token}`, + 'Content-Type': 'application/json', + }; + + for (const file of files) { + if (file.filename === 'compose.yaml') { + const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, { + method: 'PUT', + headers, + body: JSON.stringify({ content: file.content }), + signal: AbortSignal.timeout(15000), + }); + if (!putRes.ok) throw new Error('Failed to restore compose file on remote node'); + } else if (file.filename === '.env') { + const putRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, { + method: 'PUT', + headers, + body: JSON.stringify({ content: file.content }), + signal: AbortSignal.timeout(15000), + }); + if (!putRes.ok) throw new Error('Failed to restore env file on remote node'); + } + } + + if (redeploy) { + await fetch(`${baseUrl}/api/compose/${encodeURIComponent(stackName)}/up`, { + method: 'POST', + headers, + signal: AbortSignal.timeout(30000), + }); + } + } + + console.log('[Fleet] Snapshot restore:', snapshotId, 'node=', nodeId, 'stack=', stackName); + res.json({ message: 'Stack restored successfully', redeployed: redeploy }); + } catch (error) { + console.error('[Fleet Snapshot] Restore error:', error); + res.status(500).json({ error: 'Failed to restore stack from snapshot' }); + } +}); + +fleetRouter.delete('/snapshots/:id', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + + try { + const id = parseIdParam(req, res, 'id', 'snapshot ID'); + if (id === null) return; + const db = DatabaseService.getInstance(); + const snapshot = db.getSnapshot(id); + if (!snapshot) { + res.status(404).json({ error: 'Snapshot not found' }); + return; + } + if (isDebugEnabled()) { + console.debug(`[Fleet:debug] Deleting snapshot ${id} (${snapshot.node_count} node(s), ${snapshot.stack_count} stack(s))`); + } + db.deleteSnapshot(id); + console.log('[Fleet] Snapshot deleted:', id); + res.json({ message: 'Snapshot deleted' }); + } catch (error) { + console.error('[Fleet Snapshot] Delete error:', error); + res.status(500).json({ error: 'Failed to delete snapshot' }); + } +}); diff --git a/backend/src/routes/gitSources.ts b/backend/src/routes/gitSources.ts new file mode 100644 index 00000000..ec71048d --- /dev/null +++ b/backend/src/routes/gitSources.ts @@ -0,0 +1,239 @@ +import { Router, type Request, type Response } from 'express'; +import path from 'path'; +import { GitSourceService } from '../services/GitSourceService'; +import { FileSystemService } from '../services/FileSystemService'; +import { checkPermission, requirePermission } from '../middleware/permissions'; +import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; +import { triggerPostDeployScan } from '../helpers/policyGate'; +import { isValidStackName } from '../utils/validation'; +import { sendGitSourceError } from '../utils/gitSourceHttp'; + +// Reasonable upper bounds so a caller cannot flood the service with huge +// payloads. Generous compared to anything a real Git provider emits. +const MAX_REPO_URL_LENGTH = 2048; +const MAX_BRANCH_LENGTH = 256; +const MAX_COMPOSE_PATH_LENGTH = 1024; +const MAX_ENV_PATH_LENGTH = 1024; +const MAX_TOKEN_LENGTH = 8192; + +/** Router for listing git-source configuration: `GET /api/git-sources`. */ +export const gitSourcesRouter = Router(); + +gitSourcesRouter.get('/', async (req: Request, res: Response): Promise => { + try { + const all = GitSourceService.getInstance().list(); + // Filter to the subset of stacks the caller can read. Keeps scoped + // Admiral roles from discovering git config for stacks outside their grant. + const visible = all.filter(src => checkPermission(req, 'stack:read', 'stack', src.stack_name)); + res.json(visible); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +/** + * Router for per-stack git-source endpoints. Mount at `/api/stacks` so the + * `/:stackName/git-source*` paths work alongside other stack-scoped routes + * (such as the label-assignments router extracted in Phase 4A-1). + */ +export const stackGitSourceRouter = Router(); + +stackGitSourceRouter.get('/:stackName/git-source', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return; + try { + const source = GitSourceService.getInstance().get(stackName); + if (!source) { + res.status(404).json({ error: 'No Git source configured for this stack' }); + return; + } + res.json(source); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +stackGitSourceRouter.put('/:stackName/git-source', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + try { + const { + repo_url, + branch, + compose_path, + sync_env, + env_path, + auth_type, + token, + auto_apply_on_webhook, + auto_deploy_on_apply, + } = req.body ?? {}; + + if (typeof repo_url !== 'string' || !repo_url.trim()) { + res.status(400).json({ error: 'repo_url is required' }); + return; + } + if (typeof branch !== 'string' || !branch.trim()) { + res.status(400).json({ error: 'branch is required' }); + return; + } + if (typeof compose_path !== 'string' || !compose_path.trim()) { + res.status(400).json({ error: 'compose_path is required' }); + return; + } + if (auth_type !== 'none' && auth_type !== 'token') { + res.status(400).json({ error: 'auth_type must be "none" or "token"' }); + return; + } + if (!/^https:\/\//i.test(repo_url)) { + res.status(400).json({ error: 'Only HTTPS repository URLs are supported' }); + return; + } + if (repo_url.length > MAX_REPO_URL_LENGTH) { + res.status(400).json({ error: 'repo_url is too long' }); + return; + } + if (branch.length > MAX_BRANCH_LENGTH) { + res.status(400).json({ error: 'branch is too long' }); + return; + } + if (compose_path.length > MAX_COMPOSE_PATH_LENGTH) { + res.status(400).json({ error: 'compose_path is too long' }); + return; + } + if (typeof env_path === 'string' && env_path.length > MAX_ENV_PATH_LENGTH) { + res.status(400).json({ error: 'env_path is too long' }); + return; + } + if (typeof token === 'string' && token.length > MAX_TOKEN_LENGTH) { + res.status(400).json({ error: 'token is too long' }); + return; + } + + // Confirm the stack actually exists on the active node. Without this guard + // a caller could stash a git-source row for a name that does not exist + // yet and have it auto-link when a stack with that name is later created. + const stacks = await FileSystemService.getInstance(req.nodeId).getStacks(); + if (!stacks.includes(stackName)) { + res.status(404).json({ error: 'Stack not found' }); + return; + } + + const syncEnv = Boolean(sync_env); + const resolvedEnvPath = syncEnv + ? (typeof env_path === 'string' && env_path.trim() + ? env_path + : path.posix.join(path.posix.dirname(compose_path.replace(/\\/g, '/')) || '.', '.env')) + : null; + + const source = await GitSourceService.getInstance().upsert({ + stackName, + repoUrl: repo_url.trim(), + branch: branch.trim(), + composePath: compose_path.trim(), + syncEnv, + envPath: resolvedEnvPath, + authType: auth_type, + token: typeof token === 'string' ? token : undefined, + autoApplyOnWebhook: Boolean(auto_apply_on_webhook), + autoDeployOnApply: Boolean(auto_deploy_on_apply), + }); + + console.log(`[GitSource] Configured git source for ${stackName}`); + res.json(source); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +stackGitSourceRouter.delete('/:stackName/git-source', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + try { + GitSourceService.getInstance().delete(stackName); + console.log(`[GitSource] Removed git source for ${stackName}`); + res.json({ success: true }); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +stackGitSourceRouter.post('/:stackName/git-source/pull', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + try { + const result = await GitSourceService.getInstance().pull(stackName); + res.json(result); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +stackGitSourceRouter.post('/:stackName/git-source/apply', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + try { + const { commitSha, deploy } = req.body ?? {}; + if (typeof commitSha !== 'string' || !commitSha.trim()) { + res.status(400).json({ error: 'commitSha is required' }); + return; + } + const result = await GitSourceService.getInstance().apply( + stackName, + commitSha.trim(), + { deploy: typeof deploy === 'boolean' ? deploy : undefined }, + ); + invalidateNodeCaches(req.nodeId); + const shortSha = commitSha.trim().slice(0, 7); + if (result.deployed) { + console.log(`[GitSource] Applied commit ${shortSha} to ${stackName} (deployed)`); + } else if (result.deployError) { + console.warn(`[GitSource] Applied commit ${shortSha} to ${stackName}, deploy failed: ${result.deployError}`); + } else { + console.log(`[GitSource] Applied commit ${shortSha} to ${stackName}`); + } + res.json(result); + if (result.deployed) { + triggerPostDeployScan(stackName, req.nodeId).catch(err => + console.error(`[Security] Post-deploy scan failed for ${stackName}:`, err), + ); + } + } catch (error) { + sendGitSourceError(res, error); + } +}); + +stackGitSourceRouter.post('/:stackName/git-source/dismiss-pending', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + try { + GitSourceService.getInstance().dismissPending(stackName); + res.json({ success: true }); + } catch (error) { + sendGitSourceError(res, error); + } +}); diff --git a/backend/src/routes/users.ts b/backend/src/routes/users.ts new file mode 100644 index 00000000..a5999b7c --- /dev/null +++ b/backend/src/routes/users.ts @@ -0,0 +1,346 @@ +import { Router, type Request, type Response } from 'express'; +import bcrypt from 'bcrypt'; +import { DatabaseService, type UserRole, type ResourceType } from '../services/DatabaseService'; +import { LicenseService } from '../services/LicenseService'; +import { authMiddleware } from '../middleware/auth'; +import { requirePaid, requireAdmin, requireAdmiral } from '../middleware/tierGates'; +import { rejectApiTokenScope } from '../middleware/apiTokenScope'; +import { BCRYPT_SALT_ROUNDS, MIN_PASSWORD_LENGTH } from '../helpers/constants'; +import { isDebugEnabled } from '../utils/debug'; +import { getErrorMessage, isSqliteUniqueViolation } from '../utils/errors'; + +const USERS_SCOPE_MESSAGE = 'API tokens cannot access user management.'; +const VALID_USER_ROLES: UserRole[] = ['admin', 'viewer', 'deployer', 'node-admin', 'auditor']; +const VALID_ASSIGNMENT_ROLES: UserRole[] = ['admin', 'viewer', 'deployer', 'node-admin']; +const VALID_RESOURCE_TYPES: ResourceType[] = ['stack', 'node']; + +// Roles that require an Admiral license. Viewer and admin are available on +// all paid tiers; the rest need variant=admiral for per-resource scoping to +// be meaningful. +function roleRequiresAdmiral(role: UserRole): boolean { + return role === 'deployer' || role === 'node-admin' || role === 'auditor'; +} + +function validateUsername(value: unknown): string | null { + if (typeof value !== 'string' || value.length < 3 || !/^[a-zA-Z0-9_-]+$/.test(value)) { + return 'Username must be at least 3 characters (letters, numbers, underscore, hyphen)'; + } + return null; +} + +export const usersRouter = Router(); + +usersRouter.get('/', authMiddleware, async (req: Request, res: Response): Promise => { + if (rejectApiTokenScope(req, res, USERS_SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + try { + const db = DatabaseService.getInstance(); + const users = db.getUsers(); + const mfaUserIds = db.getUsersWithMfaEnabled(); + const enriched = users.map((u) => ({ + ...u, + mfaEnabled: mfaUserIds.has(u.id), + })); + res.json(enriched); + } catch (error) { + console.error('[Users] List error:', error); + res.status(500).json({ error: 'Failed to fetch users' }); + } +}); + +usersRouter.post('/', authMiddleware, async (req: Request, res: Response): Promise => { + if (rejectApiTokenScope(req, res, USERS_SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + try { + const { username, password, role } = req.body; + + if (!username || !password || !role) { + res.status(400).json({ error: 'Username, password, and role are required' }); + return; + } + const usernameError = validateUsername(username); + if (usernameError) { + res.status(400).json({ error: usernameError }); + return; + } + if (typeof password !== 'string' || password.length < MIN_PASSWORD_LENGTH) { + res.status(400).json({ error: `Password must be at least ${MIN_PASSWORD_LENGTH} characters` }); + return; + } + if (!VALID_USER_ROLES.includes(role)) { + res.status(400).json({ error: 'Role must be "admin", "viewer", "deployer", "node-admin", or "auditor"' }); + return; + } + if (roleRequiresAdmiral(role) && !requireAdmiral(req, res)) return; + + const db = DatabaseService.getInstance(); + const existing = db.getUserByUsername(username); + if (existing) { + res.status(409).json({ error: 'A user with this username already exists' }); + return; + } + + // Enforce seat limits based on license variant. + const seatLimits = LicenseService.getInstance().getSeatLimits(); + if (role === 'admin' && seatLimits.maxAdmins !== null && db.getAdminCount() >= seatLimits.maxAdmins) { + res.status(403).json({ error: `Your license allows a maximum of ${seatLimits.maxAdmins} admin account${seatLimits.maxAdmins === 1 ? '' : 's'}. Upgrade to Admiral for unlimited accounts.` }); + return; + } + if (role !== 'admin' && seatLimits.maxViewers !== null && db.getNonAdminCount() >= seatLimits.maxViewers) { + res.status(403).json({ error: `Your license allows a maximum of ${seatLimits.maxViewers} viewer account${seatLimits.maxViewers === 1 ? '' : 's'}. Upgrade to Admiral for unlimited accounts.` }); + return; + } + + const passwordHash = await bcrypt.hash(password, BCRYPT_SALT_ROUNDS); + const id = db.addUser({ username, password_hash: passwordHash, role }); + console.log('[Users] Created:', username, 'role:', role, 'by:', req.user!.username); + res.status(201).json({ id, username, role }); + } catch (error) { + console.error('[Users] Create error:', error); + res.status(500).json({ error: 'Failed to create user' }); + } +}); + +// PUT/DELETE intentionally do NOT enforce requirePaid. Admins must be able +// to manage existing users even if their license lapses. +usersRouter.put('/:id', authMiddleware, async (req: Request, res: Response): Promise => { + if (rejectApiTokenScope(req, res, USERS_SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + try { + const id = parseInt(req.params.id as string, 10); + const db = DatabaseService.getInstance(); + const user = db.getUser(id); + if (!user) { + res.status(404).json({ error: 'User not found' }); + return; + } + + const { username, password, role } = req.body; + const updates: Partial<{ username: string; password_hash: string; role: string }> = {}; + + if (username !== undefined) { + const usernameError = validateUsername(username); + if (usernameError) { + res.status(400).json({ error: usernameError }); + return; + } + const existing = db.getUserByUsername(username); + if (existing && existing.id !== id) { + res.status(409).json({ error: 'A user with this username already exists' }); + return; + } + updates.username = username; + } + + if (role !== undefined) { + if (!VALID_USER_ROLES.includes(role)) { + res.status(400).json({ error: 'Role must be "admin", "viewer", "deployer", "node-admin", or "auditor"' }); + return; + } + if (roleRequiresAdmiral(role) && !requireAdmiral(req, res)) return; + if (user.username === req.user!.username && role !== user.role) { + res.status(400).json({ error: 'Cannot change your own role' }); + return; + } + if (user.role === 'admin' && role !== 'admin' && db.getAdminCount() <= 1) { + res.status(400).json({ error: 'Cannot demote the only admin user' }); + return; + } + updates.role = role; + } + + if (password !== undefined) { + // Prevent setting passwords on SSO-provisioned users (would enable a + // local-login bypass). + if (user.auth_provider !== 'local') { + res.status(400).json({ error: 'Cannot set a password on an SSO-provisioned user.' }); + return; + } + if (typeof password !== 'string' || password.length < MIN_PASSWORD_LENGTH) { + res.status(400).json({ error: `Password must be at least ${MIN_PASSWORD_LENGTH} characters` }); + return; + } + updates.password_hash = await bcrypt.hash(password, BCRYPT_SALT_ROUNDS); + } + + db.updateUser(id, updates); + // Invalidate the user's active sessions when their role or password changes. + if (updates.role || updates.password_hash) { + db.bumpTokenVersion(id); + } + console.log('[Users] Updated user', id, 'fields:', Object.keys(updates).join(', '), 'by:', req.user!.username); + res.json({ success: true }); + } catch (error) { + console.error('[Users] Update error:', error); + res.status(500).json({ error: 'Failed to update user' }); + } +}); + +usersRouter.delete('/:id', authMiddleware, async (req: Request, res: Response): Promise => { + if (rejectApiTokenScope(req, res, USERS_SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + try { + const id = parseInt(req.params.id as string, 10); + const db = DatabaseService.getInstance(); + const user = db.getUser(id); + if (!user) { + res.status(404).json({ error: 'User not found' }); + return; + } + + if (user.username === req.user!.username) { + res.status(400).json({ error: 'Cannot delete your own account' }); + return; + } + + if (user.role === 'admin' && db.getAdminCount() <= 1) { + res.status(400).json({ error: 'Cannot delete the only admin user' }); + return; + } + + db.deleteUser(id); + console.log('[Users] Deleted:', user.username, '(id:', id, ') by:', req.user!.username); + res.json({ success: true }); + } catch (error) { + console.error('[Users] Delete error:', error); + res.status(500).json({ error: 'Failed to delete user' }); + } +}); + +/** + * Admin reset: clear a target user's MFA enrolment and force re-auth. Used + * when a user has lost their authenticator AND exhausted their backup codes, + * and another admin is available. For total lockout (including sole admin), + * see the CLI `reset-mfa` command. + */ +usersRouter.post('/:id/mfa/reset', authMiddleware, (req: Request, res: Response): void => { + if (rejectApiTokenScope(req, res, USERS_SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + try { + const id = parseInt(req.params.id as string, 10); + if (!Number.isFinite(id)) { + res.status(400).json({ error: 'Invalid user id' }); + return; + } + const db = DatabaseService.getInstance(); + const target = db.getUser(id); + if (!target) { + res.status(404).json({ error: 'User not found' }); + return; + } + db.deleteUserMfa(id); + db.bumpTokenVersion(id); + try { + db.insertAuditLog({ + timestamp: Date.now(), + username: req.user!.username, + method: 'POST', + path: req.originalUrl, + status_code: 200, + node_id: null, + ip_address: req.ip || 'unknown', + summary: `Admin reset two-factor authentication for ${target.username}`, + }); + } catch (err) { + console.warn('[MFA] Admin reset audit log write failed:', getErrorMessage(err, 'unknown')); + } + console.log('[MFA] Admin reset: target=', target.username, 'by=', req.user!.username); + if (isDebugEnabled()) { + console.log('[MFA:diag] admin-reset target=', target.username, 'actor=', req.user!.username); + } + res.json({ success: true }); + } catch (error) { + console.error('[MFA] Admin reset error:', getErrorMessage(error, 'unknown')); + res.status(500).json({ error: 'Failed to reset two-factor authentication' }); + } +}); + +// --- Scoped Role Assignments (Admiral) --- + +usersRouter.get('/:id/roles', authMiddleware, (req: Request, res: Response): void => { + if (rejectApiTokenScope(req, res, USERS_SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + if (!requireAdmiral(req, res)) return; + try { + const userId = parseInt(req.params.id as string, 10); + const db = DatabaseService.getInstance(); + if (!db.getUser(userId)) { + res.status(404).json({ error: 'User not found' }); + return; + } + const assignments = db.getAllRoleAssignments(userId); + res.json(assignments); + } catch (error) { + console.error('[Roles] List error:', error); + res.status(500).json({ error: 'Failed to fetch role assignments' }); + } +}); + +usersRouter.post('/:id/roles', authMiddleware, (req: Request, res: Response): void => { + if (rejectApiTokenScope(req, res, USERS_SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + if (!requireAdmiral(req, res)) return; + try { + const userId = parseInt(req.params.id as string, 10); + const { role, resource_type, resource_id } = req.body; + + if (!VALID_ASSIGNMENT_ROLES.includes(role)) { + res.status(400).json({ error: 'Invalid role' }); + return; + } + if (!VALID_RESOURCE_TYPES.includes(resource_type)) { + res.status(400).json({ error: 'Invalid resource type' }); + return; + } + if (!resource_id || typeof resource_id !== 'string') { + res.status(400).json({ error: 'resource_id is required' }); + return; + } + + const db = DatabaseService.getInstance(); + if (!db.getUser(userId)) { + res.status(404).json({ error: 'User not found' }); + return; + } + + try { + const id = db.addRoleAssignment({ user_id: userId, role, resource_type, resource_id }); + console.log('[Roles] Assigned', role, 'on', resource_type, resource_id, 'to user', userId, 'by:', req.user!.username); + res.status(201).json({ id, user_id: userId, role, resource_type, resource_id }); + } catch (err: unknown) { + if (isSqliteUniqueViolation(err)) { + res.status(409).json({ error: 'This role assignment already exists' }); + return; + } + throw err; + } + } catch (error) { + console.error('[Roles] Create error:', error); + res.status(500).json({ error: 'Failed to add role assignment' }); + } +}); + +usersRouter.delete('/:id/roles/:assignId', authMiddleware, (req: Request, res: Response): void => { + if (rejectApiTokenScope(req, res, USERS_SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + if (!requireAdmiral(req, res)) return; + try { + const userId = parseInt(req.params.id as string, 10); + const assignId = parseInt(req.params.assignId as string, 10); + const db = DatabaseService.getInstance(); + + const assignment = db.getRoleAssignmentById(assignId); + if (!assignment || assignment.user_id !== userId) { + res.status(404).json({ error: 'Role assignment not found' }); + return; + } + + db.deleteRoleAssignment(assignId); + console.log('[Roles] Removed assignment', assignId, 'from user', userId, 'by:', req.user!.username); + res.json({ success: true }); + } catch (error) { + console.error('[Roles] Delete error:', error); + res.status(500).json({ error: 'Failed to delete role assignment' }); + } +}); diff --git a/backend/src/routes/webhooks.ts b/backend/src/routes/webhooks.ts new file mode 100644 index 00000000..5285e8ef --- /dev/null +++ b/backend/src/routes/webhooks.ts @@ -0,0 +1,158 @@ +import { Router, type Request, type Response } from 'express'; +import { DatabaseService } from '../services/DatabaseService'; +import { WebhookService } from '../services/WebhookService'; +import { GitSourceService } from '../services/GitSourceService'; +import { LicenseService } from '../services/LicenseService'; +import { authMiddleware } from '../middleware/auth'; +import { requirePaid, requireAdmin } from '../middleware/tierGates'; +import { webhookTriggerLimiter } from '../middleware/rateLimiters'; + +const VALID_WEBHOOK_ACTIONS = ['deploy', 'restart', 'stop', 'start', 'pull', 'git-pull']; + +export const webhooksRouter = Router(); + +webhooksRouter.get('/', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + try { + const webhooks = DatabaseService.getInstance().getWebhooks(); + const svc = WebhookService.getInstance(); + res.json(webhooks.map(w => ({ ...w, secret: svc.maskSecret(w.secret) }))); + } catch (error) { + console.error('[Webhooks] List error:', error); + res.status(500).json({ error: 'Failed to list webhooks' }); + } +}); + +webhooksRouter.post('/', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + try { + const { name, stack_name, action, enabled } = req.body; + if (!name || !stack_name || !action) { + res.status(400).json({ error: 'name, stack_name, and action are required' }); + return; + } + if (!VALID_WEBHOOK_ACTIONS.includes(action)) { + res.status(400).json({ error: `action must be one of: ${VALID_WEBHOOK_ACTIONS.join(', ')}` }); + return; + } + if (action === 'git-pull' && !GitSourceService.getInstance().get(stack_name)) { + res.status(400).json({ error: 'Configure a Git source for this stack before creating a git-pull webhook' }); + return; + } + + const svc = WebhookService.getInstance(); + const secret = svc.generateSecret(); + const id = DatabaseService.getInstance().addWebhook({ + name, stack_name, action, secret, enabled: enabled !== false, + }); + + // Return the full secret only on creation. + res.status(201).json({ id, secret }); + } catch (error) { + console.error('[Webhooks] Create error:', error); + res.status(500).json({ error: 'Failed to create webhook' }); + } +}); + +webhooksRouter.put('/:id', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + try { + const id = parseInt(req.params.id as string, 10); + const webhook = DatabaseService.getInstance().getWebhook(id); + if (!webhook) { res.status(404).json({ error: 'Webhook not found' }); return; } + + const { name, stack_name, action, enabled } = req.body; + if (action && !VALID_WEBHOOK_ACTIONS.includes(action)) { + res.status(400).json({ error: `action must be one of: ${VALID_WEBHOOK_ACTIONS.join(', ')}` }); + return; + } + if (action === 'git-pull') { + const targetStack = stack_name || webhook.stack_name; + if (!GitSourceService.getInstance().get(targetStack)) { + res.status(400).json({ error: 'Configure a Git source for this stack before enabling a git-pull webhook' }); + return; + } + } + + DatabaseService.getInstance().updateWebhook(id, { name, stack_name, action, enabled }); + res.json({ success: true }); + } catch (error) { + console.error('[Webhooks] Update error:', error); + res.status(500).json({ error: 'Failed to update webhook' }); + } +}); + +webhooksRouter.delete('/:id', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + if (!requirePaid(req, res)) return; + try { + const id = parseInt(req.params.id as string, 10); + DatabaseService.getInstance().deleteWebhook(id); + res.json({ success: true }); + } catch (error) { + console.error('[Webhooks] Delete error:', error); + res.status(500).json({ error: 'Failed to delete webhook' }); + } +}); + +webhooksRouter.get('/:id/history', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requirePaid(req, res)) return; + try { + const id = parseInt(req.params.id as string, 10); + const executions = DatabaseService.getInstance().getWebhookExecutions(id); + res.json(executions); + } catch (error) { + console.error('[Webhooks] History error:', error); + res.status(500).json({ error: 'Failed to fetch webhook history' }); + } +}); + +// Public: authenticated via HMAC signature, not session cookie. +webhooksRouter.post('/:id/trigger', webhookTriggerLimiter, async (req: Request, res: Response): Promise => { + try { + const id = parseInt(req.params.id as string, 10); + const db = DatabaseService.getInstance(); + const webhook = db.getWebhook(id); + + if (!webhook || !webhook.enabled) { + res.status(404).json({ error: 'Webhook not found or disabled' }); + return; + } + + // Trigger only works with an active Skipper or Admiral license. + if (LicenseService.getInstance().getTier() !== 'paid') { + res.status(403).json({ error: 'This feature requires a Skipper or Admiral license.', code: 'PAID_REQUIRED' }); + return; + } + + const signature = req.headers['x-webhook-signature'] as string; + if (!signature) { + res.status(401).json({ error: 'Missing X-Webhook-Signature header' }); + return; + } + + const rawBody = req.rawBody?.toString('utf-8') ?? JSON.stringify(req.body ?? {}); + const svc = WebhookService.getInstance(); + if (!svc.validateSignature(rawBody, webhook.secret, signature)) { + res.status(401).json({ error: 'Invalid signature' }); + return; + } + + // Use action from body if provided, otherwise use webhook default. + const action = req.body?.action || webhook.action; + const triggerSource = req.headers['user-agent'] || req.ip || null; + + // Execute asynchronously; return 202 immediately. + res.status(202).json({ message: 'Webhook accepted', action }); + + const atomic = LicenseService.getInstance().getTier() === 'paid'; + svc.execute(id, action, triggerSource, atomic).catch(err => { + console.error(`[Webhooks] Execution error for webhook ${id}:`, err); + }); + } catch (error) { + console.error('[Webhooks] Trigger error:', error); + res.status(500).json({ error: 'Failed to process webhook' }); + } +});