feat: add an on-demand node-wide security scan with live progress (#1367)

Add a "Scan this node" action on the Security overview that scans, in one pass,
any combination of three types: image vulnerabilities, image secrets, and
compose misconfigurations. Progress streams live into the deploy-feedback modal.

- TrivyService.scanNode runs the selected scanners across the node's images and,
  for misconfig, every stack's compose file, behind a per-node lock and tolerant
  of per-item failures. The existing scanAllNodeImages becomes a thin vuln-only
  wrapper over the shared image loop, so scheduled scans are unchanged.
- POST /api/security/scan-node (admin, scanner-gated) streams sanitized progress
  to the deploy terminal and returns a combined summary. Secret scans stream
  counts only, never matched values.
- Frontend adds a "scan" action verb and a ScanNodeLauncher wired into the
  overview; the scan stays bound to the node it started on even if the active
  node changes mid-run.
This commit is contained in:
Anso
2026-06-12 19:07:45 -04:00
committed by GitHub
parent ebf66fd92a
commit ef5a3f00a7
11 changed files with 641 additions and 6 deletions
@@ -0,0 +1,106 @@
/**
* POST /api/security/scan-node -> on-demand node-wide scan (admin, scanner-gated).
* Covers the route contract: auth, admin, scanner availability, strict body
* validation, the success summary, and the per-node-busy conflict. The scan
* engine itself is mocked; TrivyService.scanNode is unit-tested separately.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import request from 'supertest';
import bcrypt from 'bcrypt';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
import type { ScanNodeResult } from '../services/TrivyService';
let tmpDir: string;
let app: import('express').Express;
let adminCookie: string;
let viewerCookie: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let LicenseService: typeof import('../services/LicenseService').LicenseService;
let TrivyService: typeof import('../services/TrivyService').default;
const SUMMARY: ScanNodeResult = {
images: { scanned: 2, skipped: 1, failed: 0, totalImages: 3, processedImages: 3, truncated: false, severity: { critical: 1, high: 2, medium: 0, low: 0, unknown: 0 }, violations: [] },
stacks: { scanned: 1, failed: 0, total: 1 },
severity: { critical: 1, high: 2, medium: 0, low: 0, unknown: 0 },
violations: [],
};
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ LicenseService } = await import('../services/LicenseService'));
TrivyService = (await import('../services/TrivyService')).default;
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
({ app } = await import('../index'));
adminCookie = await loginAsTestAdmin(app);
const viewerHash = await bcrypt.hash('snviewer1', 1);
DatabaseService.getInstance().addUser({ username: 'sn-viewer', password_hash: viewerHash, role: 'viewer' });
const res = await request(app).post('/api/auth/login').send({ username: 'sn-viewer', password: 'snviewer1' });
const cookies = res.headers['set-cookie'] as string | string[];
viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies;
});
afterAll(() => cleanupTestDb(tmpDir));
function svc() {
return TrivyService.getInstance();
}
beforeEach(() => {
vi.restoreAllMocks();
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
vi.spyOn(svc(), 'isTrivyAvailable').mockReturnValue(true);
});
describe('POST /api/security/scan-node', () => {
it('requires authentication', async () => {
const res = await request(app).post('/api/security/scan-node').send({ vulns: true });
expect(res.status).toBe(401);
});
it('requires admin (viewer is rejected)', async () => {
const res = await request(app).post('/api/security/scan-node').set('Cookie', viewerCookie).send({ vulns: true });
expect(res.status).toBe(403);
});
it('returns 503 when the scanner is unavailable', async () => {
vi.spyOn(svc(), 'isTrivyAvailable').mockReturnValue(false);
const res = await request(app).post('/api/security/scan-node').set('Cookie', adminCookie).send({ vulns: true });
expect(res.status).toBe(503);
});
it('returns 400 when no scan type is selected', async () => {
const scanNode = vi.spyOn(svc(), 'scanNode');
const res = await request(app).post('/api/security/scan-node').set('Cookie', adminCookie)
.send({ vulns: false, secrets: false, misconfig: false });
expect(res.status).toBe(400);
expect(scanNode).not.toHaveBeenCalled();
});
it('returns 400 when a scan-type flag is not a boolean', async () => {
const res = await request(app).post('/api/security/scan-node').set('Cookie', adminCookie).send({ vulns: 'yes' });
expect(res.status).toBe(400);
});
it('runs the scan and returns the summary, passing the selected types', async () => {
const scanNode = vi.spyOn(svc(), 'scanNode').mockResolvedValue(SUMMARY);
const res = await request(app).post('/api/security/scan-node').set('Cookie', adminCookie)
.send({ vulns: true, secrets: false, misconfig: true });
expect(res.status).toBe(200);
expect(res.body).toMatchObject({ stacks: { scanned: 1 }, severity: { critical: 1, high: 2 } });
// No progress socket is opened in the test, so onProgress resolves undefined.
expect(scanNode).toHaveBeenCalledWith(
expect.any(Number),
{ vulns: true, secrets: false, misconfig: true },
'manual',
undefined,
);
});
it('returns 409 when the node is already being scanned', async () => {
vi.spyOn(svc(), 'scanNode').mockRejectedValue(new Error('Already scanning this node'));
const res = await request(app).post('/api/security/scan-node').set('Cookie', adminCookie).send({ vulns: true });
expect(res.status).toBe(409);
});
});
@@ -0,0 +1,118 @@
/**
* Unit tests for TrivyService.scanNode: scan-type selection, the per-node
* concurrency lock, scanner-availability and empty-selection guards, and
* partial-failure tolerance. The actual Trivy/Docker calls are mocked so the
* orchestration is exercised without a scanner.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { DatabaseService, type VulnerabilityScan } from '../services/DatabaseService';
let tmpDir: string;
let TrivyService: typeof import('../services/TrivyService').default;
let DockerController: typeof import('../services/DockerController').default;
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
beforeAll(async () => {
tmpDir = await setupTestDb();
TrivyService = (await import('../services/TrivyService')).default;
DockerController = (await import('../services/DockerController')).default;
({ FileSystemService } = await import('../services/FileSystemService'));
});
afterAll(() => cleanupTestDb(tmpDir));
function svc() {
return TrivyService.getInstance();
}
function fakeRow(over: Partial<VulnerabilityScan> = {}): VulnerabilityScan {
return {
id: 1, node_id: 1, image_ref: 'a:1', image_digest: null, scanned_at: Date.now(),
total_vulnerabilities: 0, critical_count: 1, high_count: 2, medium_count: 0, low_count: 0, unknown_count: 0,
fixable_count: 0, secret_count: 0, misconfig_count: 0, scanners_used: 'vuln', highest_severity: 'HIGH',
os_info: null, trivy_version: null, scan_duration_ms: null, triggered_by: 'manual', status: 'completed',
error: null, stack_context: null, policy_evaluation: null, ...over,
} as VulnerabilityScan;
}
let prevSource: unknown;
beforeEach(() => {
vi.restoreAllMocks();
prevSource = (svc() as unknown as { source: unknown }).source;
(svc() as unknown as { source: string }).source = 'managed';
(svc() as unknown as { scanningNodes: Set<number> }).scanningNodes.clear();
});
afterEach(() => {
(svc() as unknown as { source: unknown }).source = prevSource;
});
describe('TrivyService.scanNode', () => {
it('rejects when no scan type is selected', async () => {
await expect(svc().scanNode(1, { vulns: false, secrets: false, misconfig: false })).rejects.toThrow(/at least one/i);
});
it('throws when the scanner is unavailable', async () => {
(svc() as unknown as { source: string }).source = 'none';
await expect(svc().scanNode(1, { vulns: true, secrets: false, misconfig: false })).rejects.toThrow(/not available/i);
});
it('refuses a second scan while the node is already scanning', async () => {
(svc() as unknown as { scanningNodes: Set<number> }).scanningNodes.add(1);
await expect(svc().scanNode(1, { vulns: true, secrets: false, misconfig: false })).rejects.toThrow(/already scanning/i);
});
it('scans images for the selected scanners and skips stacks when misconfig is off', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getImages: async () => [{ RepoTags: ['a:1'] }] } as never);
const run = vi.spyOn(svc(), 'runScanAndPersist').mockResolvedValue(fakeRow());
const stack = vi.spyOn(svc(), 'scanComposeStack');
const result = await svc().scanNode(1, { vulns: true, secrets: true, misconfig: false });
expect(run).toHaveBeenCalledWith('a:1', 1, 'manual', null, { scanners: ['vuln', 'secret'] });
expect(stack).not.toHaveBeenCalled();
expect(result.images).not.toBeNull();
expect(result.stacks).toBeNull();
expect(result.severity).toMatchObject({ critical: 1, high: 2 });
});
it('scans secrets only and keys the digest cache on the scanner set', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getImages: async () => [{ RepoTags: ['a:1'] }] } as never);
// Force a digest so the cache lookup runs; return null so the scan proceeds.
vi.spyOn(svc() as unknown as { getImageDigest: (r: string, n: number) => Promise<string | null> }, 'getImageDigest')
.mockResolvedValue('sha256:abc');
const cacheLookup = vi.spyOn(DatabaseService.getInstance(), 'getLatestScanByDigest').mockReturnValue(null);
const run = vi.spyOn(svc(), 'runScanAndPersist').mockResolvedValue(fakeRow({ scanners_used: 'secret' }));
await svc().scanNode(1, { vulns: false, secrets: true, misconfig: false });
// A secrets-only scan must not reuse a vuln-only cached row.
expect(cacheLookup).toHaveBeenCalledWith('sha256:abc', 'secret');
expect(run).toHaveBeenCalledWith('a:1', 1, 'manual', null, { scanners: ['secret'] });
});
it('scans every stack for misconfig and skips images when vulns/secrets are off', async () => {
vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({ getStacks: async () => ['web', 'db'] } as never);
const stack = vi.spyOn(svc(), 'scanComposeStack').mockResolvedValue(fakeRow({ misconfig_count: 3, scanners_used: 'config' }));
const run = vi.spyOn(svc(), 'runScanAndPersist');
const result = await svc().scanNode(1, { vulns: false, secrets: false, misconfig: true });
expect(stack).toHaveBeenCalledTimes(2);
expect(run).not.toHaveBeenCalled();
expect(result.stacks).toMatchObject({ scanned: 2, failed: 0, total: 2 });
expect(result.images).toBeNull();
});
it('counts a failed stack without aborting the batch', async () => {
vi.spyOn(FileSystemService, 'getInstance').mockReturnValue({ getStacks: async () => ['ok', 'bad'] } as never);
vi.spyOn(svc(), 'scanComposeStack').mockImplementation(async (_nodeId: number, name: string) => {
if (name === 'bad') throw new Error('boom');
return fakeRow({ misconfig_count: 1 });
});
const result = await svc().scanNode(1, { vulns: false, secrets: false, misconfig: true });
expect(result.stacks).toMatchObject({ scanned: 1, failed: 1, total: 2 });
});
});
+68
View File
@@ -18,6 +18,7 @@ import { blockIfReplica } from '../middleware/fleetSyncGuards';
import { validateStackPatternForRedos } from './fleet';
import { FINDING_SEVERITIES, POLICY_SEVERITIES } from '../utils/severity';
import { DEFAULT_POLICY_PACKS } from '../services/policy-packs';
import { getTerminalWs, DEPLOY_SESSION_HEADER } from '../websocket/generic';
const CVE_ID_RE = /^(CVE-\d{4}-\d{4,}|GHSA-[\w-]{14,})$/;
// Trivy emits misconfig rule ids in two shapes that Sencho persists verbatim:
@@ -329,6 +330,73 @@ securityRouter.post('/scan/stack', authMiddleware, async (req: Request, res: Res
}
});
// On-demand node-wide scan: images for the selected scanners (vuln/secret) and,
// when requested, every stack's compose config for misconfigurations. Streams
// sanitized progress to the deploy-feedback terminal when the client opened one.
securityRouter.post('/scan-node', authMiddleware, async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
const svc = TrivyService.getInstance();
if (!svc.isTrivyAvailable()) {
res.status(503).json({ error: 'Trivy is not available on this host' });
return;
}
const body = req.body ?? {};
if (![body.vulns, body.secrets, body.misconfig].every((v) => v === undefined || typeof v === 'boolean')) {
res.status(400).json({ error: 'vulns, secrets and misconfig must be booleans' });
return;
}
const vulns = body.vulns === true;
const secrets = body.secrets === true;
const misconfig = body.misconfig === true;
if (!vulns && !secrets && !misconfig) {
res.status(400).json({ error: 'Select at least one scan type' });
return;
}
const nodeId = req.nodeId;
// Stream progress to the deploy-feedback terminal only when the client supplied
// a session id: a bare getTerminalWs(undefined) falls back to the most recent
// terminal and could cross-stream this scan into an unrelated deploy. scanNode
// emits only counts and rule ids, never raw secret values; the wrapper here
// additionally strips CR/LF and caps line length before sending.
const sessionId = req.get(DEPLOY_SESSION_HEADER);
const ws = sessionId ? getTerminalWs(sessionId) : undefined;
const onProgress = ws
? (line: string): void => {
try { ws.send(line.replace(/[\r\n]+/g, ' ').slice(0, 2000) + '\r\n'); }
catch { /* socket closed mid-scan; the scan still runs to completion */ }
}
: undefined;
try {
DatabaseService.getInstance().insertAuditLog({
timestamp: Date.now(),
username: req.user?.username ?? 'unknown',
method: req.method,
path: req.originalUrl || req.url,
status_code: 200,
node_id: nodeId,
ip_address: req.ip ?? '',
summary: `node-wide scan triggered (vulns=${vulns} secrets=${secrets} misconfig=${misconfig})`,
});
} catch (auditErr) {
console.warn('[Security] failed to record node-scan audit log:', getErrorMessage(auditErr, 'unknown'));
}
try {
const result = await svc.scanNode(nodeId, { vulns, secrets, misconfig }, 'manual', onProgress);
res.status(200).json(result);
} catch (err) {
const message = getErrorMessage(err, 'Failed to run node scan');
if (message === 'Already scanning this node') {
res.status(409).json({ error: message });
return;
}
console.error('[Security] node-wide scan failed:', sanitizeForLog(message));
res.status(500).json({ error: 'Failed to run node scan' });
}
});
securityRouter.get('/scans', authMiddleware, (req: Request, res: Response) => {
try {
const imageRef = typeof req.query.imageRef === 'string' ? req.query.imageRef : undefined;
+127 -2
View File
@@ -150,6 +150,25 @@ export interface ScanAllNodeImagesResult {
violations: ScanAllNodeImagesViolation[];
}
/** Which scan types a node-wide scan should run. At least one must be true. */
export interface ScanNodeOptions {
vulns: boolean;
secrets: boolean;
misconfig: boolean;
}
/** Combined result of a node-wide scan (images for vuln/secret + stacks for misconfig). */
export interface ScanNodeResult {
/** Image scan totals, or null when neither vuln nor secret scanning ran. */
images: ScanAllNodeImagesResult | null;
/** Stack misconfig totals, or null when misconfig scanning did not run. */
stacks: { scanned: number; failed: number; total: number } | null;
/** Severity totals across images AND stacks (a superset of `images.severity`). */
severity: ScanAllNodeImagesSeverityTotals;
/** Policy violations; only image scans contribute, stacks never do. */
violations: ScanAllNodeImagesViolation[];
}
export interface TrivyVulnerability {
vulnerabilityId: string;
pkgName: string;
@@ -347,6 +366,9 @@ class TrivyService {
private binaryPath: string | null = null;
private source: TrivySource = 'none';
private scanningImages: Set<string> = new Set();
// Per-node lock so two node-wide scans (or a node scan and the scheduled
// sweep) never overlap an expensive Trivy run on the same node.
private scanningNodes: Set<number> = new Set();
private cacheDirEnsured: string | null = null;
private detectionTimestamp = 0;
@@ -1033,13 +1055,34 @@ class TrivyService {
}
}
/**
* Vuln-only image sweep for the scheduler. Thin compatibility wrapper over
* the generalized image loop; the signature, semantics, limits, cache, and
* ScanAllNodeImagesResult shape are unchanged.
*/
async scanAllNodeImages(
nodeId: number,
triggeredBy: VulnScanTrigger = 'scheduled',
): Promise<ScanAllNodeImagesResult> {
return this.scanNodeImages(nodeId, triggeredBy, ['vuln']);
}
/**
* Scan every image on a node with the given scanners, reusing the digest
* cache (keyed by the scanner set), throttle, and image/duration caps. Emits
* one sanitized progress line per image when `onProgress` is supplied. A
* failed image increments `failed` and does not abort the batch.
*/
private async scanNodeImages(
nodeId: number,
triggeredBy: VulnScanTrigger,
scanners: readonly TrivyScanner[],
onProgress?: (line: string) => void,
): Promise<ScanAllNodeImagesResult> {
if (this.source === 'none') {
throw new Error('Trivy is not available on this host');
}
const scannersUsed = normalizeScanners(scanners).join(',');
const batchStartedAt = Date.now();
const images = await DockerController.getInstance(nodeId).getImages();
const imageRefs = new Set<string>();
@@ -1106,27 +1149,31 @@ class TrivyService {
break;
}
processedImages++;
onProgress?.(`scanning ${ref}`);
try {
const digest = await this.getImageDigest(ref, nodeId);
if (digest) {
if (countedDigests.has(digest)) continue;
const cached =
DatabaseService.getInstance().getLatestScanByDigest(digest, 'vuln');
DatabaseService.getInstance().getLatestScanByDigest(digest, scannersUsed);
if (cached && Date.now() - cached.scanned_at < DIGEST_CACHE_TTL_MS) {
skipped++;
addSeverity(cached);
collectViolation(cached);
countedDigests.add(digest);
onProgress?.(`${ref}: cached (${cached.critical_count}C ${cached.high_count}H)`);
continue;
}
}
const fresh = await this.runScanAndPersist(ref, nodeId, triggeredBy, null);
const fresh = await this.runScanAndPersist(ref, nodeId, triggeredBy, null, { scanners });
addSeverity(fresh);
collectViolation(fresh);
scanned++;
if (digest) countedDigests.add(digest);
onProgress?.(`${ref}: ${fresh.critical_count}C ${fresh.high_count}H${fresh.secret_count ? ` ${fresh.secret_count} secret` : ''}`);
} catch (err) {
failed++;
onProgress?.(`${ref}: scan failed`);
console.warn(`[Trivy] Failed to scan ${ref}:`, getErrorMessage(err, 'unknown error'));
}
await new Promise((r) => setTimeout(r, 300));
@@ -1149,6 +1196,84 @@ class TrivyService {
};
}
/**
* On-demand node-wide scan: images for the selected scanners (vuln/secret)
* and, when requested, every stack's compose config for misconfigurations.
* Streams sanitized progress lines via `onProgress`. A per-node lock prevents
* an overlapping sweep; a failed image/stack is counted, not fatal.
*/
async scanNode(
nodeId: number,
opts: ScanNodeOptions,
triggeredBy: VulnScanTrigger = 'manual',
onProgress?: (line: string) => void,
): Promise<ScanNodeResult> {
if (this.source === 'none') {
throw new Error('Trivy is not available on this host');
}
if (!opts.vulns && !opts.secrets && !opts.misconfig) {
throw new Error('Select at least one scan type');
}
if (this.scanningNodes.has(nodeId)) {
throw new Error('Already scanning this node');
}
this.scanningNodes.add(nodeId);
const severity = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 };
const addSeverity = (s: { critical: number; high: number; medium: number; low: number; unknown: number }): void => {
severity.critical += s.critical;
severity.high += s.high;
severity.medium += s.medium;
severity.low += s.low;
severity.unknown += s.unknown;
};
const violations: ScanAllNodeImagesViolation[] = [];
let images: ScanAllNodeImagesResult | null = null;
let stacks: { scanned: number; failed: number; total: number } | null = null;
try {
const scanners: TrivyScanner[] = [];
if (opts.vulns) scanners.push('vuln');
if (opts.secrets) scanners.push('secret');
if (scanners.length > 0) {
images = await this.scanNodeImages(nodeId, triggeredBy, scanners, onProgress);
addSeverity(images.severity);
violations.push(...images.violations);
}
if (opts.misconfig) {
const stackNames = await FileSystemService.getInstance(nodeId).getStacks();
let scanned = 0;
let failed = 0;
for (const name of stackNames) {
onProgress?.(`scanning stack:${name}`);
try {
const row = await this.scanComposeStack(nodeId, name, triggeredBy);
addSeverity({
critical: row.critical_count, high: row.high_count, medium: row.medium_count,
low: row.low_count, unknown: row.unknown_count,
});
scanned++;
onProgress?.(`stack:${name}: ${row.misconfig_count} misconfigurations`);
} catch (err) {
failed++;
onProgress?.(`stack:${name}: scan failed`);
console.warn(`[Trivy] Failed to scan stack ${name}:`, getErrorMessage(err, 'unknown error'));
}
}
stacks = { scanned, failed, total: stackNames.length };
}
const imagePart = images
? `${images.scanned} scanned / ${images.skipped} cached / ${images.failed} failed images`
: 'images skipped';
const stackPart = stacks ? `, ${stacks.scanned} stacks (${stacks.failed} failed)` : '';
onProgress?.(`Scan complete: ${imagePart}${stackPart}`);
return { images, stacks, severity, violations };
} finally {
this.scanningNodes.delete(nodeId);
}
}
async generateSBOM(imageRef: string, format: SbomFormat): Promise<string> {
const binary = this.binaryPath;
if (!binary) {