mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-24 08:57:25 +00:00
feat(security): severity-aware scheduled scan notifications (#654)
Enrich scheduled vulnerability scan completion notifications with per-severity CVE counts so recipients can triage from the message body alone. Expose the scan action in the schedule creation UI, require an explicit node_id, and harden fire-and-forget alert dispatches so a failing webhook cannot crash the scheduler. Notification body now reports scanned/skipped/failed counts plus critical/high/medium totals aggregated across fresh and cached scans, reflecting the current node posture rather than only what was newly scanned on this run.
This commit is contained in:
Anso
@@ -47,7 +47,12 @@ const {
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockGetProxyTarget: vi.fn().mockReturnValue(null),
|
||||
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
|
||||
mockScanAllNodeImages: vi.fn().mockResolvedValue({ scanned: 0, skipped: 0, failed: 0 }),
|
||||
mockScanAllNodeImages: vi.fn().mockResolvedValue({
|
||||
scanned: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
severity: { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 },
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
@@ -777,9 +782,33 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
};
|
||||
}
|
||||
|
||||
function scanResult(opts: {
|
||||
scanned?: number;
|
||||
skipped?: number;
|
||||
failed?: number;
|
||||
critical?: number;
|
||||
high?: number;
|
||||
medium?: number;
|
||||
low?: number;
|
||||
unknown?: number;
|
||||
} = {}) {
|
||||
return {
|
||||
scanned: opts.scanned ?? 0,
|
||||
skipped: opts.skipped ?? 0,
|
||||
failed: opts.failed ?? 0,
|
||||
severity: {
|
||||
critical: opts.critical ?? 0,
|
||||
high: opts.high ?? 0,
|
||||
medium: opts.medium ?? 0,
|
||||
low: opts.low ?? 0,
|
||||
unknown: opts.unknown ?? 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it('dispatches info-level notification when scan completes cleanly', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeScanTask());
|
||||
mockScanAllNodeImages.mockResolvedValue({ scanned: 3, skipped: 1, failed: 0 });
|
||||
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 3, skipped: 1 }));
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(200);
|
||||
@@ -798,7 +827,7 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
|
||||
it('dispatches warning-level notification when scan has failures', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 201, name: 'flaky-scan' }));
|
||||
mockScanAllNodeImages.mockResolvedValue({ scanned: 5, skipped: 0, failed: 2 });
|
||||
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 5, failed: 2 }));
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(201);
|
||||
@@ -812,7 +841,7 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
|
||||
it('passes target_id to dispatchAlert when set', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 202, target_id: 'web-stack' }));
|
||||
mockScanAllNodeImages.mockResolvedValue({ scanned: 1, skipped: 0, failed: 0 });
|
||||
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 1 }));
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(202);
|
||||
@@ -828,9 +857,9 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeScanTask({
|
||||
id: 204,
|
||||
name: 'recovered-scan',
|
||||
last_status: 'failure', // Previous run failed
|
||||
last_status: 'failure',
|
||||
}));
|
||||
mockScanAllNodeImages.mockResolvedValue({ scanned: 2, skipped: 0, failed: 0 });
|
||||
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 2 }));
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(204);
|
||||
@@ -853,7 +882,7 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
target_id: 'my-stack',
|
||||
node_id: 1,
|
||||
created_by: 'admin',
|
||||
last_status: null, // No previous failure → no recovery notification
|
||||
last_status: null,
|
||||
});
|
||||
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Service: 'web' }]);
|
||||
|
||||
@@ -862,6 +891,77 @@ describe('SchedulerService - scheduled scan notifications', () => {
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('dispatches error-level notification when Trivy is unavailable', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 205, target_id: 'payment-stack' }));
|
||||
mockIsTrivyAvailable.mockReturnValueOnce(false);
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(205);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
expect.stringMatching(/failed.*Trivy/i),
|
||||
'payment-stack',
|
||||
);
|
||||
});
|
||||
|
||||
it('includes severity counts in the notification message', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 206 }));
|
||||
mockScanAllNodeImages.mockResolvedValue(
|
||||
scanResult({ scanned: 3, skipped: 1, critical: 2, high: 5, medium: 10 }),
|
||||
);
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(206);
|
||||
|
||||
const message = mockDispatchAlert.mock.calls[0][1] as string;
|
||||
expect(message).toContain('2 critical');
|
||||
expect(message).toContain('5 high');
|
||||
expect(message).toContain('10 medium');
|
||||
});
|
||||
|
||||
it('reports "No images to scan" when the node has nothing to scan', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 207 }));
|
||||
mockScanAllNodeImages.mockResolvedValue(scanResult());
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(207);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
expect.stringContaining('No images to scan'),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('reports "All N image(s) already scanned recently" when every image was cached', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 208 }));
|
||||
mockScanAllNodeImages.mockResolvedValue(scanResult({ skipped: 12 }));
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(208);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
expect.stringContaining('All 12 image(s) already scanned recently'),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('persists the run as success even when notification dispatch throws', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeScanTask({ id: 209 }));
|
||||
mockScanAllNodeImages.mockResolvedValue(scanResult({ scanned: 1 }));
|
||||
mockDispatchAlert.mockRejectedValueOnce(new Error('webhook down'));
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await expect(svc.triggerTask(209)).resolves.not.toThrow();
|
||||
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
||||
expect.any(Number),
|
||||
expect.objectContaining({ status: 'success' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Cleanup ────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -6249,6 +6249,9 @@ app.post('/api/scheduled-tasks', (req: Request, res: Response): void => {
|
||||
if (action === 'scan' && target_type !== 'system') {
|
||||
res.status(400).json({ error: 'Scan action requires target_type "system".' }); return;
|
||||
}
|
||||
if (action === 'scan' && !node_id) {
|
||||
res.status(400).json({ error: 'Scan action requires node_id.' }); return;
|
||||
}
|
||||
if (target_type === 'stack' && (!target_id || !node_id)) {
|
||||
res.status(400).json({ error: 'Stack operations require target_id and node_id.' }); return;
|
||||
}
|
||||
@@ -6370,6 +6373,12 @@ app.put('/api/scheduled-tasks/:id', (req: Request, res: Response): void => {
|
||||
if (finalAction === 'scan' && finalTargetType !== 'system') {
|
||||
res.status(400).json({ error: 'Scan action requires target_type "system".' }); return;
|
||||
}
|
||||
if (finalAction === 'scan') {
|
||||
const finalNodeId = node_id !== undefined ? node_id : existing.node_id;
|
||||
if (!finalNodeId) {
|
||||
res.status(400).json({ error: 'Scan action requires node_id.' }); return;
|
||||
}
|
||||
}
|
||||
|
||||
// Validate prune targets
|
||||
const validPruneTargets = ['containers', 'images', 'networks', 'volumes'];
|
||||
|
||||
@@ -13,6 +13,7 @@ import { captureLocalNodeFiles, captureRemoteNodeFiles } from '../utils/snapshot
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { NotificationService } from './NotificationService';
|
||||
import TrivyService from './TrivyService';
|
||||
import type { ScanAllNodeImagesResult } from './TrivyService';
|
||||
import TrivyInstaller from './TrivyInstaller';
|
||||
|
||||
const TRIVY_UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
||||
@@ -86,7 +87,7 @@ export class SchedulerService {
|
||||
try {
|
||||
await installer.update();
|
||||
await trivy.detectTrivy();
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
`Trivy updated from v${previous} to v${check.latest}`,
|
||||
);
|
||||
@@ -97,7 +98,7 @@ export class SchedulerService {
|
||||
} else {
|
||||
const lastNotified = settings.trivy_last_notified_version || '';
|
||||
if (lastNotified === check.latest) return;
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
`Trivy update available: v${check.latest} (currently v${check.current ?? 'unknown'})`,
|
||||
);
|
||||
@@ -139,6 +140,16 @@ export class SchedulerService {
|
||||
return expr.next().toDate().getTime();
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire a notification without awaiting completion, catching any promise
|
||||
* rejection so the scheduler never crashes on a failed dispatch.
|
||||
*/
|
||||
private safeDispatch(level: 'info' | 'warning' | 'error', message: string, stackName?: string): void {
|
||||
NotificationService.getInstance()
|
||||
.dispatchAlert(level, message, stackName)
|
||||
.catch(err => console.error('[SchedulerService] Notification dispatch failed:', getErrorMessage(err, 'unknown error')));
|
||||
}
|
||||
|
||||
private async tick(): Promise<void> {
|
||||
if (this.isProcessing) {
|
||||
console.warn('[SchedulerService] Tick skipped: previous tick still processing');
|
||||
@@ -280,13 +291,19 @@ export class SchedulerService {
|
||||
});
|
||||
console.log(`[SchedulerService] Task "${task.name}" (id=${task.id}) completed successfully`);
|
||||
if (task.action === 'scan') {
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
scanFailedCount > 0 ? 'warning' : 'info',
|
||||
const scanLevel: 'info' | 'warning' = scanFailedCount > 0 ? 'warning' : 'info';
|
||||
if (isDebugEnabled()) {
|
||||
console.log(
|
||||
`[SchedulerService:debug] Dispatching scan completion notification (level=${scanLevel}, stackContext=${task.target_id ?? 'none'})`,
|
||||
);
|
||||
}
|
||||
this.safeDispatch(
|
||||
scanLevel,
|
||||
`Scheduled scan "${task.name}" completed: ${output}`,
|
||||
task.target_id ?? undefined
|
||||
);
|
||||
} else if (task.last_status === 'failure') {
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
`Scheduled task "${task.name}" (${task.action}) recovered successfully`,
|
||||
task.target_id ?? undefined
|
||||
@@ -321,7 +338,7 @@ export class SchedulerService {
|
||||
error: errMsg,
|
||||
});
|
||||
console.error(`[SchedulerService] Task "${task.name}" (id=${task.id}) failed:`, errMsg);
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
this.safeDispatch(
|
||||
'error',
|
||||
`Scheduled task "${task.name}" (${task.action}) failed: ${errMsg}`,
|
||||
task.target_id ?? undefined
|
||||
@@ -598,7 +615,7 @@ export class SchedulerService {
|
||||
await compose.updateStack(stackName, undefined, true);
|
||||
db.clearStackUpdateStatus(nodeId, stackName);
|
||||
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
this.safeDispatch(
|
||||
'info',
|
||||
`Auto-update: stack "${stackName}" updated with new images`,
|
||||
stackName
|
||||
@@ -618,11 +635,55 @@ export class SchedulerService {
|
||||
console.log(`[SchedulerService:debug] Scan task ${task.id}: no node_id specified, using default node ${nodeId}`);
|
||||
}
|
||||
|
||||
const scanStart = Date.now();
|
||||
if (isDebugEnabled()) console.log(`[SchedulerService:debug] executeScan start: task=${task.id} node=${nodeId}`);
|
||||
|
||||
const summary = await trivy.scanAllNodeImages(nodeId, 'scheduled');
|
||||
|
||||
const parts: string[] = [`Scanned ${summary.scanned} image(s)`];
|
||||
if (summary.skipped > 0) parts.push(`${summary.skipped} skipped (cached)`);
|
||||
if (summary.failed > 0) parts.push(`${summary.failed} failed`);
|
||||
return { output: parts.join('; '), failed: summary.failed };
|
||||
if (isDebugEnabled()) {
|
||||
console.log(
|
||||
`[SchedulerService:debug] executeScan summary: scanned=${summary.scanned} skipped=${summary.skipped} failed=${summary.failed} ` +
|
||||
`critical=${summary.severity.critical} high=${summary.severity.high} medium=${summary.severity.medium} ` +
|
||||
`low=${summary.severity.low} unknown=${summary.severity.unknown} durationMs=${Date.now() - scanStart}`,
|
||||
);
|
||||
}
|
||||
|
||||
const output = formatScanOutput(summary);
|
||||
return { output, failed: summary.failed };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the human-readable completion message from a bulk scan summary.
|
||||
* Exported for unit tests.
|
||||
*/
|
||||
export function formatScanOutput(summary: ScanAllNodeImagesResult): string {
|
||||
const { scanned, skipped, failed, severity } = summary;
|
||||
|
||||
let header: string;
|
||||
if (scanned === 0 && skipped === 0 && failed === 0) {
|
||||
header = 'No images to scan';
|
||||
} else if (scanned === 0 && skipped > 0 && failed === 0) {
|
||||
header = `All ${skipped} image(s) already scanned recently (cache hit)`;
|
||||
} else {
|
||||
const parts: string[] = [`Scanned ${scanned} image(s)`];
|
||||
if (skipped > 0) parts.push(`${skipped} skipped (cached)`);
|
||||
if (failed > 0) parts.push(`${failed} failed`);
|
||||
header = parts.join('; ');
|
||||
}
|
||||
|
||||
const severityTiers: Array<[string, number]> = [
|
||||
['critical', severity.critical],
|
||||
['high', severity.high],
|
||||
['medium', severity.medium],
|
||||
];
|
||||
const nonZero = severityTiers.filter(([, n]) => n > 0);
|
||||
if (nonZero.length === 0) {
|
||||
if (scanned === 0 && skipped === 0 && failed === 0) {
|
||||
return header + '.';
|
||||
}
|
||||
return `${header}. No critical, high, or medium findings.`;
|
||||
}
|
||||
const findings = nonZero.map(([label, n]) => `${n} ${label}`).join(', ');
|
||||
return `${header}. Found ${findings}.`;
|
||||
}
|
||||
|
||||
@@ -77,6 +77,21 @@ interface TrivyRawOutput {
|
||||
Results?: TrivyRawResult[];
|
||||
}
|
||||
|
||||
export interface ScanAllNodeImagesSeverityTotals {
|
||||
critical: number;
|
||||
high: number;
|
||||
medium: number;
|
||||
low: number;
|
||||
unknown: number;
|
||||
}
|
||||
|
||||
export interface ScanAllNodeImagesResult {
|
||||
scanned: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
severity: ScanAllNodeImagesSeverityTotals;
|
||||
}
|
||||
|
||||
export interface TrivyVulnerability {
|
||||
vulnerabilityId: string;
|
||||
pkgName: string;
|
||||
@@ -874,7 +889,7 @@ class TrivyService {
|
||||
async scanAllNodeImages(
|
||||
nodeId: number,
|
||||
triggeredBy: VulnScanTrigger = 'scheduled',
|
||||
): Promise<{ scanned: number; skipped: number; failed: number }> {
|
||||
): Promise<ScanAllNodeImagesResult> {
|
||||
if (this.source === 'none') {
|
||||
throw new Error('Trivy is not available on this host');
|
||||
}
|
||||
@@ -889,26 +904,43 @@ class TrivyService {
|
||||
let scanned = 0;
|
||||
let skipped = 0;
|
||||
let failed = 0;
|
||||
const severity = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 };
|
||||
const countedDigests = new Set<string>();
|
||||
|
||||
const addSeverity = (row: VulnerabilityScan | null): void => {
|
||||
if (!row) return;
|
||||
severity.critical += row.critical_count;
|
||||
severity.high += row.high_count;
|
||||
severity.medium += row.medium_count;
|
||||
severity.low += row.low_count;
|
||||
severity.unknown += row.unknown_count;
|
||||
};
|
||||
|
||||
for (const ref of imageRefs) {
|
||||
try {
|
||||
const digest = await this.getImageDigest(ref, nodeId);
|
||||
if (digest) {
|
||||
if (countedDigests.has(digest)) continue;
|
||||
const cached =
|
||||
DatabaseService.getInstance().getLatestScanByDigest(digest, 'vuln');
|
||||
if (cached && Date.now() - cached.scanned_at < DIGEST_CACHE_TTL_MS) {
|
||||
skipped++;
|
||||
addSeverity(cached);
|
||||
countedDigests.add(digest);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
await this.runScanAndPersist(ref, nodeId, triggeredBy, null);
|
||||
const fresh = await this.runScanAndPersist(ref, nodeId, triggeredBy, null);
|
||||
addSeverity(fresh);
|
||||
scanned++;
|
||||
if (digest) countedDigests.add(digest);
|
||||
} catch (err) {
|
||||
failed++;
|
||||
console.warn(`[Trivy] Failed to scan ${ref}:`, getErrorMessage(err, 'unknown error'));
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
}
|
||||
return { scanned, skipped, failed };
|
||||
return { scanned, skipped, failed, severity };
|
||||
}
|
||||
|
||||
async generateSBOM(imageRef: string, format: SbomFormat): Promise<string> {
|
||||
|
||||
Reference in New Issue
Block a user