fix(scheduler): reject 6-field cron in Scheduled Operations (#1435)

* fix(scheduler): reject 6-field cron in Scheduled Operations

Create and edit validation parsed cron with cron-parser, which accepts both
5- and 6-field expressions, while the form, presets, and docs all describe a
5-field cron. Because the scheduler ticks once per minute, a leading seconds
field can never improve precision, so a 6-field expression was silently
accepted but never honored on its stated schedule.

Add a field-count guard on both sides: the API rejects 6-field input at
create and edit with a clear message, and the form surfaces the same error
inline and disables save. Cron nicknames such as @daily still pass. Document
the five-field requirement in the cron reference.

* chore: merge main into scheduled cron validation

* fix: avoid logging policy bypass actor in debug output
This commit is contained in:
Anso
2026-06-24 23:00:56 -04:00
committed by GitHub
parent bc8c051962
commit db8bb70b7d
10 changed files with 715 additions and 472 deletions
@@ -192,6 +192,38 @@ describe('POST /api/scheduled-tasks', () => {
expect(res.body.error).toMatch(/Invalid cron expression/);
});
it('rejects a 6-field cron expression with the seconds field', async () => {
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
...basePayload, cron_expression: '30 0 3 * * *',
});
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/5 fields/);
});
it('rejects a missing cron expression with a clear message', async () => {
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
...basePayload, cron_expression: undefined,
});
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/required/);
});
it('rejects an empty cron expression with a clear message', async () => {
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
...basePayload, cron_expression: '',
});
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/required/);
});
it('accepts a cron nickname such as @daily', async () => {
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
...basePayload, cron_expression: '@daily',
});
expect(res.status).toBe(201);
expect(res.body.cron_expression).toBe('@daily');
});
it('rejects unsupported actions', async () => {
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
...basePayload, action: 'nuke',
@@ -697,6 +729,60 @@ describe('PUT /api/scheduled-tasks/:id - stack target validation', () => {
});
});
describe('PUT /api/scheduled-tasks/:id - cron validation', () => {
let taskId: number;
beforeEach(() => {
const now = Date.now();
taskId = DatabaseService.getInstance().createScheduledTask({
name: 't', target_type: 'stack', target_id: 's', node_id: 1, action: 'update',
cron_expression: '0 3 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now,
last_run_at: null, next_run_at: null, last_status: null, last_error: null,
prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 0,
});
});
it('rejects a 6-field cron expression', async () => {
const res = await request(app)
.put(`/api/scheduled-tasks/${taskId}`)
.set('Cookie', adminCookie)
.send({ cron_expression: '30 0 3 * * *' });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/5 fields/);
});
it('accepts a valid 5-field cron expression', async () => {
const res = await request(app)
.put(`/api/scheduled-tasks/${taskId}`)
.set('Cookie', adminCookie)
.send({ cron_expression: '0 4 * * *' });
expect(res.status).toBe(200);
expect(res.body.cron_expression).toBe('0 4 * * *');
});
it('rejects a whitespace-only cron expression', async () => {
const res = await request(app)
.put(`/api/scheduled-tasks/${taskId}`)
.set('Cookie', adminCookie)
.send({ cron_expression: ' ' });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/required/);
});
it('rejects an empty cron expression', async () => {
const res = await request(app)
.put(`/api/scheduled-tasks/${taskId}`)
.set('Cookie', adminCookie)
.send({ cron_expression: '' });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/required/);
});
});
describe('scheduled-tasks state-invalidate broadcast', () => {
// Two frontend hooks (useConfigurationStatus, useNextAutoUpdateRun) refetch
// when this scope fires; locking it down prevents a silent UX regression
@@ -595,6 +595,36 @@ describe('SchedulerService - executePrune', () => {
expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging');
});
it('marks scheduled prune runs as failed when a target prune fails', async () => {
mockGetScheduledTask.mockReturnValue({
id: 74,
name: 'prune-fails',
action: 'prune',
cron_expression: '0 3 * * *',
enabled: true,
node_id: 1,
prune_targets: JSON.stringify(['images']),
created_by: 'admin',
last_status: null,
});
mockPruneSystem.mockRejectedValueOnce(new Error('docker prune failed'));
const svc = SchedulerService.getInstance();
await svc.triggerTask(74);
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
expect.any(Number),
expect.objectContaining({
status: 'failure',
error: expect.stringContaining('images: failed (docker prune failed)'),
}),
);
expect(mockUpdateScheduledTask).toHaveBeenCalledWith(
74,
expect.objectContaining({ last_status: 'failure' }),
);
});
it('fails scheduled prune tasks that target remote nodes before pruning', async () => {
mockGetScheduledTask.mockReturnValue({
id: 73,
+41 -18
View File
@@ -43,8 +43,7 @@ function parsePositiveNodeId(nodeId: unknown): number | null {
return Number.isInteger(parsedNodeId) && parsedNodeId > 0 ? parsedNodeId : null;
}
function actionRequiresNode(action: BackendScheduledAction, targetType: TargetType): boolean {
if (targetType === 'stack') return true;
function actionRequiresNode(action: BackendScheduledAction): boolean {
return getScheduledActionDefinition(action)?.requiresNode === true;
}
@@ -141,6 +140,29 @@ function validateOptionalFields(
return null;
}
/**
* Validate a cron expression for Scheduled Operations. The scheduler ticks once
* per minute, so an expression with a leading seconds field (6 or more fields)
* is rejected: its sub-minute precision could never be honored. Cron nicknames
* such as `@daily` (a single token) are left untouched. Returns an error message
* or null.
*/
function validateCronExpression(cron: unknown): string | null {
if (typeof cron !== 'string' || !cron.trim()) {
return 'Cron expression is required.';
}
if (cron.trim().split(/\s+/).length >= 6) {
return 'Cron expression must use 5 fields (minute hour day month weekday). The seconds field is not supported.';
}
try {
CronExpressionParser.parse(cron);
} catch (e) {
console.warn('[Scheduler] Invalid cron expression rejected:', sanitizeForLog(cron), sanitizeForLog(getErrorMessage(e, 'unknown')));
return 'Invalid cron expression.';
}
return null;
}
export const scheduledTasksRouter = Router();
scheduledTasksRouter.get('/', (req: Request, res: Response): void => {
@@ -202,16 +224,14 @@ scheduledTasksRouter.post('/', (req: Request, res: Response): void => {
const optionalErr = validateOptionalFields(action, target_type, prune_targets, target_services, prune_label_filter);
if (optionalErr) { res.status(400).json({ error: optionalErr }); return; }
try { CronExpressionParser.parse(cron_expression); } catch (e) {
console.warn('[Scheduler] Invalid cron expression rejected:', sanitizeForLog(cron_expression), sanitizeForLog(getErrorMessage(e, 'unknown')));
res.status(400).json({ error: 'Invalid cron expression.' }); return;
}
const cronErr = validateCronExpression(cron_expression);
if (cronErr) { res.status(400).json({ error: cronErr }); return; }
const scheduler = SchedulerService.getInstance();
const now = Date.now();
const nextRun = (enabled !== false) ? scheduler.calculateNextRun(cron_expression) : null;
const normalizedTargetId = target_type === 'stack' ? target_id : null;
const normalizedNodeId = actionRequiresNode(action, target_type) ? parsePositiveNodeId(node_id) : null;
const normalizedNodeId = actionRequiresNode(action) ? parsePositiveNodeId(node_id) : null;
const id = DatabaseService.getInstance().createScheduledTask({
name: name.trim(),
@@ -282,7 +302,7 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => {
const finalTargetId = finalTargetType === 'stack'
? (target_id !== undefined ? target_id : existing.target_id)
: null;
const finalNodeId = actionRequiresNode(finalAction, finalTargetType)
const finalNodeId = actionRequiresNode(finalAction)
? (node_id !== undefined ? node_id : existing.node_id)
: null;
const targetErr = validateActionTarget(finalAction, finalTargetType);
@@ -297,22 +317,25 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => {
const optionalErr = validateOptionalFields(finalAction, finalTargetType, prune_targets, target_services, prune_label_filter);
if (optionalErr) { res.status(400).json({ error: optionalErr }); return; }
if (cron_expression) {
try { CronExpressionParser.parse(cron_expression); } catch (e) {
console.warn('[Scheduler] Invalid cron expression rejected:', sanitizeForLog(cron_expression), sanitizeForLog(getErrorMessage(e, 'unknown')));
res.status(400).json({ error: 'Invalid cron expression.' }); return;
}
if (cron_expression !== undefined) {
const cronErr = validateCronExpression(cron_expression);
if (cronErr) { res.status(400).json({ error: cronErr }); return; }
}
const updates: Record<string, unknown> = { updated_at: Date.now() };
if (name !== undefined) updates.name = typeof name === 'string' ? name.trim() : name;
const updates: Partial<Omit<ScheduledTask, 'id'>> = { updated_at: Date.now() };
if (name !== undefined) {
if (typeof name !== 'string' || !name.trim()) {
res.status(400).json({ error: 'Name is required' }); return;
}
updates.name = name.trim();
}
if (target_type !== undefined) updates.target_type = finalTargetType;
if (target_id !== undefined || finalTargetType !== 'stack') updates.target_id = finalTargetId || null;
if (node_id !== undefined || !actionRequiresNode(finalAction, finalTargetType)) {
if (node_id !== undefined || !actionRequiresNode(finalAction)) {
updates.node_id = finalNodeId != null ? parsePositiveNodeId(finalNodeId) : null;
}
if (action !== undefined) updates.action = finalAction;
if (cron_expression !== undefined) updates.cron_expression = cron_expression;
if (cron_expression !== undefined && typeof cron_expression === 'string') updates.cron_expression = cron_expression;
if (enabled !== undefined) updates.enabled = enabled ? 1 : 0;
if (prune_targets !== undefined) {
updates.prune_targets = finalAction === 'prune' && prune_targets ? JSON.stringify(prune_targets) : null;
@@ -341,7 +364,7 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => {
updates.next_run_at = null;
}
db.updateScheduledTask(id, updates as Partial<Omit<ScheduledTask, 'id'>>);
db.updateScheduledTask(id, updates);
console.log(`[ScheduledTasks] Updated task id=${id}`);
const task = db.getScheduledTask(id);
broadcastScheduledTasksChanged();
+460 -442
View File
@@ -1,442 +1,460 @@
/**
* Pre-deploy policy gate.
*
* Extracted from `index.ts` so route handlers and the scheduler can call a
* single, unit-testable function rather than copy-paste the gate logic.
*
* The gate fails open when Trivy is missing (users are never locked out by
* tooling state) and fails closed when the compose file cannot be parsed
* (a broken stack must not silently bypass a block policy).
*/
import { ComposeService } from './ComposeService';
import { DatabaseService } from './DatabaseService';
import type { ScanPolicy, VulnSeverity, VulnerabilityScan, VulnerabilityDetail } from './DatabaseService';
import { FleetSyncService } from './FleetSyncService';
import { NotificationService } from './NotificationService';
import { sanitizeForLog } from '../utils/safeLog';
import TrivyService from './TrivyService';
import { isSeverityAtLeast } from '../utils/severity';
import { applySuppressions } from '../utils/suppression-filter';
import { validateImageRef } from '../utils/image-ref';
import { getErrorMessage } from '../utils/errors';
import { isDebugEnabled } from '../utils/debug';
import {
evaluatePolicyRisk,
describePolicyInputs,
policyInputs,
type PolicyBlockReason,
type PolicyRiskInputs,
} from '../utils/policy-risk';
export interface PolicyViolation {
imageRef: string;
severity: VulnSeverity;
criticalCount: number;
highCount: number;
/** Non-suppressed CVEs in the CISA known-exploited (KEV) set on this image. */
kevCount: number;
/** Non-suppressed Critical/High findings with a fix available on this image. */
fixableCount: number;
/** Which policy inputs matched (empty when the image could not be scanned). */
reasons: PolicyBlockReason[];
scanId: number;
}
export interface PolicyEnforcementOptions {
bypass: boolean;
actor: string;
/**
* Paid-tier deploy enforcement switch. Community keeps policies as
* evaluation-only and must not block compose starts.
*/
blockingEnabled?: boolean;
ip?: string;
/** HTTP method of the originating request; used for audit attribution. */
auditMethod?: string;
/** Request path of the originating route; used for audit attribution. */
auditPath?: string;
}
export interface PolicyEnforcementResult {
ok: boolean;
bypassed: boolean;
policy?: ScanPolicy;
violations: PolicyViolation[];
trivyMissing?: boolean;
}
const TRIVY_MISSING_NOTIFY_COOLDOWN_MS = 60 * 60 * 1000;
// Growth bounded by configured-policy fanout (only stacks with an enabled
// block_on_deploy policy can land here), not by total stack churn. Cleared
// on process restart, which is the right scope for an informational warning.
const trivyMissingNotifiedAt = new Map<string, number>();
function notifyTrivyMissingOnce(nodeId: number, stackName: string): void {
const key = `${nodeId}:${stackName}`;
const now = Date.now();
const last = trivyMissingNotifiedAt.get(key);
if (last !== undefined && now - last < TRIVY_MISSING_NOTIFY_COOLDOWN_MS) return;
trivyMissingNotifiedAt.set(key, now);
NotificationService.getInstance().dispatchAlert(
'warning',
'scan_finding',
`Pre-deploy scan for "${stackName}" skipped: Trivy not installed on this node`,
{ stackName, actor: 'system:policy' },
);
}
export function _resetTrivyMissingNotificationStateForTests(): void {
trivyMissingNotifiedAt.clear();
}
type PreflightScan = Pick<VulnerabilityScan, 'id' | 'highest_severity' | 'critical_count' | 'high_count' | 'total_vulnerabilities'>;
interface ImageRiskEvaluation {
/** Policy inputs that matched for this image. */
reasons: PolicyBlockReason[];
/** Highest non-suppressed severity; UNKNOWN means no severity remains. */
severity: VulnSeverity;
criticalCount: number;
highCount: number;
kevCount: number;
fixableCount: number;
/** CVE IDs suppressed for this image; only populated when honoring suppressions. */
suppressedCves: string[];
/** True when the same inputs would have matched if suppressions were ignored. */
rawWouldBlock: boolean;
}
interface SuppressionPass {
imageRef: string;
cves: string[];
}
/**
* Aggregate-only evaluation. Two roles: the cheap fast path for a severity-only
* policy that does not need detail rows (`failClosed=false`), and the fallback
* when the detail rows cannot be trusted. Severity stays verifiable from the
* stored aggregate counts, so it always gates. KEV/fixability cannot be read
* from aggregates: when `failClosed` is set the active KEV/fixable inputs block
* anyway, because their absence cannot be proven without the details.
*/
function aggregateFallback(
inputs: PolicyRiskInputs,
rawSeverity: VulnSeverity,
scan: PreflightScan,
failClosed: boolean,
): ImageRiskEvaluation {
const reasons: PolicyBlockReason[] = [];
if (inputs.blockOnSeverity && isSeverityAtLeast(rawSeverity, inputs.maxSeverity)) reasons.push('severity');
if (failClosed && inputs.blockOnKev) reasons.push('kev');
if (failClosed && inputs.blockOnFixable) reasons.push('fixable');
return {
reasons,
severity: rawSeverity,
criticalCount: scan.critical_count,
highCount: scan.high_count,
kevCount: 0,
fixableCount: 0,
suppressedCves: [],
rawWouldBlock: reasons.length > 0,
};
}
/**
* Resolve which of a policy's risk inputs (severity, KEV, fixability) an image
* matches. A severity-only policy that is not honoring suppressions keeps the
* cheap aggregate path (historical behavior). Any KEV/fixable input, or honoring
* suppressions, requires the per-finding detail rows: KEV/fixability cannot be
* read from the aggregate counts, so the details are loaded regardless of the
* honor flag. KEV/fixable are evaluated over the non-suppressed set only.
*/
function evaluateImageRisk(
scan: PreflightScan,
imageRef: string,
policy: ScanPolicy,
honorSuppressions: boolean,
): ImageRiskEvaluation {
const inputs = policyInputs(policy);
const rawSeverity = scan.highest_severity ?? 'UNKNOWN';
const needsDetails = inputs.blockOnKev || inputs.blockOnFixable || honorSuppressions;
if (!needsDetails) {
return aggregateFallback(inputs, rawSeverity, scan, false);
}
const db = DatabaseService.getInstance();
let findings: VulnerabilityDetail[];
let suppressions;
try {
findings = db.getAllVulnerabilityDetails(scan.id);
suppressions = db.getCveSuppressions();
} catch (err) {
// Detail read failed: severity still gates from the aggregate, but
// KEV/fixability are unknowable. Fail closed on them, consistent with the
// truncated-details path below: a policy that explicitly opted into a
// KEV/fixable gate must not silently degrade to "allow" on a transient
// read error. The admin bypass path stays available.
console.error('[Policy] Detail read failed for %s; gating severity on aggregate, failing closed on KEV/fixable:', sanitizeForLog(imageRef), sanitizeForLog(getErrorMessage(err, 'db read failed')));
return aggregateFallback(inputs, rawSeverity, scan, true);
}
// The stored detail rows must reproduce the scan's full finding set before
// KEV/fixability can be trusted. A cache-hit preflight scan keeps the
// complete aggregate counts but copies only the first N detail rows. When the
// counts disagree, gate severity on the raw aggregate and fail closed on any
// KEV/fixable input: absence of a known-exploited or fixable finding cannot be
// proven from a truncated set, so the unverifiable finding is treated as risky.
if (findings.length !== scan.total_vulnerabilities) {
if (scan.total_vulnerabilities > 0) {
console.warn(
'[Policy] Scan %d detail rows (%d) do not match its total (%d); gating severity on raw scan, failing closed on KEV/fixable',
scan.id, findings.length, scan.total_vulnerabilities,
);
}
return aggregateFallback(inputs, rawSeverity, scan, true);
}
// KEV membership is the same for the full set and the non-suppressed subset,
// so resolve intel once over every CVE and reuse it for both the effective
// decision and the suppression-pass check below.
const intel = inputs.blockOnKev ? db.getCveIntel(findings.map((f) => f.vulnerability_id)) : null;
const isKev = (cveId: string): boolean => intel?.get(cveId)?.kev === true;
const suppressedCves = new Set<string>();
let evalSet: VulnerabilityDetail[];
if (honorSuppressions) {
evalSet = [];
for (const f of applySuppressions(findings, imageRef, suppressions)) {
if (f.suppressed) { suppressedCves.add(f.vulnerability_id); continue; }
evalSet.push(f);
}
} else {
evalSet = findings;
}
const outcome = evaluatePolicyRisk(evalSet, isKev, inputs);
// When honoring suppressions, "would have blocked on the raw set" detects a
// pass that only succeeded because an accepted CVE was filtered out.
const rawWouldBlock = honorSuppressions
? evaluatePolicyRisk(findings, isKev, inputs).reasons.length > 0
: outcome.reasons.length > 0;
return {
reasons: outcome.reasons,
severity: outcome.highestSeverity,
criticalCount: outcome.criticalCount,
highCount: outcome.highCount,
kevCount: outcome.kevCount,
fixableCount: outcome.fixableCount,
suppressedCves: [...suppressedCves],
rawWouldBlock,
};
}
/**
* A deploy that would have been blocked on raw severity but proceeded because
* suppressions dropped every image below the threshold is a security-relevant
* event: record it so the suppression-driven pass is traceable in the audit log.
*/
function recordSuppressionPassAudit(
stackName: string,
nodeId: number,
policy: ScanPolicy,
passes: SuppressionPass[],
opts: PolicyEnforcementOptions,
): void {
const cves = [...new Set(passes.flatMap((p) => p.cves))];
try {
DatabaseService.getInstance().insertAuditLog({
timestamp: Date.now(),
username: opts.actor,
method: opts.auditMethod ?? 'POST',
path: opts.auditPath ?? `/api/stacks/${stackName}/deploy`,
status_code: 200,
node_id: nodeId,
ip_address: opts.ip ?? '',
summary: `policy.suppression_pass stack="${stackName}" policy="${policy.name}" images=[${passes.map((p) => p.imageRef).join(',')}] cves=[${cves.join(',')}]`,
});
} catch (err) {
console.error('[Policy] Failed to record suppression-pass audit entry:', err);
}
console.warn(
'[Policy] Deploy for "%s" allowed by suppressions: %d image(s) would have matched %s (policy "%s")',
sanitizeForLog(stackName), passes.length, describePolicyInputs(policyInputs(policy)), sanitizeForLog(policy.name),
);
}
export async function enforcePolicyPreDeploy(
stackName: string,
nodeId: number,
opts: PolicyEnforcementOptions,
): Promise<PolicyEnforcementResult> {
const db = DatabaseService.getInstance();
const policy = db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
if (!policy || !policy.enabled || !policy.block_on_deploy) {
return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
}
if (opts.blockingEnabled === false) {
return { ok: true, bypassed: false, policy, violations: [] };
}
const svc = TrivyService.getInstance();
if (!svc.isTrivyAvailable()) {
notifyTrivyMissingOnce(nodeId, stackName);
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
}
let imageRefs: string[] = [];
try {
imageRefs = await ComposeService.getInstance(nodeId).listStackImages(stackName);
} catch (err) {
const message = getErrorMessage(err, 'compose parse failed');
console.error('[Policy] listStackImages failed for %s:', sanitizeForLog(stackName), sanitizeForLog(message));
return {
ok: false,
bypassed: false,
policy,
violations: [{
imageRef: '(compose parse error)',
severity: 'UNKNOWN',
criticalCount: 0,
highCount: 0,
kevCount: 0,
fixableCount: 0,
reasons: [],
scanId: 0,
}],
};
}
return enforcePolicyForImageRefs(stackName, nodeId, imageRefs, opts, policy);
}
export async function enforcePolicyForImageRefs(
stackName: string,
nodeId: number,
imageRefs: string[],
opts: PolicyEnforcementOptions,
matchedPolicy?: ScanPolicy,
failClosedInvalidRefs = false,
): Promise<PolicyEnforcementResult> {
const db = DatabaseService.getInstance();
const policy = matchedPolicy ?? db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
if (!policy || !policy.enabled || !policy.block_on_deploy) {
return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
}
const svc = TrivyService.getInstance();
if (!svc.isTrivyAvailable()) {
notifyTrivyMissingOnce(nodeId, stackName);
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
}
const honorSuppressions = db.getGlobalSettings()['deploy_block_honor_suppressions'] === '1';
const debug = isDebugEnabled();
if (debug) {
console.log(
'[Policy:debug] Evaluating "%s" against policy "%s" (inputs=%s, images=%d, honorSuppressions=%s)',
sanitizeForLog(stackName), sanitizeForLog(policy.name), describePolicyInputs(policyInputs(policy)), imageRefs.length, honorSuppressions,
);
}
const violations: PolicyViolation[] = [];
const suppressionPasses: SuppressionPass[] = [];
for (const imageRef of imageRefs) {
if (!validateImageRef(imageRef)) {
if (failClosedInvalidRefs) {
violations.push({
imageRef,
severity: 'UNKNOWN',
criticalCount: 0,
highCount: 0,
kevCount: 0,
fixableCount: 0,
reasons: [],
scanId: 0,
});
}
continue;
}
try {
const scan = await svc.scanImagePreflight(imageRef, nodeId, stackName);
const evaluated = evaluateImageRisk(scan, imageRef, policy, honorSuppressions);
if (debug) {
console.log(
'[Policy:debug] %s scanned: severity=%s kev=%d fixable=%d matched=[%s]',
sanitizeForLog(imageRef), evaluated.severity, evaluated.kevCount, evaluated.fixableCount, evaluated.reasons.join(','),
);
}
if (evaluated.reasons.length > 0) {
violations.push({
imageRef,
severity: evaluated.severity,
criticalCount: evaluated.criticalCount,
highCount: evaluated.highCount,
kevCount: evaluated.kevCount,
fixableCount: evaluated.fixableCount,
reasons: evaluated.reasons,
scanId: scan.id,
});
} else if (
honorSuppressions &&
evaluated.suppressedCves.length > 0 &&
evaluated.rawWouldBlock
) {
suppressionPasses.push({ imageRef, cves: evaluated.suppressedCves });
}
} catch (err) {
const message = getErrorMessage(err, 'pre-flight scan failed');
console.error(`[Policy] scanImagePreflight failed for ${imageRef}:`, message);
violations.push({
imageRef,
severity: 'UNKNOWN',
criticalCount: 0,
highCount: 0,
kevCount: 0,
fixableCount: 0,
reasons: [],
scanId: 0,
});
}
}
if (violations.length === 0) {
if (suppressionPasses.length > 0) {
recordSuppressionPassAudit(stackName, nodeId, policy, suppressionPasses, opts);
}
return { ok: true, bypassed: false, policy, violations: [] };
}
if (opts.bypass) {
try {
db.insertAuditLog({
timestamp: Date.now(),
username: opts.actor,
method: opts.auditMethod ?? 'POST',
path: opts.auditPath ?? `/api/stacks/${stackName}/deploy`,
status_code: 200,
node_id: nodeId,
ip_address: opts.ip ?? '',
summary: `policy.bypass stack="${stackName}" policy="${policy.name}" violations=${violations.length} images=[${violations.map((v) => v.imageRef).join(',')}]`,
});
} catch (err) {
console.error('[Policy] Failed to record bypass audit entry:', err);
}
if (debug) {
console.log(
'[Policy:debug] Bypass by "%s" for "%s" (%d violation(s))',
sanitizeForLog(opts.actor), sanitizeForLog(stackName), violations.length,
);
}
return { ok: true, bypassed: true, policy, violations };
}
console.warn(
'[Policy] Blocked deploy for "%s": %d image(s) matched %s (policy "%s")',
sanitizeForLog(stackName), violations.length, describePolicyInputs(policyInputs(policy)), sanitizeForLog(policy.name),
);
return { ok: false, bypassed: false, policy, violations };
}
/**
* Pre-deploy policy gate.
*
* Extracted from `index.ts` so route handlers and the scheduler can call a
* single, unit-testable function rather than copy-paste the gate logic.
*
* The gate fails open when Trivy is missing (users are never locked out by
* tooling state) and fails closed when the compose file cannot be parsed
* (a broken stack must not silently bypass a block policy).
*/
import { ComposeService } from './ComposeService';
import { DatabaseService } from './DatabaseService';
import type { ScanPolicy, VulnSeverity, VulnerabilityScan, VulnerabilityDetail } from './DatabaseService';
import { FleetSyncService } from './FleetSyncService';
import { NotificationService } from './NotificationService';
import { sanitizeForLog } from '../utils/safeLog';
import TrivyService from './TrivyService';
import { isSeverityAtLeast } from '../utils/severity';
import { applySuppressions } from '../utils/suppression-filter';
import { validateImageRef } from '../utils/image-ref';
import { getErrorMessage } from '../utils/errors';
import { isDebugEnabled } from '../utils/debug';
import {
evaluatePolicyRisk,
describePolicyInputs,
policyInputs,
type PolicyBlockReason,
type PolicyRiskInputs,
} from '../utils/policy-risk';
export interface PolicyViolation {
imageRef: string;
severity: VulnSeverity;
criticalCount: number;
highCount: number;
/** Non-suppressed CVEs in the CISA known-exploited (KEV) set on this image. */
kevCount: number;
/** Non-suppressed Critical/High findings with a fix available on this image. */
fixableCount: number;
/** Which policy inputs matched (empty when the image could not be scanned). */
reasons: PolicyBlockReason[];
scanId: number;
}
export interface PolicyEnforcementOptions {
bypass: boolean;
actor: string;
/**
* Paid-tier deploy enforcement switch. Community keeps policies as
* evaluation-only and must not block compose starts.
*/
blockingEnabled?: boolean;
ip?: string;
/** HTTP method of the originating request; used for audit attribution. */
auditMethod?: string;
/** Request path of the originating route; used for audit attribution. */
auditPath?: string;
}
export interface PolicyEnforcementResult {
ok: boolean;
bypassed: boolean;
policy?: ScanPolicy;
violations: PolicyViolation[];
trivyMissing?: boolean;
}
const TRIVY_MISSING_NOTIFY_COOLDOWN_MS = 60 * 60 * 1000;
// Growth bounded by configured-policy fanout (only stacks with an enabled
// block_on_deploy policy can land here), not by total stack churn. Cleared
// on process restart, which is the right scope for an informational warning.
const trivyMissingNotifiedAt = new Map<string, number>();
function notifyTrivyMissingOnce(nodeId: number, stackName: string): void {
const key = `${nodeId}:${stackName}`;
const now = Date.now();
const last = trivyMissingNotifiedAt.get(key);
if (last !== undefined && now - last < TRIVY_MISSING_NOTIFY_COOLDOWN_MS) return;
trivyMissingNotifiedAt.set(key, now);
NotificationService.getInstance().dispatchAlert(
'warning',
'scan_finding',
`Pre-deploy scan for "${stackName}" skipped: Trivy not installed on this node`,
{ stackName, actor: 'system:policy' },
);
}
export function _resetTrivyMissingNotificationStateForTests(): void {
trivyMissingNotifiedAt.clear();
}
type PreflightScan = Pick<VulnerabilityScan, 'id' | 'highest_severity' | 'critical_count' | 'high_count' | 'total_vulnerabilities'>;
interface ImageRiskEvaluation {
/** Policy inputs that matched for this image. */
reasons: PolicyBlockReason[];
/** Highest non-suppressed severity; UNKNOWN means no severity remains. */
severity: VulnSeverity;
criticalCount: number;
highCount: number;
kevCount: number;
fixableCount: number;
/** CVE IDs suppressed for this image; only populated when honoring suppressions. */
suppressedCves: string[];
/** True when the same inputs would have matched if suppressions were ignored. */
rawWouldBlock: boolean;
}
interface SuppressionPass {
imageRef: string;
cves: string[];
}
/**
* Aggregate-only evaluation. Two roles: the cheap fast path for a severity-only
* policy that does not need detail rows (`failClosed=false`), and the fallback
* when the detail rows cannot be trusted. Severity stays verifiable from the
* stored aggregate counts, so it always gates. KEV/fixability cannot be read
* from aggregates: when `failClosed` is set the active KEV/fixable inputs block
* anyway, because their absence cannot be proven without the details.
*/
function aggregateFallback(
inputs: PolicyRiskInputs,
rawSeverity: VulnSeverity,
scan: PreflightScan,
failClosed: boolean,
): ImageRiskEvaluation {
const reasons: PolicyBlockReason[] = [];
if (inputs.blockOnSeverity && isSeverityAtLeast(rawSeverity, inputs.maxSeverity)) reasons.push('severity');
if (failClosed && inputs.blockOnKev) reasons.push('kev');
if (failClosed && inputs.blockOnFixable) reasons.push('fixable');
return {
reasons,
severity: rawSeverity,
criticalCount: scan.critical_count,
highCount: scan.high_count,
kevCount: 0,
fixableCount: 0,
suppressedCves: [],
rawWouldBlock: reasons.length > 0,
};
}
/**
* Resolve which of a policy's risk inputs (severity, KEV, fixability) an image
* matches. A severity-only policy that is not honoring suppressions keeps the
* cheap aggregate path (historical behavior). Any KEV/fixable input, or honoring
* suppressions, requires the per-finding detail rows: KEV/fixability cannot be
* read from the aggregate counts, so the details are loaded regardless of the
* honor flag. KEV/fixable are evaluated over the non-suppressed set only.
*/
function evaluateImageRisk(
scan: PreflightScan,
imageRef: string,
policy: ScanPolicy,
honorSuppressions: boolean,
): ImageRiskEvaluation {
const inputs = policyInputs(policy);
const rawSeverity = scan.highest_severity ?? 'UNKNOWN';
const needsDetails = inputs.blockOnKev || inputs.blockOnFixable || honorSuppressions;
if (!needsDetails) {
return aggregateFallback(inputs, rawSeverity, scan, false);
}
const db = DatabaseService.getInstance();
let findings: VulnerabilityDetail[];
let suppressions;
try {
findings = db.getAllVulnerabilityDetails(scan.id);
suppressions = db.getCveSuppressions();
} catch (err) {
// Detail read failed: severity still gates from the aggregate, but
// KEV/fixability are unknowable. Fail closed on them, consistent with the
// truncated-details path below: a policy that explicitly opted into a
// KEV/fixable gate must not silently degrade to "allow" on a transient
// read error. The admin bypass path stays available.
console.error('[Policy] Detail read failed for %s; gating severity on aggregate, failing closed on KEV/fixable:', sanitizeForLog(imageRef), sanitizeForLog(getErrorMessage(err, 'db read failed')));
return aggregateFallback(inputs, rawSeverity, scan, true);
}
// The stored detail rows must reproduce the scan's full finding set before
// KEV/fixability can be trusted. A cache-hit preflight scan keeps the
// complete aggregate counts but copies only the first N detail rows. When the
// counts disagree, gate severity on the raw aggregate and fail closed on any
// KEV/fixable input: absence of a known-exploited or fixable finding cannot be
// proven from a truncated set, so the unverifiable finding is treated as risky.
if (findings.length !== scan.total_vulnerabilities) {
if (scan.total_vulnerabilities > 0) {
console.warn(
'[Policy] Scan %d detail rows (%d) do not match its total (%d); gating severity on raw scan, failing closed on KEV/fixable',
scan.id, findings.length, scan.total_vulnerabilities,
);
}
return aggregateFallback(inputs, rawSeverity, scan, true);
}
// KEV membership is the same for the full set and the non-suppressed subset,
// so resolve intel once over every CVE and reuse it for both the effective
// decision and the suppression-pass check below.
const intel = inputs.blockOnKev ? db.getCveIntel(findings.map((f) => f.vulnerability_id)) : null;
const isKev = (cveId: string): boolean => intel?.get(cveId)?.kev === true;
const suppressedCves = new Set<string>();
let evalSet: VulnerabilityDetail[];
if (honorSuppressions) {
evalSet = [];
for (const f of applySuppressions(findings, imageRef, suppressions)) {
if (f.suppressed) { suppressedCves.add(f.vulnerability_id); continue; }
evalSet.push(f);
}
} else {
evalSet = findings;
}
const outcome = evaluatePolicyRisk(evalSet, isKev, inputs);
// When honoring suppressions, "would have blocked on the raw set" detects a
// pass that only succeeded because an accepted CVE was filtered out.
const rawWouldBlock = honorSuppressions
? evaluatePolicyRisk(findings, isKev, inputs).reasons.length > 0
: outcome.reasons.length > 0;
return {
reasons: outcome.reasons,
severity: outcome.highestSeverity,
criticalCount: outcome.criticalCount,
highCount: outcome.highCount,
kevCount: outcome.kevCount,
fixableCount: outcome.fixableCount,
suppressedCves: [...suppressedCves],
rawWouldBlock,
};
}
/**
* A deploy that would have been blocked on raw severity but proceeded because
* suppressions dropped every image below the threshold is a security-relevant
* event: record it so the suppression-driven pass is traceable in the audit log.
*/
function recordSuppressionPassAudit(
stackName: string,
nodeId: number,
policy: ScanPolicy,
passes: SuppressionPass[],
opts: PolicyEnforcementOptions,
): void {
const cves = [...new Set(passes.flatMap((p) => p.cves))];
try {
DatabaseService.getInstance().insertAuditLog({
timestamp: Date.now(),
username: opts.actor,
method: opts.auditMethod ?? 'POST',
path: opts.auditPath ?? `/api/stacks/${stackName}/deploy`,
status_code: 200,
node_id: nodeId,
ip_address: opts.ip ?? '',
summary: `policy.suppression_pass stack="${stackName}" policy="${policy.name}" images=[${passes.map((p) => p.imageRef).join(',')}] cves=[${cves.join(',')}]`,
});
} catch (err) {
console.error('[Policy] Failed to record suppression-pass audit entry:', err);
}
console.warn(
'[Policy] Deploy for "%s" allowed by suppressions: %d image(s) would have matched %s (policy "%s")',
sanitizeForLog(stackName), passes.length, describePolicyInputs(policyInputs(policy)), sanitizeForLog(policy.name),
);
}
export async function enforcePolicyPreDeploy(
stackName: string,
nodeId: number,
opts: PolicyEnforcementOptions,
): Promise<PolicyEnforcementResult> {
const db = DatabaseService.getInstance();
const policy = db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
if (!policy || !policy.enabled || !policy.block_on_deploy) {
return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
}
if (opts.blockingEnabled === false) {
return { ok: true, bypassed: false, policy, violations: [] };
}
const svc = TrivyService.getInstance();
if (!svc.isTrivyAvailable()) {
notifyTrivyMissingOnce(nodeId, stackName);
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
}
let imageRefs: string[] = [];
try {
imageRefs = await ComposeService.getInstance(nodeId).listStackImages(stackName);
} catch (err) {
const message = getErrorMessage(err, 'compose parse failed');
console.error('[Policy] listStackImages failed for %s:', sanitizeForLog(stackName), sanitizeForLog(message));
return {
ok: false,
bypassed: false,
policy,
violations: [{
imageRef: '(compose parse error)',
severity: 'UNKNOWN',
criticalCount: 0,
highCount: 0,
kevCount: 0,
fixableCount: 0,
reasons: [],
scanId: 0,
}],
};
}
return enforcePolicyForImageRefs(stackName, nodeId, imageRefs, opts, policy);
}
export async function enforcePolicyForImageRefs(
stackName: string,
nodeId: number,
imageRefs: string[],
opts: PolicyEnforcementOptions,
matchedPolicy?: ScanPolicy,
failClosedInvalidRefs = false,
): Promise<PolicyEnforcementResult> {
const db = DatabaseService.getInstance();
const policy = matchedPolicy ?? db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
if (!policy || !policy.enabled || !policy.block_on_deploy) {
return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
}
const svc = TrivyService.getInstance();
if (!svc.isTrivyAvailable()) {
notifyTrivyMissingOnce(nodeId, stackName);
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
}
const honorSuppressions = db.getGlobalSettings()['deploy_block_honor_suppressions'] === '1';
const debug = isDebugEnabled();
if (debug) {
console.log(
'[Policy:debug] Evaluating "%s" against policy "%s" (inputs=%s, images=%d, honorSuppressions=%s)',
sanitizeForLog(stackName), sanitizeForLog(policy.name), describePolicyInputs(policyInputs(policy)), imageRefs.length, honorSuppressions,
);
}
const violations: PolicyViolation[] = [];
const suppressionPasses: SuppressionPass[] = [];
for (const imageRef of imageRefs) {
if (!validateImageRef(imageRef)) {
if (failClosedInvalidRefs) {
violations.push({
imageRef,
severity: 'UNKNOWN',
criticalCount: 0,
highCount: 0,
kevCount: 0,
fixableCount: 0,
reasons: [],
scanId: 0,
});
}
continue;
}
let scan: VulnerabilityScan;
try {
scan = await svc.scanImagePreflight(imageRef, nodeId, stackName);
} catch (err) {
const message = getErrorMessage(err, 'pre-flight scan failed');
console.error(`[Policy] scanImagePreflight failed for ${imageRef}:`, message);
violations.push({
imageRef,
severity: 'UNKNOWN',
criticalCount: 0,
highCount: 0,
kevCount: 0,
fixableCount: 0,
reasons: [],
scanId: 0,
});
continue;
}
try {
const evaluated = evaluateImageRisk(scan, imageRef, policy, honorSuppressions);
if (debug) {
console.log(
'[Policy:debug] %s scanned: severity=%s kev=%d fixable=%d matched=[%s]',
sanitizeForLog(imageRef), evaluated.severity, evaluated.kevCount, evaluated.fixableCount, evaluated.reasons.join(','),
);
}
if (evaluated.reasons.length > 0) {
violations.push({
imageRef,
severity: evaluated.severity,
criticalCount: evaluated.criticalCount,
highCount: evaluated.highCount,
kevCount: evaluated.kevCount,
fixableCount: evaluated.fixableCount,
reasons: evaluated.reasons,
scanId: scan.id,
});
} else if (
honorSuppressions &&
evaluated.suppressedCves.length > 0 &&
evaluated.rawWouldBlock
) {
suppressionPasses.push({ imageRef, cves: evaluated.suppressedCves });
}
} catch (err) {
const message = getErrorMessage(err, 'policy evaluation failed');
console.error(`[Policy] policy evaluation failed for ${imageRef}:`, message);
violations.push({
imageRef,
severity: 'UNKNOWN',
criticalCount: 0,
highCount: 0,
kevCount: 0,
fixableCount: 0,
reasons: [],
scanId: scan.id,
});
}
}
if (violations.length === 0) {
if (suppressionPasses.length > 0) {
recordSuppressionPassAudit(stackName, nodeId, policy, suppressionPasses, opts);
}
return { ok: true, bypassed: false, policy, violations: [] };
}
if (opts.bypass) {
try {
db.insertAuditLog({
timestamp: Date.now(),
username: opts.actor,
method: opts.auditMethod ?? 'POST',
path: opts.auditPath ?? `/api/stacks/${stackName}/deploy`,
status_code: 200,
node_id: nodeId,
ip_address: opts.ip ?? '',
summary: `policy.bypass stack="${stackName}" policy="${policy.name}" violations=${violations.length} images=[${violations.map((v) => v.imageRef).join(',')}]`,
});
} catch (err) {
console.error('[Policy] Failed to record bypass audit entry:', err);
}
if (debug) {
console.log(
'[Policy:debug] Bypass for "%s" (%d violation(s))',
sanitizeForLog(stackName), violations.length,
);
}
return { ok: true, bypassed: true, policy, violations };
}
console.warn(
'[Policy] Blocked deploy for "%s": %d image(s) matched %s (policy "%s")',
sanitizeForLog(stackName), violations.length, describePolicyInputs(policyInputs(policy)), sanitizeForLog(policy.name),
);
return { ok: false, bypassed: false, policy, violations };
}
+11 -1
View File
@@ -332,6 +332,10 @@ export class SchedulerService {
case 'auto_start':
output = await this.executeAutoStart(task);
break;
default: {
const unhandledAction: never = task.action;
throw new Error(`Unsupported scheduled action: ${unhandledAction}`);
}
}
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} action completed in ${Date.now() - actionStart}ms`);
@@ -673,6 +677,7 @@ export class SchedulerService {
: [...allTargets];
const labelFilter = task.prune_label_filter || undefined;
const results: string[] = [];
const failures: string[] = [];
for (const target of targets) {
try {
@@ -680,11 +685,16 @@ export class SchedulerService {
results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
} catch (error: unknown) {
const msg = error instanceof Error ? error.message : String(error);
results.push(`${target}: failed (${msg})`);
const failure = `${target}: failed (${msg})`;
results.push(failure);
failures.push(failure);
}
}
const filterSuffix = labelFilter ? ` (label: ${labelFilter})` : '';
if (failures.length > 0) {
throw new Error(`System prune failed${filterSuffix}: ${results.join('; ')}`);
}
return `System prune completed${filterSuffix}: ${results.join('; ')}`;
}