mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
fix(scheduler): reject 6-field cron in Scheduled Operations (#1435)
* fix(scheduler): reject 6-field cron in Scheduled Operations Create and edit validation parsed cron with cron-parser, which accepts both 5- and 6-field expressions, while the form, presets, and docs all describe a 5-field cron. Because the scheduler ticks once per minute, a leading seconds field can never improve precision, so a 6-field expression was silently accepted but never honored on its stated schedule. Add a field-count guard on both sides: the API rejects 6-field input at create and edit with a clear message, and the form surfaces the same error inline and disables save. Cron nicknames such as @daily still pass. Document the five-field requirement in the cron reference. * chore: merge main into scheduled cron validation * fix: avoid logging policy bypass actor in debug output
This commit is contained in:
Anso
@@ -192,6 +192,38 @@ describe('POST /api/scheduled-tasks', () => {
|
||||
expect(res.body.error).toMatch(/Invalid cron expression/);
|
||||
});
|
||||
|
||||
it('rejects a 6-field cron expression with the seconds field', async () => {
|
||||
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
|
||||
...basePayload, cron_expression: '30 0 3 * * *',
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/5 fields/);
|
||||
});
|
||||
|
||||
it('rejects a missing cron expression with a clear message', async () => {
|
||||
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
|
||||
...basePayload, cron_expression: undefined,
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/required/);
|
||||
});
|
||||
|
||||
it('rejects an empty cron expression with a clear message', async () => {
|
||||
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
|
||||
...basePayload, cron_expression: '',
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/required/);
|
||||
});
|
||||
|
||||
it('accepts a cron nickname such as @daily', async () => {
|
||||
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
|
||||
...basePayload, cron_expression: '@daily',
|
||||
});
|
||||
expect(res.status).toBe(201);
|
||||
expect(res.body.cron_expression).toBe('@daily');
|
||||
});
|
||||
|
||||
it('rejects unsupported actions', async () => {
|
||||
const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({
|
||||
...basePayload, action: 'nuke',
|
||||
@@ -697,6 +729,60 @@ describe('PUT /api/scheduled-tasks/:id - stack target validation', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('PUT /api/scheduled-tasks/:id - cron validation', () => {
|
||||
let taskId: number;
|
||||
|
||||
beforeEach(() => {
|
||||
const now = Date.now();
|
||||
taskId = DatabaseService.getInstance().createScheduledTask({
|
||||
name: 't', target_type: 'stack', target_id: 's', node_id: 1, action: 'update',
|
||||
cron_expression: '0 3 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now,
|
||||
last_run_at: null, next_run_at: null, last_status: null, last_error: null,
|
||||
prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects a 6-field cron expression', async () => {
|
||||
const res = await request(app)
|
||||
.put(`/api/scheduled-tasks/${taskId}`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ cron_expression: '30 0 3 * * *' });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/5 fields/);
|
||||
});
|
||||
|
||||
it('accepts a valid 5-field cron expression', async () => {
|
||||
const res = await request(app)
|
||||
.put(`/api/scheduled-tasks/${taskId}`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ cron_expression: '0 4 * * *' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.cron_expression).toBe('0 4 * * *');
|
||||
});
|
||||
|
||||
it('rejects a whitespace-only cron expression', async () => {
|
||||
const res = await request(app)
|
||||
.put(`/api/scheduled-tasks/${taskId}`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ cron_expression: ' ' });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/required/);
|
||||
});
|
||||
|
||||
it('rejects an empty cron expression', async () => {
|
||||
const res = await request(app)
|
||||
.put(`/api/scheduled-tasks/${taskId}`)
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ cron_expression: '' });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/required/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scheduled-tasks state-invalidate broadcast', () => {
|
||||
// Two frontend hooks (useConfigurationStatus, useNextAutoUpdateRun) refetch
|
||||
// when this scope fires; locking it down prevents a silent UX regression
|
||||
|
||||
@@ -595,6 +595,36 @@ describe('SchedulerService - executePrune', () => {
|
||||
expect(mockPruneSystem).toHaveBeenCalledWith('containers', 'env=staging');
|
||||
});
|
||||
|
||||
it('marks scheduled prune runs as failed when a target prune fails', async () => {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 74,
|
||||
name: 'prune-fails',
|
||||
action: 'prune',
|
||||
cron_expression: '0 3 * * *',
|
||||
enabled: true,
|
||||
node_id: 1,
|
||||
prune_targets: JSON.stringify(['images']),
|
||||
created_by: 'admin',
|
||||
last_status: null,
|
||||
});
|
||||
mockPruneSystem.mockRejectedValueOnce(new Error('docker prune failed'));
|
||||
|
||||
const svc = SchedulerService.getInstance();
|
||||
await svc.triggerTask(74);
|
||||
|
||||
expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
|
||||
expect.any(Number),
|
||||
expect.objectContaining({
|
||||
status: 'failure',
|
||||
error: expect.stringContaining('images: failed (docker prune failed)'),
|
||||
}),
|
||||
);
|
||||
expect(mockUpdateScheduledTask).toHaveBeenCalledWith(
|
||||
74,
|
||||
expect.objectContaining({ last_status: 'failure' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('fails scheduled prune tasks that target remote nodes before pruning', async () => {
|
||||
mockGetScheduledTask.mockReturnValue({
|
||||
id: 73,
|
||||
|
||||
@@ -43,8 +43,7 @@ function parsePositiveNodeId(nodeId: unknown): number | null {
|
||||
return Number.isInteger(parsedNodeId) && parsedNodeId > 0 ? parsedNodeId : null;
|
||||
}
|
||||
|
||||
function actionRequiresNode(action: BackendScheduledAction, targetType: TargetType): boolean {
|
||||
if (targetType === 'stack') return true;
|
||||
function actionRequiresNode(action: BackendScheduledAction): boolean {
|
||||
return getScheduledActionDefinition(action)?.requiresNode === true;
|
||||
}
|
||||
|
||||
@@ -141,6 +140,29 @@ function validateOptionalFields(
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a cron expression for Scheduled Operations. The scheduler ticks once
|
||||
* per minute, so an expression with a leading seconds field (6 or more fields)
|
||||
* is rejected: its sub-minute precision could never be honored. Cron nicknames
|
||||
* such as `@daily` (a single token) are left untouched. Returns an error message
|
||||
* or null.
|
||||
*/
|
||||
function validateCronExpression(cron: unknown): string | null {
|
||||
if (typeof cron !== 'string' || !cron.trim()) {
|
||||
return 'Cron expression is required.';
|
||||
}
|
||||
if (cron.trim().split(/\s+/).length >= 6) {
|
||||
return 'Cron expression must use 5 fields (minute hour day month weekday). The seconds field is not supported.';
|
||||
}
|
||||
try {
|
||||
CronExpressionParser.parse(cron);
|
||||
} catch (e) {
|
||||
console.warn('[Scheduler] Invalid cron expression rejected:', sanitizeForLog(cron), sanitizeForLog(getErrorMessage(e, 'unknown')));
|
||||
return 'Invalid cron expression.';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export const scheduledTasksRouter = Router();
|
||||
|
||||
scheduledTasksRouter.get('/', (req: Request, res: Response): void => {
|
||||
@@ -202,16 +224,14 @@ scheduledTasksRouter.post('/', (req: Request, res: Response): void => {
|
||||
const optionalErr = validateOptionalFields(action, target_type, prune_targets, target_services, prune_label_filter);
|
||||
if (optionalErr) { res.status(400).json({ error: optionalErr }); return; }
|
||||
|
||||
try { CronExpressionParser.parse(cron_expression); } catch (e) {
|
||||
console.warn('[Scheduler] Invalid cron expression rejected:', sanitizeForLog(cron_expression), sanitizeForLog(getErrorMessage(e, 'unknown')));
|
||||
res.status(400).json({ error: 'Invalid cron expression.' }); return;
|
||||
}
|
||||
const cronErr = validateCronExpression(cron_expression);
|
||||
if (cronErr) { res.status(400).json({ error: cronErr }); return; }
|
||||
|
||||
const scheduler = SchedulerService.getInstance();
|
||||
const now = Date.now();
|
||||
const nextRun = (enabled !== false) ? scheduler.calculateNextRun(cron_expression) : null;
|
||||
const normalizedTargetId = target_type === 'stack' ? target_id : null;
|
||||
const normalizedNodeId = actionRequiresNode(action, target_type) ? parsePositiveNodeId(node_id) : null;
|
||||
const normalizedNodeId = actionRequiresNode(action) ? parsePositiveNodeId(node_id) : null;
|
||||
|
||||
const id = DatabaseService.getInstance().createScheduledTask({
|
||||
name: name.trim(),
|
||||
@@ -282,7 +302,7 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => {
|
||||
const finalTargetId = finalTargetType === 'stack'
|
||||
? (target_id !== undefined ? target_id : existing.target_id)
|
||||
: null;
|
||||
const finalNodeId = actionRequiresNode(finalAction, finalTargetType)
|
||||
const finalNodeId = actionRequiresNode(finalAction)
|
||||
? (node_id !== undefined ? node_id : existing.node_id)
|
||||
: null;
|
||||
const targetErr = validateActionTarget(finalAction, finalTargetType);
|
||||
@@ -297,22 +317,25 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => {
|
||||
const optionalErr = validateOptionalFields(finalAction, finalTargetType, prune_targets, target_services, prune_label_filter);
|
||||
if (optionalErr) { res.status(400).json({ error: optionalErr }); return; }
|
||||
|
||||
if (cron_expression) {
|
||||
try { CronExpressionParser.parse(cron_expression); } catch (e) {
|
||||
console.warn('[Scheduler] Invalid cron expression rejected:', sanitizeForLog(cron_expression), sanitizeForLog(getErrorMessage(e, 'unknown')));
|
||||
res.status(400).json({ error: 'Invalid cron expression.' }); return;
|
||||
}
|
||||
if (cron_expression !== undefined) {
|
||||
const cronErr = validateCronExpression(cron_expression);
|
||||
if (cronErr) { res.status(400).json({ error: cronErr }); return; }
|
||||
}
|
||||
|
||||
const updates: Record<string, unknown> = { updated_at: Date.now() };
|
||||
if (name !== undefined) updates.name = typeof name === 'string' ? name.trim() : name;
|
||||
const updates: Partial<Omit<ScheduledTask, 'id'>> = { updated_at: Date.now() };
|
||||
if (name !== undefined) {
|
||||
if (typeof name !== 'string' || !name.trim()) {
|
||||
res.status(400).json({ error: 'Name is required' }); return;
|
||||
}
|
||||
updates.name = name.trim();
|
||||
}
|
||||
if (target_type !== undefined) updates.target_type = finalTargetType;
|
||||
if (target_id !== undefined || finalTargetType !== 'stack') updates.target_id = finalTargetId || null;
|
||||
if (node_id !== undefined || !actionRequiresNode(finalAction, finalTargetType)) {
|
||||
if (node_id !== undefined || !actionRequiresNode(finalAction)) {
|
||||
updates.node_id = finalNodeId != null ? parsePositiveNodeId(finalNodeId) : null;
|
||||
}
|
||||
if (action !== undefined) updates.action = finalAction;
|
||||
if (cron_expression !== undefined) updates.cron_expression = cron_expression;
|
||||
if (cron_expression !== undefined && typeof cron_expression === 'string') updates.cron_expression = cron_expression;
|
||||
if (enabled !== undefined) updates.enabled = enabled ? 1 : 0;
|
||||
if (prune_targets !== undefined) {
|
||||
updates.prune_targets = finalAction === 'prune' && prune_targets ? JSON.stringify(prune_targets) : null;
|
||||
@@ -341,7 +364,7 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => {
|
||||
updates.next_run_at = null;
|
||||
}
|
||||
|
||||
db.updateScheduledTask(id, updates as Partial<Omit<ScheduledTask, 'id'>>);
|
||||
db.updateScheduledTask(id, updates);
|
||||
console.log(`[ScheduledTasks] Updated task id=${id}`);
|
||||
const task = db.getScheduledTask(id);
|
||||
broadcastScheduledTasksChanged();
|
||||
|
||||
@@ -1,442 +1,460 @@
|
||||
/**
|
||||
* Pre-deploy policy gate.
|
||||
*
|
||||
* Extracted from `index.ts` so route handlers and the scheduler can call a
|
||||
* single, unit-testable function rather than copy-paste the gate logic.
|
||||
*
|
||||
* The gate fails open when Trivy is missing (users are never locked out by
|
||||
* tooling state) and fails closed when the compose file cannot be parsed
|
||||
* (a broken stack must not silently bypass a block policy).
|
||||
*/
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import type { ScanPolicy, VulnSeverity, VulnerabilityScan, VulnerabilityDetail } from './DatabaseService';
|
||||
import { FleetSyncService } from './FleetSyncService';
|
||||
import { NotificationService } from './NotificationService';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import TrivyService from './TrivyService';
|
||||
import { isSeverityAtLeast } from '../utils/severity';
|
||||
import { applySuppressions } from '../utils/suppression-filter';
|
||||
import { validateImageRef } from '../utils/image-ref';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import {
|
||||
evaluatePolicyRisk,
|
||||
describePolicyInputs,
|
||||
policyInputs,
|
||||
type PolicyBlockReason,
|
||||
type PolicyRiskInputs,
|
||||
} from '../utils/policy-risk';
|
||||
|
||||
export interface PolicyViolation {
|
||||
imageRef: string;
|
||||
severity: VulnSeverity;
|
||||
criticalCount: number;
|
||||
highCount: number;
|
||||
/** Non-suppressed CVEs in the CISA known-exploited (KEV) set on this image. */
|
||||
kevCount: number;
|
||||
/** Non-suppressed Critical/High findings with a fix available on this image. */
|
||||
fixableCount: number;
|
||||
/** Which policy inputs matched (empty when the image could not be scanned). */
|
||||
reasons: PolicyBlockReason[];
|
||||
scanId: number;
|
||||
}
|
||||
|
||||
export interface PolicyEnforcementOptions {
|
||||
bypass: boolean;
|
||||
actor: string;
|
||||
/**
|
||||
* Paid-tier deploy enforcement switch. Community keeps policies as
|
||||
* evaluation-only and must not block compose starts.
|
||||
*/
|
||||
blockingEnabled?: boolean;
|
||||
ip?: string;
|
||||
/** HTTP method of the originating request; used for audit attribution. */
|
||||
auditMethod?: string;
|
||||
/** Request path of the originating route; used for audit attribution. */
|
||||
auditPath?: string;
|
||||
}
|
||||
|
||||
export interface PolicyEnforcementResult {
|
||||
ok: boolean;
|
||||
bypassed: boolean;
|
||||
policy?: ScanPolicy;
|
||||
violations: PolicyViolation[];
|
||||
trivyMissing?: boolean;
|
||||
}
|
||||
|
||||
const TRIVY_MISSING_NOTIFY_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
// Growth bounded by configured-policy fanout (only stacks with an enabled
|
||||
// block_on_deploy policy can land here), not by total stack churn. Cleared
|
||||
// on process restart, which is the right scope for an informational warning.
|
||||
const trivyMissingNotifiedAt = new Map<string, number>();
|
||||
|
||||
function notifyTrivyMissingOnce(nodeId: number, stackName: string): void {
|
||||
const key = `${nodeId}:${stackName}`;
|
||||
const now = Date.now();
|
||||
const last = trivyMissingNotifiedAt.get(key);
|
||||
if (last !== undefined && now - last < TRIVY_MISSING_NOTIFY_COOLDOWN_MS) return;
|
||||
trivyMissingNotifiedAt.set(key, now);
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'scan_finding',
|
||||
`Pre-deploy scan for "${stackName}" skipped: Trivy not installed on this node`,
|
||||
{ stackName, actor: 'system:policy' },
|
||||
);
|
||||
}
|
||||
|
||||
export function _resetTrivyMissingNotificationStateForTests(): void {
|
||||
trivyMissingNotifiedAt.clear();
|
||||
}
|
||||
|
||||
type PreflightScan = Pick<VulnerabilityScan, 'id' | 'highest_severity' | 'critical_count' | 'high_count' | 'total_vulnerabilities'>;
|
||||
|
||||
interface ImageRiskEvaluation {
|
||||
/** Policy inputs that matched for this image. */
|
||||
reasons: PolicyBlockReason[];
|
||||
/** Highest non-suppressed severity; UNKNOWN means no severity remains. */
|
||||
severity: VulnSeverity;
|
||||
criticalCount: number;
|
||||
highCount: number;
|
||||
kevCount: number;
|
||||
fixableCount: number;
|
||||
/** CVE IDs suppressed for this image; only populated when honoring suppressions. */
|
||||
suppressedCves: string[];
|
||||
/** True when the same inputs would have matched if suppressions were ignored. */
|
||||
rawWouldBlock: boolean;
|
||||
}
|
||||
|
||||
interface SuppressionPass {
|
||||
imageRef: string;
|
||||
cves: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate-only evaluation. Two roles: the cheap fast path for a severity-only
|
||||
* policy that does not need detail rows (`failClosed=false`), and the fallback
|
||||
* when the detail rows cannot be trusted. Severity stays verifiable from the
|
||||
* stored aggregate counts, so it always gates. KEV/fixability cannot be read
|
||||
* from aggregates: when `failClosed` is set the active KEV/fixable inputs block
|
||||
* anyway, because their absence cannot be proven without the details.
|
||||
*/
|
||||
function aggregateFallback(
|
||||
inputs: PolicyRiskInputs,
|
||||
rawSeverity: VulnSeverity,
|
||||
scan: PreflightScan,
|
||||
failClosed: boolean,
|
||||
): ImageRiskEvaluation {
|
||||
const reasons: PolicyBlockReason[] = [];
|
||||
if (inputs.blockOnSeverity && isSeverityAtLeast(rawSeverity, inputs.maxSeverity)) reasons.push('severity');
|
||||
if (failClosed && inputs.blockOnKev) reasons.push('kev');
|
||||
if (failClosed && inputs.blockOnFixable) reasons.push('fixable');
|
||||
return {
|
||||
reasons,
|
||||
severity: rawSeverity,
|
||||
criticalCount: scan.critical_count,
|
||||
highCount: scan.high_count,
|
||||
kevCount: 0,
|
||||
fixableCount: 0,
|
||||
suppressedCves: [],
|
||||
rawWouldBlock: reasons.length > 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve which of a policy's risk inputs (severity, KEV, fixability) an image
|
||||
* matches. A severity-only policy that is not honoring suppressions keeps the
|
||||
* cheap aggregate path (historical behavior). Any KEV/fixable input, or honoring
|
||||
* suppressions, requires the per-finding detail rows: KEV/fixability cannot be
|
||||
* read from the aggregate counts, so the details are loaded regardless of the
|
||||
* honor flag. KEV/fixable are evaluated over the non-suppressed set only.
|
||||
*/
|
||||
function evaluateImageRisk(
|
||||
scan: PreflightScan,
|
||||
imageRef: string,
|
||||
policy: ScanPolicy,
|
||||
honorSuppressions: boolean,
|
||||
): ImageRiskEvaluation {
|
||||
const inputs = policyInputs(policy);
|
||||
const rawSeverity = scan.highest_severity ?? 'UNKNOWN';
|
||||
const needsDetails = inputs.blockOnKev || inputs.blockOnFixable || honorSuppressions;
|
||||
if (!needsDetails) {
|
||||
return aggregateFallback(inputs, rawSeverity, scan, false);
|
||||
}
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
let findings: VulnerabilityDetail[];
|
||||
let suppressions;
|
||||
try {
|
||||
findings = db.getAllVulnerabilityDetails(scan.id);
|
||||
suppressions = db.getCveSuppressions();
|
||||
} catch (err) {
|
||||
// Detail read failed: severity still gates from the aggregate, but
|
||||
// KEV/fixability are unknowable. Fail closed on them, consistent with the
|
||||
// truncated-details path below: a policy that explicitly opted into a
|
||||
// KEV/fixable gate must not silently degrade to "allow" on a transient
|
||||
// read error. The admin bypass path stays available.
|
||||
console.error('[Policy] Detail read failed for %s; gating severity on aggregate, failing closed on KEV/fixable:', sanitizeForLog(imageRef), sanitizeForLog(getErrorMessage(err, 'db read failed')));
|
||||
return aggregateFallback(inputs, rawSeverity, scan, true);
|
||||
}
|
||||
|
||||
// The stored detail rows must reproduce the scan's full finding set before
|
||||
// KEV/fixability can be trusted. A cache-hit preflight scan keeps the
|
||||
// complete aggregate counts but copies only the first N detail rows. When the
|
||||
// counts disagree, gate severity on the raw aggregate and fail closed on any
|
||||
// KEV/fixable input: absence of a known-exploited or fixable finding cannot be
|
||||
// proven from a truncated set, so the unverifiable finding is treated as risky.
|
||||
if (findings.length !== scan.total_vulnerabilities) {
|
||||
if (scan.total_vulnerabilities > 0) {
|
||||
console.warn(
|
||||
'[Policy] Scan %d detail rows (%d) do not match its total (%d); gating severity on raw scan, failing closed on KEV/fixable',
|
||||
scan.id, findings.length, scan.total_vulnerabilities,
|
||||
);
|
||||
}
|
||||
return aggregateFallback(inputs, rawSeverity, scan, true);
|
||||
}
|
||||
|
||||
// KEV membership is the same for the full set and the non-suppressed subset,
|
||||
// so resolve intel once over every CVE and reuse it for both the effective
|
||||
// decision and the suppression-pass check below.
|
||||
const intel = inputs.blockOnKev ? db.getCveIntel(findings.map((f) => f.vulnerability_id)) : null;
|
||||
const isKev = (cveId: string): boolean => intel?.get(cveId)?.kev === true;
|
||||
|
||||
const suppressedCves = new Set<string>();
|
||||
let evalSet: VulnerabilityDetail[];
|
||||
if (honorSuppressions) {
|
||||
evalSet = [];
|
||||
for (const f of applySuppressions(findings, imageRef, suppressions)) {
|
||||
if (f.suppressed) { suppressedCves.add(f.vulnerability_id); continue; }
|
||||
evalSet.push(f);
|
||||
}
|
||||
} else {
|
||||
evalSet = findings;
|
||||
}
|
||||
|
||||
const outcome = evaluatePolicyRisk(evalSet, isKev, inputs);
|
||||
// When honoring suppressions, "would have blocked on the raw set" detects a
|
||||
// pass that only succeeded because an accepted CVE was filtered out.
|
||||
const rawWouldBlock = honorSuppressions
|
||||
? evaluatePolicyRisk(findings, isKev, inputs).reasons.length > 0
|
||||
: outcome.reasons.length > 0;
|
||||
return {
|
||||
reasons: outcome.reasons,
|
||||
severity: outcome.highestSeverity,
|
||||
criticalCount: outcome.criticalCount,
|
||||
highCount: outcome.highCount,
|
||||
kevCount: outcome.kevCount,
|
||||
fixableCount: outcome.fixableCount,
|
||||
suppressedCves: [...suppressedCves],
|
||||
rawWouldBlock,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A deploy that would have been blocked on raw severity but proceeded because
|
||||
* suppressions dropped every image below the threshold is a security-relevant
|
||||
* event: record it so the suppression-driven pass is traceable in the audit log.
|
||||
*/
|
||||
function recordSuppressionPassAudit(
|
||||
stackName: string,
|
||||
nodeId: number,
|
||||
policy: ScanPolicy,
|
||||
passes: SuppressionPass[],
|
||||
opts: PolicyEnforcementOptions,
|
||||
): void {
|
||||
const cves = [...new Set(passes.flatMap((p) => p.cves))];
|
||||
try {
|
||||
DatabaseService.getInstance().insertAuditLog({
|
||||
timestamp: Date.now(),
|
||||
username: opts.actor,
|
||||
method: opts.auditMethod ?? 'POST',
|
||||
path: opts.auditPath ?? `/api/stacks/${stackName}/deploy`,
|
||||
status_code: 200,
|
||||
node_id: nodeId,
|
||||
ip_address: opts.ip ?? '',
|
||||
summary: `policy.suppression_pass stack="${stackName}" policy="${policy.name}" images=[${passes.map((p) => p.imageRef).join(',')}] cves=[${cves.join(',')}]`,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[Policy] Failed to record suppression-pass audit entry:', err);
|
||||
}
|
||||
console.warn(
|
||||
'[Policy] Deploy for "%s" allowed by suppressions: %d image(s) would have matched %s (policy "%s")',
|
||||
sanitizeForLog(stackName), passes.length, describePolicyInputs(policyInputs(policy)), sanitizeForLog(policy.name),
|
||||
);
|
||||
}
|
||||
|
||||
export async function enforcePolicyPreDeploy(
|
||||
stackName: string,
|
||||
nodeId: number,
|
||||
opts: PolicyEnforcementOptions,
|
||||
): Promise<PolicyEnforcementResult> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const policy = db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
|
||||
|
||||
if (!policy || !policy.enabled || !policy.block_on_deploy) {
|
||||
return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
|
||||
}
|
||||
|
||||
if (opts.blockingEnabled === false) {
|
||||
return { ok: true, bypassed: false, policy, violations: [] };
|
||||
}
|
||||
|
||||
const svc = TrivyService.getInstance();
|
||||
if (!svc.isTrivyAvailable()) {
|
||||
notifyTrivyMissingOnce(nodeId, stackName);
|
||||
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
|
||||
}
|
||||
|
||||
let imageRefs: string[] = [];
|
||||
try {
|
||||
imageRefs = await ComposeService.getInstance(nodeId).listStackImages(stackName);
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'compose parse failed');
|
||||
console.error('[Policy] listStackImages failed for %s:', sanitizeForLog(stackName), sanitizeForLog(message));
|
||||
return {
|
||||
ok: false,
|
||||
bypassed: false,
|
||||
policy,
|
||||
violations: [{
|
||||
imageRef: '(compose parse error)',
|
||||
severity: 'UNKNOWN',
|
||||
criticalCount: 0,
|
||||
highCount: 0,
|
||||
kevCount: 0,
|
||||
fixableCount: 0,
|
||||
reasons: [],
|
||||
scanId: 0,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return enforcePolicyForImageRefs(stackName, nodeId, imageRefs, opts, policy);
|
||||
}
|
||||
|
||||
export async function enforcePolicyForImageRefs(
|
||||
stackName: string,
|
||||
nodeId: number,
|
||||
imageRefs: string[],
|
||||
opts: PolicyEnforcementOptions,
|
||||
matchedPolicy?: ScanPolicy,
|
||||
failClosedInvalidRefs = false,
|
||||
): Promise<PolicyEnforcementResult> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const policy = matchedPolicy ?? db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
|
||||
|
||||
if (!policy || !policy.enabled || !policy.block_on_deploy) {
|
||||
return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
|
||||
}
|
||||
|
||||
const svc = TrivyService.getInstance();
|
||||
if (!svc.isTrivyAvailable()) {
|
||||
notifyTrivyMissingOnce(nodeId, stackName);
|
||||
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
|
||||
}
|
||||
|
||||
const honorSuppressions = db.getGlobalSettings()['deploy_block_honor_suppressions'] === '1';
|
||||
|
||||
const debug = isDebugEnabled();
|
||||
if (debug) {
|
||||
console.log(
|
||||
'[Policy:debug] Evaluating "%s" against policy "%s" (inputs=%s, images=%d, honorSuppressions=%s)',
|
||||
sanitizeForLog(stackName), sanitizeForLog(policy.name), describePolicyInputs(policyInputs(policy)), imageRefs.length, honorSuppressions,
|
||||
);
|
||||
}
|
||||
|
||||
const violations: PolicyViolation[] = [];
|
||||
const suppressionPasses: SuppressionPass[] = [];
|
||||
for (const imageRef of imageRefs) {
|
||||
if (!validateImageRef(imageRef)) {
|
||||
if (failClosedInvalidRefs) {
|
||||
violations.push({
|
||||
imageRef,
|
||||
severity: 'UNKNOWN',
|
||||
criticalCount: 0,
|
||||
highCount: 0,
|
||||
kevCount: 0,
|
||||
fixableCount: 0,
|
||||
reasons: [],
|
||||
scanId: 0,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const scan = await svc.scanImagePreflight(imageRef, nodeId, stackName);
|
||||
const evaluated = evaluateImageRisk(scan, imageRef, policy, honorSuppressions);
|
||||
if (debug) {
|
||||
console.log(
|
||||
'[Policy:debug] %s scanned: severity=%s kev=%d fixable=%d matched=[%s]',
|
||||
sanitizeForLog(imageRef), evaluated.severity, evaluated.kevCount, evaluated.fixableCount, evaluated.reasons.join(','),
|
||||
);
|
||||
}
|
||||
if (evaluated.reasons.length > 0) {
|
||||
violations.push({
|
||||
imageRef,
|
||||
severity: evaluated.severity,
|
||||
criticalCount: evaluated.criticalCount,
|
||||
highCount: evaluated.highCount,
|
||||
kevCount: evaluated.kevCount,
|
||||
fixableCount: evaluated.fixableCount,
|
||||
reasons: evaluated.reasons,
|
||||
scanId: scan.id,
|
||||
});
|
||||
} else if (
|
||||
honorSuppressions &&
|
||||
evaluated.suppressedCves.length > 0 &&
|
||||
evaluated.rawWouldBlock
|
||||
) {
|
||||
suppressionPasses.push({ imageRef, cves: evaluated.suppressedCves });
|
||||
}
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'pre-flight scan failed');
|
||||
console.error(`[Policy] scanImagePreflight failed for ${imageRef}:`, message);
|
||||
violations.push({
|
||||
imageRef,
|
||||
severity: 'UNKNOWN',
|
||||
criticalCount: 0,
|
||||
highCount: 0,
|
||||
kevCount: 0,
|
||||
fixableCount: 0,
|
||||
reasons: [],
|
||||
scanId: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (violations.length === 0) {
|
||||
if (suppressionPasses.length > 0) {
|
||||
recordSuppressionPassAudit(stackName, nodeId, policy, suppressionPasses, opts);
|
||||
}
|
||||
return { ok: true, bypassed: false, policy, violations: [] };
|
||||
}
|
||||
|
||||
if (opts.bypass) {
|
||||
try {
|
||||
db.insertAuditLog({
|
||||
timestamp: Date.now(),
|
||||
username: opts.actor,
|
||||
method: opts.auditMethod ?? 'POST',
|
||||
path: opts.auditPath ?? `/api/stacks/${stackName}/deploy`,
|
||||
status_code: 200,
|
||||
node_id: nodeId,
|
||||
ip_address: opts.ip ?? '',
|
||||
summary: `policy.bypass stack="${stackName}" policy="${policy.name}" violations=${violations.length} images=[${violations.map((v) => v.imageRef).join(',')}]`,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[Policy] Failed to record bypass audit entry:', err);
|
||||
}
|
||||
if (debug) {
|
||||
console.log(
|
||||
'[Policy:debug] Bypass by "%s" for "%s" (%d violation(s))',
|
||||
sanitizeForLog(opts.actor), sanitizeForLog(stackName), violations.length,
|
||||
);
|
||||
}
|
||||
return { ok: true, bypassed: true, policy, violations };
|
||||
}
|
||||
|
||||
console.warn(
|
||||
'[Policy] Blocked deploy for "%s": %d image(s) matched %s (policy "%s")',
|
||||
sanitizeForLog(stackName), violations.length, describePolicyInputs(policyInputs(policy)), sanitizeForLog(policy.name),
|
||||
);
|
||||
return { ok: false, bypassed: false, policy, violations };
|
||||
}
|
||||
/**
|
||||
* Pre-deploy policy gate.
|
||||
*
|
||||
* Extracted from `index.ts` so route handlers and the scheduler can call a
|
||||
* single, unit-testable function rather than copy-paste the gate logic.
|
||||
*
|
||||
* The gate fails open when Trivy is missing (users are never locked out by
|
||||
* tooling state) and fails closed when the compose file cannot be parsed
|
||||
* (a broken stack must not silently bypass a block policy).
|
||||
*/
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import type { ScanPolicy, VulnSeverity, VulnerabilityScan, VulnerabilityDetail } from './DatabaseService';
|
||||
import { FleetSyncService } from './FleetSyncService';
|
||||
import { NotificationService } from './NotificationService';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import TrivyService from './TrivyService';
|
||||
import { isSeverityAtLeast } from '../utils/severity';
|
||||
import { applySuppressions } from '../utils/suppression-filter';
|
||||
import { validateImageRef } from '../utils/image-ref';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import {
|
||||
evaluatePolicyRisk,
|
||||
describePolicyInputs,
|
||||
policyInputs,
|
||||
type PolicyBlockReason,
|
||||
type PolicyRiskInputs,
|
||||
} from '../utils/policy-risk';
|
||||
|
||||
export interface PolicyViolation {
|
||||
imageRef: string;
|
||||
severity: VulnSeverity;
|
||||
criticalCount: number;
|
||||
highCount: number;
|
||||
/** Non-suppressed CVEs in the CISA known-exploited (KEV) set on this image. */
|
||||
kevCount: number;
|
||||
/** Non-suppressed Critical/High findings with a fix available on this image. */
|
||||
fixableCount: number;
|
||||
/** Which policy inputs matched (empty when the image could not be scanned). */
|
||||
reasons: PolicyBlockReason[];
|
||||
scanId: number;
|
||||
}
|
||||
|
||||
export interface PolicyEnforcementOptions {
|
||||
bypass: boolean;
|
||||
actor: string;
|
||||
/**
|
||||
* Paid-tier deploy enforcement switch. Community keeps policies as
|
||||
* evaluation-only and must not block compose starts.
|
||||
*/
|
||||
blockingEnabled?: boolean;
|
||||
ip?: string;
|
||||
/** HTTP method of the originating request; used for audit attribution. */
|
||||
auditMethod?: string;
|
||||
/** Request path of the originating route; used for audit attribution. */
|
||||
auditPath?: string;
|
||||
}
|
||||
|
||||
export interface PolicyEnforcementResult {
|
||||
ok: boolean;
|
||||
bypassed: boolean;
|
||||
policy?: ScanPolicy;
|
||||
violations: PolicyViolation[];
|
||||
trivyMissing?: boolean;
|
||||
}
|
||||
|
||||
const TRIVY_MISSING_NOTIFY_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
// Growth bounded by configured-policy fanout (only stacks with an enabled
|
||||
// block_on_deploy policy can land here), not by total stack churn. Cleared
|
||||
// on process restart, which is the right scope for an informational warning.
|
||||
const trivyMissingNotifiedAt = new Map<string, number>();
|
||||
|
||||
function notifyTrivyMissingOnce(nodeId: number, stackName: string): void {
|
||||
const key = `${nodeId}:${stackName}`;
|
||||
const now = Date.now();
|
||||
const last = trivyMissingNotifiedAt.get(key);
|
||||
if (last !== undefined && now - last < TRIVY_MISSING_NOTIFY_COOLDOWN_MS) return;
|
||||
trivyMissingNotifiedAt.set(key, now);
|
||||
NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'scan_finding',
|
||||
`Pre-deploy scan for "${stackName}" skipped: Trivy not installed on this node`,
|
||||
{ stackName, actor: 'system:policy' },
|
||||
);
|
||||
}
|
||||
|
||||
export function _resetTrivyMissingNotificationStateForTests(): void {
|
||||
trivyMissingNotifiedAt.clear();
|
||||
}
|
||||
|
||||
type PreflightScan = Pick<VulnerabilityScan, 'id' | 'highest_severity' | 'critical_count' | 'high_count' | 'total_vulnerabilities'>;
|
||||
|
||||
interface ImageRiskEvaluation {
|
||||
/** Policy inputs that matched for this image. */
|
||||
reasons: PolicyBlockReason[];
|
||||
/** Highest non-suppressed severity; UNKNOWN means no severity remains. */
|
||||
severity: VulnSeverity;
|
||||
criticalCount: number;
|
||||
highCount: number;
|
||||
kevCount: number;
|
||||
fixableCount: number;
|
||||
/** CVE IDs suppressed for this image; only populated when honoring suppressions. */
|
||||
suppressedCves: string[];
|
||||
/** True when the same inputs would have matched if suppressions were ignored. */
|
||||
rawWouldBlock: boolean;
|
||||
}
|
||||
|
||||
interface SuppressionPass {
|
||||
imageRef: string;
|
||||
cves: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate-only evaluation. Two roles: the cheap fast path for a severity-only
|
||||
* policy that does not need detail rows (`failClosed=false`), and the fallback
|
||||
* when the detail rows cannot be trusted. Severity stays verifiable from the
|
||||
* stored aggregate counts, so it always gates. KEV/fixability cannot be read
|
||||
* from aggregates: when `failClosed` is set the active KEV/fixable inputs block
|
||||
* anyway, because their absence cannot be proven without the details.
|
||||
*/
|
||||
function aggregateFallback(
|
||||
inputs: PolicyRiskInputs,
|
||||
rawSeverity: VulnSeverity,
|
||||
scan: PreflightScan,
|
||||
failClosed: boolean,
|
||||
): ImageRiskEvaluation {
|
||||
const reasons: PolicyBlockReason[] = [];
|
||||
if (inputs.blockOnSeverity && isSeverityAtLeast(rawSeverity, inputs.maxSeverity)) reasons.push('severity');
|
||||
if (failClosed && inputs.blockOnKev) reasons.push('kev');
|
||||
if (failClosed && inputs.blockOnFixable) reasons.push('fixable');
|
||||
return {
|
||||
reasons,
|
||||
severity: rawSeverity,
|
||||
criticalCount: scan.critical_count,
|
||||
highCount: scan.high_count,
|
||||
kevCount: 0,
|
||||
fixableCount: 0,
|
||||
suppressedCves: [],
|
||||
rawWouldBlock: reasons.length > 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve which of a policy's risk inputs (severity, KEV, fixability) an image
|
||||
* matches. A severity-only policy that is not honoring suppressions keeps the
|
||||
* cheap aggregate path (historical behavior). Any KEV/fixable input, or honoring
|
||||
* suppressions, requires the per-finding detail rows: KEV/fixability cannot be
|
||||
* read from the aggregate counts, so the details are loaded regardless of the
|
||||
* honor flag. KEV/fixable are evaluated over the non-suppressed set only.
|
||||
*/
|
||||
function evaluateImageRisk(
|
||||
scan: PreflightScan,
|
||||
imageRef: string,
|
||||
policy: ScanPolicy,
|
||||
honorSuppressions: boolean,
|
||||
): ImageRiskEvaluation {
|
||||
const inputs = policyInputs(policy);
|
||||
const rawSeverity = scan.highest_severity ?? 'UNKNOWN';
|
||||
const needsDetails = inputs.blockOnKev || inputs.blockOnFixable || honorSuppressions;
|
||||
if (!needsDetails) {
|
||||
return aggregateFallback(inputs, rawSeverity, scan, false);
|
||||
}
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
let findings: VulnerabilityDetail[];
|
||||
let suppressions;
|
||||
try {
|
||||
findings = db.getAllVulnerabilityDetails(scan.id);
|
||||
suppressions = db.getCveSuppressions();
|
||||
} catch (err) {
|
||||
// Detail read failed: severity still gates from the aggregate, but
|
||||
// KEV/fixability are unknowable. Fail closed on them, consistent with the
|
||||
// truncated-details path below: a policy that explicitly opted into a
|
||||
// KEV/fixable gate must not silently degrade to "allow" on a transient
|
||||
// read error. The admin bypass path stays available.
|
||||
console.error('[Policy] Detail read failed for %s; gating severity on aggregate, failing closed on KEV/fixable:', sanitizeForLog(imageRef), sanitizeForLog(getErrorMessage(err, 'db read failed')));
|
||||
return aggregateFallback(inputs, rawSeverity, scan, true);
|
||||
}
|
||||
|
||||
// The stored detail rows must reproduce the scan's full finding set before
|
||||
// KEV/fixability can be trusted. A cache-hit preflight scan keeps the
|
||||
// complete aggregate counts but copies only the first N detail rows. When the
|
||||
// counts disagree, gate severity on the raw aggregate and fail closed on any
|
||||
// KEV/fixable input: absence of a known-exploited or fixable finding cannot be
|
||||
// proven from a truncated set, so the unverifiable finding is treated as risky.
|
||||
if (findings.length !== scan.total_vulnerabilities) {
|
||||
if (scan.total_vulnerabilities > 0) {
|
||||
console.warn(
|
||||
'[Policy] Scan %d detail rows (%d) do not match its total (%d); gating severity on raw scan, failing closed on KEV/fixable',
|
||||
scan.id, findings.length, scan.total_vulnerabilities,
|
||||
);
|
||||
}
|
||||
return aggregateFallback(inputs, rawSeverity, scan, true);
|
||||
}
|
||||
|
||||
// KEV membership is the same for the full set and the non-suppressed subset,
|
||||
// so resolve intel once over every CVE and reuse it for both the effective
|
||||
// decision and the suppression-pass check below.
|
||||
const intel = inputs.blockOnKev ? db.getCveIntel(findings.map((f) => f.vulnerability_id)) : null;
|
||||
const isKev = (cveId: string): boolean => intel?.get(cveId)?.kev === true;
|
||||
|
||||
const suppressedCves = new Set<string>();
|
||||
let evalSet: VulnerabilityDetail[];
|
||||
if (honorSuppressions) {
|
||||
evalSet = [];
|
||||
for (const f of applySuppressions(findings, imageRef, suppressions)) {
|
||||
if (f.suppressed) { suppressedCves.add(f.vulnerability_id); continue; }
|
||||
evalSet.push(f);
|
||||
}
|
||||
} else {
|
||||
evalSet = findings;
|
||||
}
|
||||
|
||||
const outcome = evaluatePolicyRisk(evalSet, isKev, inputs);
|
||||
// When honoring suppressions, "would have blocked on the raw set" detects a
|
||||
// pass that only succeeded because an accepted CVE was filtered out.
|
||||
const rawWouldBlock = honorSuppressions
|
||||
? evaluatePolicyRisk(findings, isKev, inputs).reasons.length > 0
|
||||
: outcome.reasons.length > 0;
|
||||
return {
|
||||
reasons: outcome.reasons,
|
||||
severity: outcome.highestSeverity,
|
||||
criticalCount: outcome.criticalCount,
|
||||
highCount: outcome.highCount,
|
||||
kevCount: outcome.kevCount,
|
||||
fixableCount: outcome.fixableCount,
|
||||
suppressedCves: [...suppressedCves],
|
||||
rawWouldBlock,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A deploy that would have been blocked on raw severity but proceeded because
|
||||
* suppressions dropped every image below the threshold is a security-relevant
|
||||
* event: record it so the suppression-driven pass is traceable in the audit log.
|
||||
*/
|
||||
function recordSuppressionPassAudit(
|
||||
stackName: string,
|
||||
nodeId: number,
|
||||
policy: ScanPolicy,
|
||||
passes: SuppressionPass[],
|
||||
opts: PolicyEnforcementOptions,
|
||||
): void {
|
||||
const cves = [...new Set(passes.flatMap((p) => p.cves))];
|
||||
try {
|
||||
DatabaseService.getInstance().insertAuditLog({
|
||||
timestamp: Date.now(),
|
||||
username: opts.actor,
|
||||
method: opts.auditMethod ?? 'POST',
|
||||
path: opts.auditPath ?? `/api/stacks/${stackName}/deploy`,
|
||||
status_code: 200,
|
||||
node_id: nodeId,
|
||||
ip_address: opts.ip ?? '',
|
||||
summary: `policy.suppression_pass stack="${stackName}" policy="${policy.name}" images=[${passes.map((p) => p.imageRef).join(',')}] cves=[${cves.join(',')}]`,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[Policy] Failed to record suppression-pass audit entry:', err);
|
||||
}
|
||||
console.warn(
|
||||
'[Policy] Deploy for "%s" allowed by suppressions: %d image(s) would have matched %s (policy "%s")',
|
||||
sanitizeForLog(stackName), passes.length, describePolicyInputs(policyInputs(policy)), sanitizeForLog(policy.name),
|
||||
);
|
||||
}
|
||||
|
||||
export async function enforcePolicyPreDeploy(
|
||||
stackName: string,
|
||||
nodeId: number,
|
||||
opts: PolicyEnforcementOptions,
|
||||
): Promise<PolicyEnforcementResult> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const policy = db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
|
||||
|
||||
if (!policy || !policy.enabled || !policy.block_on_deploy) {
|
||||
return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
|
||||
}
|
||||
|
||||
if (opts.blockingEnabled === false) {
|
||||
return { ok: true, bypassed: false, policy, violations: [] };
|
||||
}
|
||||
|
||||
const svc = TrivyService.getInstance();
|
||||
if (!svc.isTrivyAvailable()) {
|
||||
notifyTrivyMissingOnce(nodeId, stackName);
|
||||
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
|
||||
}
|
||||
|
||||
let imageRefs: string[] = [];
|
||||
try {
|
||||
imageRefs = await ComposeService.getInstance(nodeId).listStackImages(stackName);
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'compose parse failed');
|
||||
console.error('[Policy] listStackImages failed for %s:', sanitizeForLog(stackName), sanitizeForLog(message));
|
||||
return {
|
||||
ok: false,
|
||||
bypassed: false,
|
||||
policy,
|
||||
violations: [{
|
||||
imageRef: '(compose parse error)',
|
||||
severity: 'UNKNOWN',
|
||||
criticalCount: 0,
|
||||
highCount: 0,
|
||||
kevCount: 0,
|
||||
fixableCount: 0,
|
||||
reasons: [],
|
||||
scanId: 0,
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
return enforcePolicyForImageRefs(stackName, nodeId, imageRefs, opts, policy);
|
||||
}
|
||||
|
||||
export async function enforcePolicyForImageRefs(
|
||||
stackName: string,
|
||||
nodeId: number,
|
||||
imageRefs: string[],
|
||||
opts: PolicyEnforcementOptions,
|
||||
matchedPolicy?: ScanPolicy,
|
||||
failClosedInvalidRefs = false,
|
||||
): Promise<PolicyEnforcementResult> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const policy = matchedPolicy ?? db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
|
||||
|
||||
if (!policy || !policy.enabled || !policy.block_on_deploy) {
|
||||
return { ok: true, bypassed: false, policy: policy ?? undefined, violations: [] };
|
||||
}
|
||||
|
||||
const svc = TrivyService.getInstance();
|
||||
if (!svc.isTrivyAvailable()) {
|
||||
notifyTrivyMissingOnce(nodeId, stackName);
|
||||
return { ok: true, bypassed: false, policy, violations: [], trivyMissing: true };
|
||||
}
|
||||
|
||||
const honorSuppressions = db.getGlobalSettings()['deploy_block_honor_suppressions'] === '1';
|
||||
|
||||
const debug = isDebugEnabled();
|
||||
if (debug) {
|
||||
console.log(
|
||||
'[Policy:debug] Evaluating "%s" against policy "%s" (inputs=%s, images=%d, honorSuppressions=%s)',
|
||||
sanitizeForLog(stackName), sanitizeForLog(policy.name), describePolicyInputs(policyInputs(policy)), imageRefs.length, honorSuppressions,
|
||||
);
|
||||
}
|
||||
|
||||
const violations: PolicyViolation[] = [];
|
||||
const suppressionPasses: SuppressionPass[] = [];
|
||||
for (const imageRef of imageRefs) {
|
||||
if (!validateImageRef(imageRef)) {
|
||||
if (failClosedInvalidRefs) {
|
||||
violations.push({
|
||||
imageRef,
|
||||
severity: 'UNKNOWN',
|
||||
criticalCount: 0,
|
||||
highCount: 0,
|
||||
kevCount: 0,
|
||||
fixableCount: 0,
|
||||
reasons: [],
|
||||
scanId: 0,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let scan: VulnerabilityScan;
|
||||
try {
|
||||
scan = await svc.scanImagePreflight(imageRef, nodeId, stackName);
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'pre-flight scan failed');
|
||||
console.error(`[Policy] scanImagePreflight failed for ${imageRef}:`, message);
|
||||
violations.push({
|
||||
imageRef,
|
||||
severity: 'UNKNOWN',
|
||||
criticalCount: 0,
|
||||
highCount: 0,
|
||||
kevCount: 0,
|
||||
fixableCount: 0,
|
||||
reasons: [],
|
||||
scanId: 0,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const evaluated = evaluateImageRisk(scan, imageRef, policy, honorSuppressions);
|
||||
if (debug) {
|
||||
console.log(
|
||||
'[Policy:debug] %s scanned: severity=%s kev=%d fixable=%d matched=[%s]',
|
||||
sanitizeForLog(imageRef), evaluated.severity, evaluated.kevCount, evaluated.fixableCount, evaluated.reasons.join(','),
|
||||
);
|
||||
}
|
||||
if (evaluated.reasons.length > 0) {
|
||||
violations.push({
|
||||
imageRef,
|
||||
severity: evaluated.severity,
|
||||
criticalCount: evaluated.criticalCount,
|
||||
highCount: evaluated.highCount,
|
||||
kevCount: evaluated.kevCount,
|
||||
fixableCount: evaluated.fixableCount,
|
||||
reasons: evaluated.reasons,
|
||||
scanId: scan.id,
|
||||
});
|
||||
} else if (
|
||||
honorSuppressions &&
|
||||
evaluated.suppressedCves.length > 0 &&
|
||||
evaluated.rawWouldBlock
|
||||
) {
|
||||
suppressionPasses.push({ imageRef, cves: evaluated.suppressedCves });
|
||||
}
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'policy evaluation failed');
|
||||
console.error(`[Policy] policy evaluation failed for ${imageRef}:`, message);
|
||||
violations.push({
|
||||
imageRef,
|
||||
severity: 'UNKNOWN',
|
||||
criticalCount: 0,
|
||||
highCount: 0,
|
||||
kevCount: 0,
|
||||
fixableCount: 0,
|
||||
reasons: [],
|
||||
scanId: scan.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (violations.length === 0) {
|
||||
if (suppressionPasses.length > 0) {
|
||||
recordSuppressionPassAudit(stackName, nodeId, policy, suppressionPasses, opts);
|
||||
}
|
||||
return { ok: true, bypassed: false, policy, violations: [] };
|
||||
}
|
||||
|
||||
if (opts.bypass) {
|
||||
try {
|
||||
db.insertAuditLog({
|
||||
timestamp: Date.now(),
|
||||
username: opts.actor,
|
||||
method: opts.auditMethod ?? 'POST',
|
||||
path: opts.auditPath ?? `/api/stacks/${stackName}/deploy`,
|
||||
status_code: 200,
|
||||
node_id: nodeId,
|
||||
ip_address: opts.ip ?? '',
|
||||
summary: `policy.bypass stack="${stackName}" policy="${policy.name}" violations=${violations.length} images=[${violations.map((v) => v.imageRef).join(',')}]`,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[Policy] Failed to record bypass audit entry:', err);
|
||||
}
|
||||
if (debug) {
|
||||
console.log(
|
||||
'[Policy:debug] Bypass for "%s" (%d violation(s))',
|
||||
sanitizeForLog(stackName), violations.length,
|
||||
);
|
||||
}
|
||||
return { ok: true, bypassed: true, policy, violations };
|
||||
}
|
||||
|
||||
console.warn(
|
||||
'[Policy] Blocked deploy for "%s": %d image(s) matched %s (policy "%s")',
|
||||
sanitizeForLog(stackName), violations.length, describePolicyInputs(policyInputs(policy)), sanitizeForLog(policy.name),
|
||||
);
|
||||
return { ok: false, bypassed: false, policy, violations };
|
||||
}
|
||||
|
||||
@@ -332,6 +332,10 @@ export class SchedulerService {
|
||||
case 'auto_start':
|
||||
output = await this.executeAutoStart(task);
|
||||
break;
|
||||
default: {
|
||||
const unhandledAction: never = task.action;
|
||||
throw new Error(`Unsupported scheduled action: ${unhandledAction}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} action completed in ${Date.now() - actionStart}ms`);
|
||||
@@ -673,6 +677,7 @@ export class SchedulerService {
|
||||
: [...allTargets];
|
||||
const labelFilter = task.prune_label_filter || undefined;
|
||||
const results: string[] = [];
|
||||
const failures: string[] = [];
|
||||
|
||||
for (const target of targets) {
|
||||
try {
|
||||
@@ -680,11 +685,16 @@ export class SchedulerService {
|
||||
results.push(`${target}: ${result.reclaimedBytes ?? 0} bytes reclaimed`);
|
||||
} catch (error: unknown) {
|
||||
const msg = error instanceof Error ? error.message : String(error);
|
||||
results.push(`${target}: failed (${msg})`);
|
||||
const failure = `${target}: failed (${msg})`;
|
||||
results.push(failure);
|
||||
failures.push(failure);
|
||||
}
|
||||
}
|
||||
|
||||
const filterSuffix = labelFilter ? ` (label: ${labelFilter})` : '';
|
||||
if (failures.length > 0) {
|
||||
throw new Error(`System prune failed${filterSuffix}: ${results.join('; ')}`);
|
||||
}
|
||||
return `System prune completed${filterSuffix}: ${results.join('; ')}`;
|
||||
}
|
||||
|
||||
|
||||
@@ -110,7 +110,7 @@ A Vulnerability Scan task runs Trivy against every image on the selected node an
|
||||
When a scheduled scan finishes, Sencho dispatches a completion notification. The severity reflects the outcome (info on a clean run, warning when findings are present); the category is `scan_finding`. The full message format and how it surfaces in the bell is documented in [Alerts & Notifications · Vulnerability scanning](/features/alerts-notifications#vulnerability-scanning).
|
||||
|
||||
<Frame>
|
||||
<img src="/images/scheduled-operations/create-scan.png" alt="The New scheduled task modal configured for a Vulnerability Scan. Name reads 'Nightly vulnerability scan'. Action is Vulnerability Scan. Node is Local with the helper text 'Every image on the selected node will be scanned.' Cron Expression is '0 3 * * *' with the preview 'At 03:00 AM'. The Enabled toggle reads ON. Cancel and Create buttons sit at the bottom right." />
|
||||
<img src="/images/scheduled-operations/create-scan.png" alt="The New scheduled task modal configured for a Vulnerability Scan. Name reads 'Nightly vulnerability scan'. Action is Vulnerability Scan. Node is Local with helper text explaining every image on the selected node will be scanned and scans run on local nodes only. Cron Expression is '0 3 * * *' with the preview 'At 03:00 AM'. The Enabled toggle reads ON. Cancel and Create buttons sit at the bottom right." />
|
||||
</Frame>
|
||||
|
||||
### Prune label filter
|
||||
@@ -118,7 +118,7 @@ When a scheduled scan finishes, Sencho dispatches a completion notification. The
|
||||
When creating a System Prune task, the **Label Filter (optional)** input scopes the prune to resources matching a specific Docker label. Use `key=value` format (for example, `com.docker.compose.project=staging`). Leave the field empty to prune every unused resource of the selected types.
|
||||
|
||||
<Frame>
|
||||
<img src="/images/scheduled-operations/create-prune.png" alt="The New scheduled task modal configured for a System Prune. Name reads 'Weekly cleanup'. Action is System Prune. The Prune Targets group shows four checked boxes: Containers, Images, Networks, Volumes. The Label Filter (optional) input reads 'com.docker.compose.project=staging' with the helper text 'Only prune resources matching this Docker label.' Cron Expression is '0 4 * * 0' with the preview 'At 04:00 AM, only on Sunday'. The Enabled toggle reads ON. The Delete after successful run checkbox is unchecked. Cancel and Create buttons sit at the bottom right." />
|
||||
<img src="/images/scheduled-operations/create-prune.png" alt="The New scheduled task modal configured for a System Prune. Name reads 'Weekly cleanup'. Action is System Prune. Node is Local. The Prune Targets group shows four checked boxes: Containers, Images, Networks, Volumes. The Label Filter (optional) input reads 'com.docker.compose.project=staging' with the helper text 'Only prune resources matching this Docker label.' Cron Expression is '0 4 * * 0' with the preview 'At 04:00 AM, only on Sunday'. The Enabled toggle reads ON. The Delete after successful run checkbox is unchecked. Cancel and Create buttons sit at the bottom right." />
|
||||
</Frame>
|
||||
|
||||
### Stack lifecycle scheduling
|
||||
@@ -158,6 +158,8 @@ Sencho uses standard 5-field cron expressions:
|
||||
* * * * *
|
||||
```
|
||||
|
||||
Schedules run on a one-minute granularity, so a six-field expression with a leading seconds field is not accepted: enter exactly five fields, or the create and edit forms reject it.
|
||||
|
||||
Common examples:
|
||||
|
||||
| Expression | Meaning |
|
||||
@@ -230,7 +232,7 @@ A background Scheduler Service evaluates due tasks every 60 seconds. When a task
|
||||
4. On failure, an `error`-level notification goes out through the configured channels. On recovery from a previously failing state, an `info`-level recovery notification is dispatched.
|
||||
5. The next run time is recalculated from the cron expression.
|
||||
|
||||
Most actions execute on whichever Sencho instance owns the schedule. The exception is **Auto-update Stack** and **Auto-update All Stacks**: when the target is a remote node, the gateway forwards execution to the remote node's `/api/auto-update/execute` endpoint, mirroring the per-node update flow used by the manual button. The remaining actions execute directly against the local instance's Compose directory, which is why lifecycle schedules for a remote node must be created from that node's own UI.
|
||||
Schedules are stored on the hub. Stack-targeted actions execute against the selected node, so lifecycle and per-stack update schedules can target either the hub or a connected remote node. System Prune and Vulnerability Scan run on local nodes only. Fleet Snapshot captures every node from the hub schedule.
|
||||
|
||||
If a task is still running from a previous firing, the scheduler skips the new firing to prevent overlap.
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import { toast } from '@/components/ui/toast-store';
|
||||
import { apiFetch, fetchForNode } from '@/lib/api';
|
||||
import { Combobox } from '@/components/ui/combobox';
|
||||
import type { ScheduledTask, TaskRun, NodeOption } from '@/types/scheduling';
|
||||
import { getCronDescription, formatTimestamp } from '@/lib/scheduling';
|
||||
import { getCronDescription, getCronFieldError, formatTimestamp } from '@/lib/scheduling';
|
||||
import {
|
||||
SCHEDULED_ACTIONS,
|
||||
SCHEDULED_ACTION_CATEGORIES,
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
resolveTaskAction,
|
||||
} from '@/lib/scheduledActions';
|
||||
|
||||
const DEFAULT_PRUNE_TARGETS = ['containers', 'images', 'networks', 'volumes'];
|
||||
const TIMELINE_WINDOW_HOURS = 24;
|
||||
const TIMELINE_WINDOW_MS = TIMELINE_WINDOW_HOURS * 60 * 60 * 1000;
|
||||
|
||||
@@ -72,7 +73,7 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter, p
|
||||
const [formCron, setFormCron] = useState('0 3 * * *');
|
||||
const [formEnabled, setFormEnabled] = useState(true);
|
||||
const [formDeleteAfterRun, setFormDeleteAfterRun] = useState(false);
|
||||
const [formPruneTargets, setFormPruneTargets] = useState<string[]>(['containers', 'images', 'networks', 'volumes']);
|
||||
const [formPruneTargets, setFormPruneTargets] = useState<string[]>(DEFAULT_PRUNE_TARGETS);
|
||||
const [formTargetServices, setFormTargetServices] = useState<string[]>([]);
|
||||
const [formPruneLabelFilter, setFormPruneLabelFilter] = useState('');
|
||||
const [availableServices, setAvailableServices] = useState<string[]>([]);
|
||||
@@ -153,7 +154,8 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter, p
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (formAction !== 'restart' || !formTargetId) {
|
||||
const actionDef = getActionById(formAction);
|
||||
if (!actionDef?.supportsServiceSelection || !formTargetId) {
|
||||
setAvailableServices([]);
|
||||
return;
|
||||
}
|
||||
@@ -198,7 +200,7 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter, p
|
||||
setFormCron('0 3 * * *');
|
||||
setFormEnabled(true);
|
||||
setFormDeleteAfterRun(false);
|
||||
setFormPruneTargets(['containers', 'images', 'networks', 'volumes']);
|
||||
setFormPruneTargets(DEFAULT_PRUNE_TARGETS);
|
||||
setFormTargetServices([]);
|
||||
setFormPruneLabelFilter('');
|
||||
setDialogOpen(true);
|
||||
@@ -215,7 +217,7 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter, p
|
||||
setFormEnabled(task.enabled === 1);
|
||||
setFormDeleteAfterRun((task.delete_after_run ?? 0) === 1);
|
||||
setFormPruneTargets(
|
||||
task.prune_targets ? JSON.parse(task.prune_targets) : ['containers', 'images', 'networks', 'volumes']
|
||||
task.prune_targets ? JSON.parse(task.prune_targets) : DEFAULT_PRUNE_TARGETS
|
||||
);
|
||||
setFormTargetServices(
|
||||
task.target_services ? JSON.parse(task.target_services) : []
|
||||
@@ -338,6 +340,7 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter, p
|
||||
|
||||
const currentAction = getActionById(formAction);
|
||||
const cronDescription = getCronDescription(formCron);
|
||||
const cronFieldError = getCronFieldError(formCron);
|
||||
const nodeOptions = useMemo(() => nodes.map(n => ({ value: String(n.id), label: n.name })), [nodes]);
|
||||
// Scan and prune run on the hub-local Docker daemon only; remote nodes are excluded from their pickers.
|
||||
const localNodeOptions = useMemo(
|
||||
@@ -346,7 +349,7 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter, p
|
||||
);
|
||||
const currentNodeOptions = currentAction?.nodeScope === 'local' ? localNodeOptions : nodeOptions;
|
||||
const isSaveDisabled =
|
||||
saving || !currentAction || !formName || !formCron
|
||||
saving || !currentAction || !formName || !formCron || !!cronFieldError
|
||||
|| (!!currentAction?.requiresStack && (!formTargetId || !formNodeId))
|
||||
|| (!!currentAction?.requiresNode && !currentAction.requiresStack && !formNodeId)
|
||||
|| (formAction === 'prune' && formPruneTargets.length === 0);
|
||||
@@ -748,7 +751,7 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter, p
|
||||
<div className="space-y-2">
|
||||
<Label>Prune Targets</Label>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
{['containers', 'images', 'networks', 'volumes'].map(target => (
|
||||
{DEFAULT_PRUNE_TARGETS.map(target => (
|
||||
<label key={target} className="flex items-center gap-2 text-sm cursor-pointer">
|
||||
<Checkbox
|
||||
checked={formPruneTargets.includes(target)}
|
||||
@@ -784,7 +787,9 @@ export default function ScheduledOperationsView({ filterNodeId, onClearFilter, p
|
||||
onChange={e => setFormCron(e.target.value)}
|
||||
className="font-mono"
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">{cronDescription}</p>
|
||||
{cronFieldError
|
||||
? <p className="text-xs text-destructive">{cronFieldError}</p>
|
||||
: <p className="text-xs text-muted-foreground">{cronDescription}</p>}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
|
||||
@@ -177,6 +177,32 @@ describe('ScheduledOperationsView', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('disables Create when the cron expression has a seconds field', async () => {
|
||||
render(<ScheduledOperationsView />);
|
||||
|
||||
await userEvent.click(await screen.findByRole('button', { name: /New Schedule/ }));
|
||||
await userEvent.type(await screen.findByPlaceholderText('e.g. Nightly stack restart'), 'cleanup');
|
||||
|
||||
await userEvent.click(screen.getAllByRole('combobox')[0]);
|
||||
await userEvent.click(await screen.findByRole('button', { name: 'System Prune' }));
|
||||
await userEvent.click(screen.getAllByRole('combobox')[1]);
|
||||
await userEvent.click(await screen.findByRole('button', { name: 'hub' }));
|
||||
|
||||
const createButton = screen.getByRole('button', { name: 'Create' });
|
||||
expect(createButton).toBeEnabled();
|
||||
|
||||
const cronInput = screen.getByPlaceholderText('0 3 * * *');
|
||||
await userEvent.clear(cronInput);
|
||||
await userEvent.type(cronInput, '30 0 3 * * *');
|
||||
|
||||
expect(screen.getByText(/seconds field is not supported/i)).toBeInTheDocument();
|
||||
expect(createButton).toBeDisabled();
|
||||
|
||||
await userEvent.clear(cronInput);
|
||||
await userEvent.type(cronInput, '0 4 * * *');
|
||||
expect(createButton).toBeEnabled();
|
||||
});
|
||||
|
||||
it('keeps Create disabled for a prune until a node is selected', async () => {
|
||||
render(<ScheduledOperationsView />);
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { getCronFieldError } from './scheduling';
|
||||
|
||||
describe('getCronFieldError', () => {
|
||||
it('accepts a standard 5-field expression', () => {
|
||||
expect(getCronFieldError('0 3 * * *')).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects a 6-field expression with a seconds field', () => {
|
||||
expect(getCronFieldError('30 0 3 * * *')).toMatch(/5 fields/);
|
||||
});
|
||||
|
||||
it('rejects expressions with extra fields beyond six', () => {
|
||||
expect(getCronFieldError('0 0 3 * * * 2026')).toMatch(/5 fields/);
|
||||
});
|
||||
|
||||
it('accepts cron nicknames such as @daily', () => {
|
||||
expect(getCronFieldError('@daily')).toBeNull();
|
||||
});
|
||||
|
||||
it('ignores empty or whitespace-only input', () => {
|
||||
expect(getCronFieldError('')).toBeNull();
|
||||
expect(getCronFieldError(' ')).toBeNull();
|
||||
});
|
||||
|
||||
it('tolerates irregular spacing between five fields', () => {
|
||||
expect(getCronFieldError(' 0 3 * * * ')).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,20 @@ export function getCronDescription(expression: string): string {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject cron expressions with a leading seconds field (6 or more fields). The
|
||||
* scheduler is minute-granular, so the seconds field could never be honored.
|
||||
* Nicknames like `@daily` (one token) pass. Returns an error message or null
|
||||
* when the field count is acceptable.
|
||||
*/
|
||||
export function getCronFieldError(expression: string): string | null {
|
||||
const trimmed = expression.trim();
|
||||
if (trimmed && trimmed.split(/\s+/).length >= 6) {
|
||||
return 'Use 5 fields (minute hour day month weekday). The seconds field is not supported.';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function formatTimestamp(ts: number | null): string {
|
||||
if (ts == null) return '-';
|
||||
return new Date(ts).toLocaleString();
|
||||
|
||||
Reference in New Issue
Block a user