feat(fleet): replicate scan policies across managed nodes (#649)

Scan policies now propagate from the control Sencho instance to every
registered remote. The control is the source of truth; replicas render
rules read-only with a managed-by-control banner. Pushes fire on every
policy write, record per-node success and failure on a new
fleet_sync_status table, and use node_proxy Bearer tokens exclusively
so only sibling Senchos can apply incoming sync payloads. Policy scope
now travels as a string identity (api_url or a local sentinel) so
node-scoped rules evaluate correctly on each target.
This commit is contained in:
Anso
2026-04-16 23:57:08 -04:00
committed by GitHub
parent 8ee0c0c476
commit 708d15b2b3
10 changed files with 888 additions and 35 deletions
@@ -0,0 +1,133 @@
/**
* Route-level tests for fleet sync endpoints introduced by PR 3.
* Focus: auth gating on /api/fleet/sync/:resource (node_proxy only) and
* payload validation. Service-level behavior is covered separately in
* fleet-sync-service.test.ts.
*/
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let adminAuthHeader: string;
let nodeProxyAuthHeader: string;
let LicenseService: typeof import('../services/LicenseService').LicenseService;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
({ LicenseService } = await import('../services/LicenseService'));
const adminToken = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
adminAuthHeader = `Bearer ${adminToken}`;
const proxyToken = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
nodeProxyAuthHeader = `Bearer ${proxyToken}`;
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
describe('GET /api/fleet/role', () => {
it('returns 401 without auth', async () => {
const res = await request(app).get('/api/fleet/role');
expect(res.status).toBe(401);
});
it('returns the current role for an authenticated admin', async () => {
const res = await request(app).get('/api/fleet/role').set('Authorization', adminAuthHeader);
expect(res.status).toBe(200);
expect(res.body).toEqual({ role: 'control' });
});
});
describe('POST /api/fleet/sync/:resource auth gate', () => {
const validRow = {
name: 'from-control',
node_identity: '',
stack_pattern: null,
max_severity: 'CRITICAL',
block_on_deploy: 0,
enabled: 1,
};
it('rejects unauthenticated callers with 401', async () => {
const res = await request(app).post('/api/fleet/sync/scan_policies').send({ rows: [validRow] });
expect(res.status).toBe(401);
});
it('rejects admin user session tokens with 403 NODE_PROXY_REQUIRED', async () => {
const res = await request(app)
.post('/api/fleet/sync/scan_policies')
.set('Authorization', adminAuthHeader)
.send({ rows: [validRow] });
expect(res.status).toBe(403);
expect(res.body.code).toBe('NODE_PROXY_REQUIRED');
});
it('rejects unknown resources with 400', async () => {
const res = await request(app)
.post('/api/fleet/sync/foo')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: [] });
expect(res.status).toBe(400);
});
it('rejects payloads without a rows array', async () => {
const res = await request(app)
.post('/api/fleet/sync/scan_policies')
.set('Authorization', nodeProxyAuthHeader)
.send({});
expect(res.status).toBe(400);
});
it('rejects rows with invalid max_severity', async () => {
const res = await request(app)
.post('/api/fleet/sync/scan_policies')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: [{ ...validRow, max_severity: 'GIGA' }] });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/max_severity/);
});
it('rejects rows with non-flag enabled value', async () => {
const res = await request(app)
.post('/api/fleet/sync/scan_policies')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: [{ ...validRow, enabled: 2 }] });
expect(res.status).toBe(400);
});
it('rejects payloads exceeding the row cap', async () => {
const bigRows = Array.from({ length: 5001 }, () => validRow);
const res = await request(app)
.post('/api/fleet/sync/scan_policies')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: bigRows });
expect(res.status).toBe(413);
});
});
describe('GET /api/fleet/sync-status', () => {
it('returns 401 without auth', async () => {
const res = await request(app).get('/api/fleet/sync-status');
expect(res.status).toBe(401);
});
it('returns 403 PAID_REQUIRED on community tier', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
const res = await request(app).get('/api/fleet/sync-status').set('Authorization', adminAuthHeader);
expect(res.status).toBe(403);
expect(res.body.code).toBe('PAID_REQUIRED');
vi.restoreAllMocks();
});
it('returns an empty list for an admin on paid tier', async () => {
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
const res = await request(app).get('/api/fleet/sync-status').set('Authorization', adminAuthHeader);
expect(res.status).toBe(200);
expect(Array.isArray(res.body)).toBe(true);
vi.restoreAllMocks();
});
});
@@ -0,0 +1,182 @@
/**
* Unit tests for FleetSyncService: the service that replicates security
* configuration from a control Sencho instance to every registered remote.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
const {
mockGetNodes,
mockGetNode,
mockGetScanPolicies,
mockReplaceReplicatedScanPolicies,
mockRecordFleetSyncSuccess,
mockRecordFleetSyncFailure,
mockGetSystemState,
mockSetSystemState,
mockAxiosPost,
} = vi.hoisted(() => ({
mockGetNodes: vi.fn().mockReturnValue([]),
mockGetNode: vi.fn(),
mockGetScanPolicies: vi.fn().mockReturnValue([]),
mockReplaceReplicatedScanPolicies: vi.fn(),
mockRecordFleetSyncSuccess: vi.fn(),
mockRecordFleetSyncFailure: vi.fn(),
mockGetSystemState: vi.fn().mockReturnValue(null),
mockSetSystemState: vi.fn(),
mockAxiosPost: vi.fn().mockResolvedValue({ data: { success: true } }),
}));
vi.mock('../services/DatabaseService', () => ({
DatabaseService: {
getInstance: () => ({
getNodes: mockGetNodes,
getScanPolicies: mockGetScanPolicies,
replaceReplicatedScanPolicies: mockReplaceReplicatedScanPolicies,
recordFleetSyncSuccess: mockRecordFleetSyncSuccess,
recordFleetSyncFailure: mockRecordFleetSyncFailure,
getSystemState: mockGetSystemState,
setSystemState: mockSetSystemState,
}),
},
}));
vi.mock('../services/NodeRegistry', () => ({
NodeRegistry: {
getInstance: () => ({
getNode: mockGetNode,
}),
},
}));
vi.mock('axios', () => ({
default: { post: mockAxiosPost },
AxiosError: class AxiosError extends Error {
response?: { status: number; statusText: string; data: unknown };
},
}));
import { FleetSyncService, LOCAL_IDENTITY_SENTINEL } from '../services/FleetSyncService';
beforeEach(() => {
vi.clearAllMocks();
mockGetSystemState.mockReturnValue(null);
});
describe('FleetSyncService.getRole', () => {
it('returns control when no fleet_role is set in system_state', () => {
mockGetSystemState.mockReturnValue(null);
expect(FleetSyncService.getRole()).toBe('control');
});
it('returns replica when fleet_role system_state is "replica"', () => {
mockGetSystemState.mockImplementation((key: string) => (key === 'fleet_role' ? 'replica' : null));
expect(FleetSyncService.getRole()).toBe('replica');
});
});
describe('FleetSyncService.getSelfIdentity', () => {
it('returns LOCAL_IDENTITY_SENTINEL on control nodes', () => {
mockGetSystemState.mockReturnValue(null);
expect(FleetSyncService.getSelfIdentity()).toBe(LOCAL_IDENTITY_SENTINEL);
});
it('returns the cached target identity on replicas', () => {
mockGetSystemState.mockImplementation((key: string) => {
if (key === 'fleet_role') return 'replica';
if (key === 'fleet_self_identity') return 'https://sencho.example.com';
return null;
});
expect(FleetSyncService.getSelfIdentity()).toBe('https://sencho.example.com');
});
});
describe('FleetSyncService.resolveIdentityForNodeId', () => {
it('returns empty string when nodeId is null (fleet-wide policy)', () => {
expect(FleetSyncService.resolveIdentityForNodeId(null)).toBe('');
});
it('returns LOCAL_IDENTITY_SENTINEL when the node is local', () => {
mockGetNode.mockReturnValue({ id: 1, type: 'local', api_url: '', api_token: '' });
expect(FleetSyncService.resolveIdentityForNodeId(1)).toBe(LOCAL_IDENTITY_SENTINEL);
});
it('returns the remote api_url for remote nodes', () => {
mockGetNode.mockReturnValue({ id: 2, type: 'remote', api_url: 'https://remote.example.com', api_token: 'tok' });
expect(FleetSyncService.resolveIdentityForNodeId(2)).toBe('https://remote.example.com');
});
});
describe('FleetSyncService.pushResource', () => {
it('does not push when this instance is a replica', async () => {
mockGetSystemState.mockImplementation((key: string) => (key === 'fleet_role' ? 'replica' : null));
await FleetSyncService.getInstance().pushResource('scan_policies');
expect(mockAxiosPost).not.toHaveBeenCalled();
});
it('skips nodes without api_url or api_token', async () => {
mockGetNodes.mockReturnValue([
{ id: 1, type: 'local', api_url: '', api_token: '' },
{ id: 2, type: 'remote', api_url: '', api_token: 'tok' },
{ id: 3, type: 'remote', api_url: 'https://good.example', api_token: '' },
]);
await FleetSyncService.getInstance().pushResource('scan_policies');
expect(mockAxiosPost).not.toHaveBeenCalled();
});
it('pushes to every configured remote with local rows only', async () => {
mockGetNodes.mockReturnValue([
{ id: 2, type: 'remote', api_url: 'https://a.example', api_token: 'tokA', name: 'A' },
{ id: 3, type: 'remote', api_url: 'https://b.example', api_token: 'tokB', name: 'B' },
]);
mockGetScanPolicies.mockReturnValue([
{ id: 1, name: 'local-1', node_identity: '', replicated_from_control: 0, created_at: 1, updated_at: 1 },
{ id: 2, name: 'mirrored', node_identity: 'https://somewhere', replicated_from_control: 1, created_at: 1, updated_at: 1 },
]);
await FleetSyncService.getInstance().pushResource('scan_policies');
expect(mockAxiosPost).toHaveBeenCalledTimes(2);
const firstCall = mockAxiosPost.mock.calls[0];
expect(firstCall[0]).toBe('https://a.example/api/fleet/sync/scan_policies');
expect(firstCall[1].rows).toHaveLength(1);
expect(firstCall[1].rows[0].name).toBe('local-1');
expect(firstCall[1].targetIdentity).toBe('https://a.example');
expect(firstCall[2].headers.Authorization).toBe('Bearer tokA');
expect(mockRecordFleetSyncSuccess).toHaveBeenCalledWith(2, 'scan_policies');
expect(mockRecordFleetSyncSuccess).toHaveBeenCalledWith(3, 'scan_policies');
});
it('records per-node failure without throwing when one remote errors', async () => {
mockGetNodes.mockReturnValue([
{ id: 2, type: 'remote', api_url: 'https://fail.example', api_token: 'tok', name: 'fail' },
{ id: 3, type: 'remote', api_url: 'https://ok.example', api_token: 'tok2', name: 'ok' },
]);
mockAxiosPost.mockImplementation((url: string) => {
if (url.includes('fail.example')) return Promise.reject(new Error('network error'));
return Promise.resolve({ data: { success: true } });
});
await expect(FleetSyncService.getInstance().pushResource('scan_policies')).resolves.not.toThrow();
expect(mockRecordFleetSyncFailure).toHaveBeenCalledWith(2, 'scan_policies', expect.stringContaining('network error'));
expect(mockRecordFleetSyncSuccess).toHaveBeenCalledWith(3, 'scan_policies');
});
});
describe('FleetSyncService.applyIncomingSync', () => {
it('promotes this instance to replica and caches target identity', () => {
const rows = [{
id: 0, name: 'from-control', node_id: null, node_identity: '',
stack_pattern: null, max_severity: 'CRITICAL' as const,
block_on_deploy: 0, enabled: 1, replicated_from_control: 1,
created_at: 1, updated_at: 1,
}];
FleetSyncService.getInstance().applyIncomingSync('scan_policies', rows, 'https://me.example');
expect(mockSetSystemState).toHaveBeenCalledWith('fleet_role', 'replica');
expect(mockSetSystemState).toHaveBeenCalledWith('fleet_self_identity', 'https://me.example');
expect(mockReplaceReplicatedScanPolicies).toHaveBeenCalledWith(rows);
});
it('skips identity caching when targetIdentity is empty', () => {
FleetSyncService.getInstance().applyIncomingSync('scan_policies', [], '');
expect(mockSetSystemState).toHaveBeenCalledWith('fleet_role', 'replica');
expect(mockSetSystemState).not.toHaveBeenCalledWith('fleet_self_identity', expect.anything());
});
});
+105 -3
View File
@@ -28,6 +28,7 @@ import { ImageUpdateService } from './services/ImageUpdateService';
import { templateService } from './services/TemplateService';
import { ErrorParser } from './utils/ErrorParser';
import { NodeRegistry } from './services/NodeRegistry';
import { FleetSyncService } from './services/FleetSyncService';
import { LicenseService, type LicenseTier, type LicenseVariant, isLicenseTier, isLicenseVariant, normalizeTier, normalizeVariant, PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './services/LicenseService';
import { WebhookService } from './services/WebhookService';
import { SSOService } from './services/SSOService';
@@ -1570,6 +1571,16 @@ const requireAdmin = (req: Request, res: Response): boolean => {
return true;
};
// Only accept calls from a sibling Sencho using its node_proxy Bearer token.
// Browser sessions, API tokens, and console tokens are all rejected.
const requireNodeProxy = (req: Request, res: Response): boolean => {
if (req.user?.username !== 'node-proxy') {
res.status(403).json({ error: 'Node proxy authentication required.', code: 'NODE_PROXY_REQUIRED' });
return false;
}
return true;
};
const requireBody = (req: Request, res: Response): boolean => {
if (!req.body || typeof req.body !== 'object') {
res.status(400).json({ error: 'Request body is required' });
@@ -1607,7 +1618,7 @@ async function triggerPostDeployScan(
if (imageRefs.size === 0) return;
const db = DatabaseService.getInstance();
const policy = db.getMatchingPolicy(nodeId, stackName);
const policy = db.getMatchingPolicy(nodeId, stackName, FleetSyncService.getSelfIdentity());
for (const imageRef of imageRefs) {
try {
@@ -1956,6 +1967,75 @@ interface FleetNodeOverview {
stacks: string[] | null;
}
// Fleet role: tells the frontend whether this Sencho is the control or a replica.
// The control serves read+write for security rules. Replicas are read-only and managed upstream.
app.get('/api/fleet/role', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
res.json({ role: FleetSyncService.getRole() });
});
const MAX_SYNC_ROWS = 5000;
const VALID_SEVERITY = new Set(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']);
const isIntFlag = (v: unknown): v is 0 | 1 => v === 0 || v === 1;
function validateScanPolicyRow(row: unknown): string | null {
if (!row || typeof row !== 'object') return 'row must be an object';
const r = row as Record<string, unknown>;
if (typeof r.name !== 'string' || r.name.length === 0 || r.name.length > 200) return 'name must be a non-empty string';
if (typeof r.max_severity !== 'string' || !VALID_SEVERITY.has(r.max_severity)) return 'max_severity must be CRITICAL, HIGH, MEDIUM, or LOW';
if (r.stack_pattern !== null && typeof r.stack_pattern !== 'string') return 'stack_pattern must be a string or null';
if (typeof r.stack_pattern === 'string' && r.stack_pattern.length > 200) return 'stack_pattern is too long';
if (typeof r.node_identity !== 'string') return 'node_identity must be a string';
if (r.node_identity.length > 500) return 'node_identity is too long';
if (!isIntFlag(r.block_on_deploy)) return 'block_on_deploy must be 0 or 1';
if (!isIntFlag(r.enabled)) return 'enabled must be 0 or 1';
return null;
}
// Fleet sync: receive a full replacement of a replicated resource from the control.
// Only scan_policies are supported today; future resources (CVE suppressions) will plug in here.
// Restricted to node_proxy Bearer tokens so only a sibling Sencho can push.
app.post('/api/fleet/sync/:resource', authMiddleware, (req: Request, res: Response): void => {
if (!requireNodeProxy(req, res)) return;
const resource = req.params.resource;
if (resource !== 'scan_policies') {
res.status(400).json({ error: `Unsupported sync resource: ${resource}` });
return;
}
const body = req.body ?? {};
const rows = Array.isArray(body.rows) ? body.rows : null;
const targetIdentity = typeof body.targetIdentity === 'string' ? body.targetIdentity : '';
if (!rows) {
res.status(400).json({ error: 'rows array is required' });
return;
}
if (rows.length > MAX_SYNC_ROWS) {
res.status(413).json({ error: `Too many rows (max ${MAX_SYNC_ROWS})` });
return;
}
for (let i = 0; i < rows.length; i++) {
const err = validateScanPolicyRow(rows[i]);
if (err) {
res.status(400).json({ error: `Invalid row at index ${i}: ${err}` });
return;
}
}
try {
FleetSyncService.getInstance().applyIncomingSync(resource, rows, targetIdentity);
res.json({ success: true, applied: rows.length });
} catch (error) {
console.error('[FleetSync] Failed to apply incoming sync:', error);
res.status(500).json({ error: 'Failed to apply sync' });
}
});
// Fleet sync status: surfaces per-node replication results so operators can spot stale replicas.
app.get('/api/fleet/sync-status', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requirePaid(req, res)) return;
res.json(DatabaseService.getInstance().getFleetSyncStatuses());
});
app.get('/api/fleet/overview', authMiddleware, async (_req: Request, res: Response): Promise<void> => {
try {
const debug = isDebugEnabled();
@@ -7495,6 +7575,10 @@ app.get('/api/security/policies', authMiddleware, (req: Request, res: Response):
app.post('/api/security/policies', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requirePaid(req, res)) return;
if (FleetSyncService.getRole() === 'replica') {
res.status(403).json({ error: 'Security policies are managed from the control node.' });
return;
}
const { name, node_id, stack_pattern, max_severity, block_on_deploy, enabled } = req.body ?? {};
if (!name || typeof name !== 'string' || !name.trim()) {
res.status(400).json({ error: 'Policy name is required' }); return;
@@ -7504,14 +7588,18 @@ app.post('/api/security/policies', authMiddleware, (req: Request, res: Response)
res.status(400).json({ error: 'max_severity must be CRITICAL, HIGH, MEDIUM, or LOW' }); return;
}
try {
const resolvedNodeId = node_id != null ? Number(node_id) : null;
const policy = DatabaseService.getInstance().createScanPolicy({
name: name.trim(),
node_id: node_id != null ? Number(node_id) : null,
node_id: resolvedNodeId,
node_identity: FleetSyncService.resolveIdentityForNodeId(resolvedNodeId),
stack_pattern: stack_pattern ? String(stack_pattern) : null,
max_severity,
block_on_deploy: block_on_deploy ? 1 : 0,
enabled: enabled === false ? 0 : 1,
replicated_from_control: 0,
});
FleetSyncService.getInstance().pushResourceAsync('scan_policies');
res.status(201).json(policy);
} catch (error) {
console.error('[Security] Failed to create policy:', error);
@@ -7522,6 +7610,10 @@ app.post('/api/security/policies', authMiddleware, (req: Request, res: Response)
app.put('/api/security/policies/:id', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requirePaid(req, res)) return;
if (FleetSyncService.getRole() === 'replica') {
res.status(403).json({ error: 'Security policies are managed from the control node.' });
return;
}
const id = Number(req.params.id);
if (!Number.isFinite(id)) {
res.status(400).json({ error: 'Invalid policy id' }); return;
@@ -7529,7 +7621,11 @@ app.put('/api/security/policies/:id', authMiddleware, (req: Request, res: Respon
const body = req.body ?? {};
const updates: Record<string, unknown> = {};
if (body.name !== undefined) updates.name = String(body.name).trim();
if (body.node_id !== undefined) updates.node_id = body.node_id != null ? Number(body.node_id) : null;
if (body.node_id !== undefined) {
const resolvedNodeId = body.node_id != null ? Number(body.node_id) : null;
updates.node_id = resolvedNodeId;
updates.node_identity = FleetSyncService.resolveIdentityForNodeId(resolvedNodeId);
}
if (body.stack_pattern !== undefined) updates.stack_pattern = body.stack_pattern ? String(body.stack_pattern) : null;
if (body.max_severity !== undefined) {
const validSeverities = new Set(['CRITICAL', 'HIGH', 'MEDIUM', 'LOW']);
@@ -7544,17 +7640,23 @@ app.put('/api/security/policies/:id', authMiddleware, (req: Request, res: Respon
if (!policy) {
res.status(404).json({ error: 'Policy not found' }); return;
}
FleetSyncService.getInstance().pushResourceAsync('scan_policies');
res.json(policy);
});
app.delete('/api/security/policies/:id', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requirePaid(req, res)) return;
if (FleetSyncService.getRole() === 'replica') {
res.status(403).json({ error: 'Security policies are managed from the control node.' });
return;
}
const id = Number(req.params.id);
if (!Number.isFinite(id)) {
res.status(400).json({ error: 'Invalid policy id' }); return;
}
DatabaseService.getInstance().deleteScanPolicy(id);
FleetSyncService.getInstance().pushResourceAsync('scan_policies');
res.json({ success: true });
});
+138 -8
View File
@@ -308,14 +308,24 @@ export interface ScanPolicy {
id: number;
name: string;
node_id: number | null;
node_identity: string;
stack_pattern: string | null;
max_severity: VulnSeverity;
block_on_deploy: number;
enabled: number;
replicated_from_control: number;
created_at: number;
updated_at: number;
}
export interface FleetSyncStatus {
node_id: number;
resource: string;
last_success_at: number | null;
last_failure_at: number | null;
last_error: string | null;
}
export interface ScanSummary {
image_ref: string;
highest_severity: VulnSeverity | null;
@@ -352,6 +362,7 @@ export class DatabaseService {
this.migrateRegistries();
this.migrateRoleAssignments();
this.migrateNotificationRoutes();
this.migrateScanPolicyFleetColumns();
}
public static getInstance(): DatabaseService {
@@ -613,6 +624,15 @@ export class DatabaseService {
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS fleet_sync_status (
node_id INTEGER NOT NULL,
resource TEXT NOT NULL,
last_success_at INTEGER,
last_failure_at INTEGER,
last_error TEXT,
PRIMARY KEY (node_id, resource)
);
CREATE TABLE IF NOT EXISTS stack_labels (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id INTEGER NOT NULL DEFAULT 0,
@@ -878,6 +898,18 @@ export class DatabaseService {
try { this.db.prepare('ALTER TABLE notification_history ADD COLUMN dispatch_error TEXT').run(); } catch { /* already exists */ }
}
private migrateScanPolicyFleetColumns(): void {
const tryAddColumn = (table: string, col: string, def: string) => {
try {
this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run();
} catch {
/* column already present */
}
};
tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''");
tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0');
}
// --- Agents ---
public getAgents(): Agent[] {
@@ -2309,20 +2341,29 @@ export class DatabaseService {
const now = Date.now();
const result = this.db
.prepare(
`INSERT INTO scan_policies (name, node_id, stack_pattern, max_severity, block_on_deploy, enabled, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
`INSERT INTO scan_policies (name, node_id, node_identity, stack_pattern, max_severity, block_on_deploy, enabled, replicated_from_control, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run(
policy.name,
policy.node_id,
policy.node_identity ?? '',
policy.stack_pattern,
policy.max_severity,
policy.block_on_deploy,
policy.enabled,
policy.replicated_from_control ?? 0,
now,
now,
);
return { ...policy, id: result.lastInsertRowid as number, created_at: now, updated_at: now };
return {
...policy,
node_identity: policy.node_identity ?? '',
replicated_from_control: policy.replicated_from_control ?? 0,
id: result.lastInsertRowid as number,
created_at: now,
updated_at: now,
};
}
public updateScanPolicy(
@@ -2332,8 +2373,8 @@ export class DatabaseService {
const existing = this.getScanPolicy(id);
if (!existing) return null;
const ALLOWED_COLUMNS = new Set([
'name', 'node_id', 'stack_pattern', 'max_severity',
'block_on_deploy', 'enabled',
'name', 'node_id', 'node_identity', 'stack_pattern', 'max_severity',
'block_on_deploy', 'enabled', 'replicated_from_control',
]);
const fields: string[] = [];
const values: unknown[] = [];
@@ -2356,9 +2397,41 @@ export class DatabaseService {
this.db.prepare('DELETE FROM scan_policies WHERE id = ?').run(id);
}
/**
* Replace all policies that were replicated from a control node with the
* provided rows in a single transaction. Local-only policies (created on
* this instance directly) are left untouched.
*/
public replaceReplicatedScanPolicies(rows: ScanPolicy[]): void {
const now = Date.now();
const deleteStmt = this.db.prepare('DELETE FROM scan_policies WHERE replicated_from_control = 1');
const insertStmt = this.db.prepare(
`INSERT INTO scan_policies (name, node_id, node_identity, stack_pattern, max_severity, block_on_deploy, enabled, replicated_from_control, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?)`,
);
const txn = this.db.transaction((policies: ScanPolicy[]) => {
deleteStmt.run();
for (const p of policies) {
insertStmt.run(
p.name,
null,
p.node_identity ?? '',
p.stack_pattern,
p.max_severity,
p.block_on_deploy,
p.enabled,
p.created_at ?? now,
p.updated_at ?? now,
);
}
});
txn(rows);
}
public getMatchingPolicy(
nodeId: number,
stackName: string | null,
selfIdentity: string,
): ScanPolicy | null {
const policies = this.db
.prepare(
@@ -2373,11 +2446,22 @@ export class DatabaseService {
);
return regex.test(stackName);
};
const scoped = policies.filter((p) => matchesStack(p.stack_pattern));
const matchesIdentity = (p: ScanPolicy): boolean => {
// Locally created policies (never replicated) apply based on node_id logic already filtered.
if (p.replicated_from_control === 0) return true;
// Replicated policies without a specific identity are fleet-wide.
if (!p.node_identity) return true;
// Identity-scoped replicated policies only apply to their target instance.
return p.node_identity === selfIdentity;
};
const scoped = policies.filter((p) => matchesStack(p.stack_pattern) && matchesIdentity(p));
if (scoped.length === 0) return null;
const isNodeScoped = (p: ScanPolicy): boolean => Boolean(p.node_id) || Boolean(p.node_identity);
scoped.sort((a, b) => {
if (a.node_id && !b.node_id) return -1;
if (!a.node_id && b.node_id) return 1;
const aNode = isNodeScoped(a);
const bNode = isNodeScoped(b);
if (aNode && !bNode) return -1;
if (!aNode && bNode) return 1;
if (a.stack_pattern && !b.stack_pattern) return -1;
if (!a.stack_pattern && b.stack_pattern) return 1;
return 0;
@@ -2385,6 +2469,52 @@ export class DatabaseService {
return scoped[0];
}
// --- Fleet Sync Status ---
public getFleetSyncStatuses(): FleetSyncStatus[] {
return this.db
.prepare('SELECT * FROM fleet_sync_status ORDER BY node_id, resource')
.all() as FleetSyncStatus[];
}
public recordFleetSyncSuccess(nodeId: number, resource: string): void {
const now = Date.now();
this.db
.prepare(
`INSERT INTO fleet_sync_status (node_id, resource, last_success_at, last_failure_at, last_error)
VALUES (?, ?, ?, NULL, NULL)
ON CONFLICT(node_id, resource) DO UPDATE SET
last_success_at = excluded.last_success_at,
last_error = NULL`,
)
.run(nodeId, resource, now);
}
public recordFleetSyncFailure(nodeId: number, resource: string, error: string): void {
const now = Date.now();
this.db
.prepare(
`INSERT INTO fleet_sync_status (node_id, resource, last_failure_at, last_error)
VALUES (?, ?, ?, ?)
ON CONFLICT(node_id, resource) DO UPDATE SET
last_failure_at = excluded.last_failure_at,
last_error = excluded.last_error`,
)
.run(nodeId, resource, now, error);
}
public getFailedSyncTargets(resource: string, maxAgeMs: number): FleetSyncStatus[] {
const cutoff = Date.now() - maxAgeMs;
return this.db
.prepare(
`SELECT * FROM fleet_sync_status
WHERE resource = ?
AND (last_failure_at IS NOT NULL AND last_failure_at > ?)
AND (last_success_at IS NULL OR last_success_at < last_failure_at)`,
)
.all(resource, cutoff) as FleetSyncStatus[];
}
// --- Stack Labels ---
public getLabels(nodeId: number): Label[] {
+187
View File
@@ -0,0 +1,187 @@
import axios, { AxiosError } from 'axios';
import { DatabaseService, Node, ScanPolicy } from './DatabaseService';
import { NodeRegistry } from './NodeRegistry';
export type FleetResource = 'scan_policies';
export type FleetRole = 'control' | 'replica';
export const LOCAL_IDENTITY_SENTINEL = 'local';
/**
* FleetSyncService replicates security configuration from a control Sencho
* instance to every managed remote node. Security rules live on the control's
* SQLite database; each write triggers a push of the full table to every
* remote that has an api_url and api_token configured.
*
* Push failures for a specific remote are logged and recorded on the
* fleet_sync_status table so the UI and future retry logic can see stale
* nodes.
*/
export class FleetSyncService {
private static instance: FleetSyncService;
private constructor() {}
public static getInstance(): FleetSyncService {
if (!FleetSyncService.instance) {
FleetSyncService.instance = new FleetSyncService();
}
return FleetSyncService.instance;
}
/**
* Resolve the fleet role for this instance.
* A node becomes a replica the first time it accepts a fleet sync push.
*/
public static getRole(): FleetRole {
return DatabaseService.getInstance().getSystemState('fleet_role') === 'replica' ? 'replica' : 'control';
}
/**
* The identity string used when matching scan policies on this instance.
* Control nodes use the LOCAL_IDENTITY_SENTINEL. Replicas use the
* identity they were told during the most recent sync push. If a replica
* is missing its cached identity (e.g. the sync row has been corrupted),
* return the empty string; callers treat this as fleet-wide only and log.
*/
public static getSelfIdentity(): string {
if (FleetSyncService.getRole() === 'replica') {
const cached = DatabaseService.getInstance().getSystemState('fleet_self_identity');
if (!cached) {
if (!FleetSyncService.warnedMissingIdentity) {
console.warn(
'[FleetSync] Replica has no cached self-identity. Identity-scoped policies will not apply until the next sync push.',
);
FleetSyncService.warnedMissingIdentity = true;
}
return '';
}
return cached;
}
return LOCAL_IDENTITY_SENTINEL;
}
private static warnedMissingIdentity = false;
/**
* Map a policy's node_id to a node_identity string.
* - NULL node_id → '' (fleet-wide)
* - Local node → LOCAL_IDENTITY_SENTINEL
* - Remote node → the node's api_url
*/
public static resolveIdentityForNodeId(nodeId: number | null | undefined): string {
if (nodeId == null) return '';
const node = NodeRegistry.getInstance().getNode(nodeId);
if (!node) return '';
if (node.type === 'remote' && node.api_url) return node.api_url;
return LOCAL_IDENTITY_SENTINEL;
}
/**
* Push the current state of a resource to every remote node.
* Failures are recorded but do not bubble up to the caller.
*/
public async pushResource(resource: FleetResource): Promise<void> {
if (FleetSyncService.getRole() === 'replica') {
// Replicas never push; they only receive.
return;
}
const db = DatabaseService.getInstance();
const nodes = db.getNodes().filter((n): n is Node & { id: number } => {
return n.type === 'remote' && Boolean(n.api_url) && Boolean(n.api_token) && n.id != null;
});
if (nodes.length === 0) return;
const rows = this.loadResource(resource);
const pushedAt = Date.now();
await Promise.all(
nodes.map(async (node) => {
const baseUrl = (node.api_url ?? '').replace(/\/$/, '');
try {
await axios.post(
`${baseUrl}/api/fleet/sync/${resource}`,
{
rows,
pushedAt,
targetIdentity: node.api_url,
},
{
headers: { Authorization: `Bearer ${node.api_token}` },
timeout: 15_000,
},
);
db.recordFleetSyncSuccess(node.id, resource);
} catch (err) {
const message = this.formatError(err);
console.warn(
`[FleetSync] Failed to push ${resource} to "${node.name}" (${baseUrl}): ${message}`,
);
db.recordFleetSyncFailure(node.id, resource, message);
}
}),
);
}
/**
* Fire and forget helper for write handlers. Errors are already logged
* inside pushResource; this swallows any residual rejection so request
* handlers can stay synchronous.
*/
public pushResourceAsync(resource: FleetResource): void {
this.pushResource(resource).catch((err) => {
console.error(`[FleetSync] Unexpected error pushing ${resource}:`, err);
});
}
/**
* Apply a received sync payload on a replica.
* This promotes the instance to 'replica' mode if not already, caches
* the target identity it was told, and replaces replicated rows atomically.
*/
public applyIncomingSync(resource: FleetResource, rows: ScanPolicy[], targetIdentity: string): void {
const db = DatabaseService.getInstance();
db.setSystemState('fleet_role', 'replica');
if (targetIdentity) {
db.setSystemState('fleet_self_identity', targetIdentity);
}
if (resource === 'scan_policies') {
db.replaceReplicatedScanPolicies(rows);
}
}
private loadResource(resource: FleetResource): unknown[] {
const db = DatabaseService.getInstance();
if (resource === 'scan_policies') {
return db
.getScanPolicies()
.filter((p) => p.replicated_from_control === 0)
.map((p) => ({
name: p.name,
node_identity: p.node_identity,
stack_pattern: p.stack_pattern,
max_severity: p.max_severity,
block_on_deploy: p.block_on_deploy,
enabled: p.enabled,
created_at: p.created_at,
updated_at: p.updated_at,
}));
}
return [];
}
private formatError(err: unknown): string {
if (err instanceof AxiosError) {
if (err.response) {
const data = err.response.data;
const detail = typeof data === 'object' && data && 'error' in data
? String((data as { error: unknown }).error)
: err.response.statusText;
return `HTTP ${err.response.status}: ${detail}`;
}
return err.message;
}
return err instanceof Error ? err.message : String(err);
}
}
+1
View File
@@ -103,6 +103,7 @@
"features/host-console",
"features/multi-node",
"features/fleet-view",
"features/fleet-sync",
"features/remote-updates",
"features/stack-labels",
"features/alerts-notifications",
+71
View File
@@ -0,0 +1,71 @@
---
title: "Fleet Sync"
description: "How security rules replicate from the control Sencho instance to remote nodes."
---
When you manage several Sencho instances as a fleet, the control instance (the one where you added remote nodes in **Settings → Nodes**) acts as the source of truth for security configuration. Rules you create on the control replicate automatically to every remote so the entire fleet enforces the same policies.
Today this covers **vulnerability scan policies**. Additional resources (CVE suppressions, scan filters) will reuse the same mechanism.
## Control vs replica
Every Sencho instance has a **role** that determines whether it accepts writes for replicated resources:
| Role | Behavior |
|------|----------|
| **Control** | Default for any standalone instance and for the node you point your browser at when managing a fleet. Accepts create, edit, and delete for security rules. Pushes changes to every remote. |
| **Replica** | An instance that has received at least one sync push from a control. Shows rules read-only with a banner indicating they are managed upstream. Returns `403 Forbidden` for direct write attempts. |
Role detection is automatic: a Sencho instance becomes a replica the first time it accepts a sync push, and stays that way until the underlying database is reset.
<Frame>
<img src="/images/fleet-sync/replica-banner.png" alt="Settings page on a replica showing the 'Managed by control node' banner above the policies list" />
</Frame>
## How replication works
1. You create, edit, or delete a scan policy on the control.
2. The control commits the change to its local database.
3. The control iterates every remote node registered in **Settings → Nodes** that has an API URL and token configured.
4. For each remote, the control posts the full current list of policies to `/api/fleet/sync/scan_policies`, tagged with the remote's own identity.
5. The remote replaces its replicated rows in a single transaction and records the result.
Local-only rules created directly on a remote coexist with the replicated set; sync only replaces rows flagged as coming from the control.
## Policy scope across a fleet
When you create a policy on the control, its scope is preserved during replication:
- **Fleet-wide** (no specific node selected): applies on every instance in the fleet.
- **Specific node**: applies only on the instance whose identity matches. A policy targeting `Node A` evaluates only during scans that run on Node A, even after it has replicated to every other remote.
Only one policy is evaluated per deploy. The most specific match wins, in this order:
1. Node-scoped and stack-scoped policies.
2. Node-scoped, stack-wildcard policies.
3. Fleet-wide, stack-scoped policies.
4. Fleet-wide, stack-wildcard policies.
## Handling remote failures
Pushes happen in parallel with a 15-second timeout per remote. A failing remote (offline, expired token, network issue) does not stop the push from succeeding on other remotes. The control records the last success and failure timestamp per node; subsequent writes on the control retry the push, so a remote that comes back online picks up the latest state on the next policy change.
## Troubleshooting
### Security rules are missing on this node
You are looking at a replica. Navigate to the control Sencho instance (the one you use to manage the fleet) and view its **Settings → Security** page; rules you create there replicate here automatically.
If rules are expected to be present but are not, the most recent sync push may have failed. Check **Settings → Security** on the control (it surfaces sync status per node) or make any small change to a policy to re-trigger a push.
### A remote rejects the sync
A remote rejects a push if its bearer token is invalid or the request does not reach it. Verify the node's API URL and token in **Settings → Nodes** on the control, then use **Test Connection** to confirm the remote is reachable.
### A rule I deleted on the control is still visible on a replica
Sync happens on every write. If you deleted a rule but the replica still shows it, the push probably failed. Make any trivial edit to another rule to trigger a fresh push.
### I need to demote a replica back to a standalone instance
Reset the `fleet_role` entry in the replica's local state (requires direct database access). For most operators, the cleaner approach is to remove the node from the control's **Settings → Nodes** list; the replica remains in replica mode with its last replicated snapshot, which is harmless if it will no longer receive traffic.
+4
View File
@@ -233,3 +233,7 @@ When a post-deploy scan fails for a specific image (for example because Trivy co
### Compare button is disabled
Two completed scans are required to run a comparison. If you have only one scan for an image, trigger a second scan from the Resources Hub (or wait for a scheduled scan), then return to the Scan history page and tick both.
### Scan policies are missing on one of my nodes
Scan policies are managed from the control Sencho instance and replicate to every remote. When you view **Settings → Security** on a remote Sencho (a replica), you will see a banner explaining that rules are managed upstream. See [Fleet Sync](/features/fleet-sync) for how replication works and how to investigate push failures.
@@ -28,8 +28,8 @@ import { toast } from '@/components/ui/toast-store';
import { apiFetch } from '@/lib/api';
import { PaidGate } from '@/components/PaidGate';
import { TierBadge } from '@/components/TierBadge';
import { ShieldCheck, Plus, Trash2, Pencil, Download, RefreshCw, Loader2 } from 'lucide-react';
import type { ScanPolicy, VulnSeverity } from '@/types/security';
import { ShieldCheck, Plus, Trash2, Pencil, Download, RefreshCw, Loader2, Info } from 'lucide-react';
import type { FleetRole, ScanPolicy, VulnSeverity } from '@/types/security';
import { useLicense } from '@/context/LicenseContext';
import { useTrivyStatus } from '@/hooks/useTrivyStatus';
@@ -88,6 +88,8 @@ export function SecuritySection({ isPaid }: { isPaid: boolean }) {
const { status: trivy, updateCheck, refresh: refreshTrivy, refreshUpdateCheck } = useTrivyStatus();
const [trivyBusy, setTrivyBusy] = useState<null | 'install' | 'update' | 'uninstall' | 'auto-update'>(null);
const [uninstallConfirm, setUninstallConfirm] = useState(false);
const [fleetRole, setFleetRole] = useState<FleetRole>('control');
const isReplica = fleetRole === 'replica';
const runTrivyOp = async (
op: 'install' | 'update' | 'uninstall',
@@ -160,6 +162,23 @@ export function SecuritySection({ isPaid }: { isPaid: boolean }) {
else setLoading(false);
}, [isPaid]);
useEffect(() => {
let cancelled = false;
(async () => {
try {
const res = await apiFetch('/fleet/role', { localOnly: true });
if (!res.ok) return;
const data = await res.json();
if (!cancelled && (data?.role === 'control' || data?.role === 'replica')) {
setFleetRole(data.role);
}
} catch {
/* fallback: treat as control if the check fails */
}
})();
return () => { cancelled = true; };
}, []);
const openCreate = () => {
setEditingId(null);
setForm(EMPTY_FORM);
@@ -265,12 +284,30 @@ export function SecuritySection({ isPaid }: { isPaid: boolean }) {
Policies evaluate every post-deploy scan and alert (or block) when severity exceeds the threshold.
</p>
</div>
<Button size="sm" onClick={openCreate}>
<Plus className="w-4 h-4 mr-1.5" />
Add Policy
</Button>
{!isReplica && (
<Button size="sm" onClick={openCreate}>
<Plus className="w-4 h-4 mr-1.5" />
Add Policy
</Button>
)}
</div>
{isReplica && (
<div
role="status"
aria-live="polite"
className="flex items-start gap-2 rounded-lg border border-card-border bg-muted/30 px-4 py-3"
>
<Info className="w-4 h-4 text-muted-foreground shrink-0 mt-0.5" strokeWidth={1.5} aria-hidden="true" />
<div className="text-sm">
<div className="font-medium">Managed by control node</div>
<p className="text-xs text-muted-foreground mt-0.5">
Security policies replicate from the control Sencho instance. View them here for audit; edit them on the control.
</p>
</div>
</div>
)}
<div className="rounded-lg border border-card-border border-t-card-border-top bg-card shadow-card-bevel p-4 space-y-3">
<div className="flex items-center justify-between gap-3">
<div className="flex items-center gap-2 min-w-0">
@@ -384,24 +421,26 @@ export function SecuritySection({ isPaid }: { isPaid: boolean }) {
</Badge>
)}
</div>
<div className="flex items-center gap-1 shrink-0">
<Button
variant="ghost"
size="icon"
className="h-8 w-8"
onClick={() => openEdit(policy)}
>
<Pencil className="w-3.5 h-3.5 text-muted-foreground" strokeWidth={1.5} />
</Button>
<Button
variant="ghost"
size="icon"
className="h-8 w-8 text-destructive/60 hover:bg-destructive hover:text-destructive-foreground"
onClick={() => setDeleteId(policy.id)}
>
<Trash2 className="w-3.5 h-3.5" strokeWidth={1.5} />
</Button>
</div>
{!isReplica && (
<div className="flex items-center gap-1 shrink-0">
<Button
variant="ghost"
size="icon"
className="h-8 w-8"
onClick={() => openEdit(policy)}
>
<Pencil className="w-3.5 h-3.5 text-muted-foreground" strokeWidth={1.5} />
</Button>
<Button
variant="ghost"
size="icon"
className="h-8 w-8 text-destructive/60 hover:bg-destructive hover:text-destructive-foreground"
onClick={() => setDeleteId(policy.id)}
>
<Trash2 className="w-3.5 h-3.5" strokeWidth={1.5} />
</Button>
</div>
)}
</div>
<div className="text-xs text-muted-foreground">
Scope: {policy.stack_pattern ? (
+4
View File
@@ -73,14 +73,18 @@ export interface ScanPolicy {
id: number;
name: string;
node_id: number | null;
node_identity: string;
stack_pattern: string | null;
max_severity: VulnSeverity;
block_on_deploy: number;
enabled: number;
replicated_from_control: number;
created_at: number;
updated_at: number;
}
export type FleetRole = 'control' | 'replica';
export interface ScanCompareVulnerability {
vulnerability_id: string;
pkg_name: string;