feat(fleet-sync): anchor replicas to a control fingerprint (#968)

A replica now binds to the first control that pushes to it. Subsequent
pushes from a different control are rejected with 409
CONTROL_IDENTITY_MISMATCH until an admin explicitly reanchors. Closes
the cross-control hijack window where any node_proxy bearer signed
against the replica's secret could overwrite security policies.

Wire protocol:
- Sender includes a stable 16-hex-char `controlIdentity` derived by
  SHA-256-truncating `system_state.instance_id` (the local UUID written
  once by LicenseService.initialize on first boot). Hostname rotations
  do not flag drift; only a SQLite reset or explicit reanchor breaks
  the binding.
- Receiver caches the fingerprint inside the same transaction that
  applies the row replacement and watermark write. Three states:
  null (fresh install), '' (post-reanchor), '<fingerprint>' (anchored).
- Empty `controlIdentity` is treated as legacy and accepted, so older
  controls keep working during rollout.

New endpoint:
- POST /api/fleet/role/reanchor (admin, requires `{override: true}`):
  clears the cached fingerprint, both `received_pushed_at:*` watermarks,
  and replicated rows of both resources, all in one transaction. The
  static cached fingerprint is also flushed defensively.

Public surface additions:
- `FleetSyncService.getControlIdentity()`: stable fingerprint for the
  outgoing push body.
- `FleetSyncService.reanchor()`: admin-driven anchor reset.
- `ControlIdentityMismatchError`: typed sentinel the route translates
  to 409 with structured body `{error, code, expected, got}`.

Tests:
- 9 new vitest cases covering first-sync persistence, mismatch
  rejection, matching acceptance, empty-incoming back-compat,
  post-reanchor un-anchored state, fingerprint stability, missing
  instance_id fallback, route-level mismatch, route-level reanchor
  with override gating.
- One ordered route-level scenario instead of cross-dependent it()
  blocks so test reordering cannot silently break the suite.
- Full backend suite: 1769 pass / 5 skipped.
This commit is contained in:
Anso
2026-05-07 13:00:21 -04:00
committed by GitHub
parent 27660f622b
commit f3757b43c6
6 changed files with 341 additions and 14 deletions
@@ -187,3 +187,89 @@ describe('POST /api/fleet/sync/:resource pushedAt protocol', () => {
expect(res.body.error).toMatch(/Sync payload too large/);
});
});
describe('POST /api/fleet/sync/:resource control anchor', () => {
// One ordered scenario rather than four cross-dependent it() blocks: anchor,
// then exercise reject / legacy-empty / matching paths against the anchored
// state. Keeps the chronology explicit so reordering or test-isolation
// changes cannot silently break the suite.
it('anchors on first sync, then enforces / accepts / re-allows in order', async () => {
const reanchorRes = await request(app)
.post('/api/fleet/role/reanchor')
.set('Authorization', adminAuthHeader)
.send({ override: true });
expect(reanchorRes.status).toBe(200);
const firstSync = await request(app)
.post('/api/fleet/sync/cve_suppressions')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: [], targetIdentity: 'https://me.example', controlIdentity: 'fingerprint-anchor1' });
expect(firstSync.status).toBe(200);
const mismatch = await request(app)
.post('/api/fleet/sync/cve_suppressions')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: [], targetIdentity: 'https://me.example', controlIdentity: 'fingerprint-different' });
expect(mismatch.status).toBe(409);
expect(mismatch.body.code).toBe('CONTROL_IDENTITY_MISMATCH');
expect(mismatch.body.expected).toBe('fingerprint-anchor1');
expect(mismatch.body.got).toBe('fingerprint-different');
const legacy = await request(app)
.post('/api/fleet/sync/cve_suppressions')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: [], targetIdentity: 'https://me.example' });
expect(legacy.status).toBe(200);
const matching = await request(app)
.post('/api/fleet/sync/cve_suppressions')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: [], targetIdentity: 'https://me.example', controlIdentity: 'fingerprint-anchor1' });
expect(matching.status).toBe(200);
});
});
describe('POST /api/fleet/role/reanchor', () => {
it('returns 401 without auth', async () => {
const res = await request(app).post('/api/fleet/role/reanchor').send({ override: true });
expect(res.status).toBe(401);
});
it('returns 400 without override:true (no accidental reanchor)', async () => {
const res = await request(app)
.post('/api/fleet/role/reanchor')
.set('Authorization', adminAuthHeader)
.send({});
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/override/);
});
it('clears the cached fingerprint and accepts the next push from a different control', async () => {
// Establish a known anchor first so the assertion of "different control
// can now write" actually proves the reanchor cleared something.
const reanchorBefore = await request(app)
.post('/api/fleet/role/reanchor')
.set('Authorization', adminAuthHeader)
.send({ override: true });
expect(reanchorBefore.status).toBe(200);
const anchor = await request(app)
.post('/api/fleet/sync/cve_suppressions')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: [], targetIdentity: 'https://me.example', controlIdentity: 'fingerprint-prior' });
expect(anchor.status).toBe(200);
const reanchor = await request(app)
.post('/api/fleet/role/reanchor')
.set('Authorization', adminAuthHeader)
.send({ override: true });
expect(reanchor.status).toBe(200);
expect(reanchor.body.success).toBe(true);
const next = await request(app)
.post('/api/fleet/sync/cve_suppressions')
.set('Authorization', nodeProxyAuthHeader)
.send({ rows: [], targetIdentity: 'https://me.example', controlIdentity: 'fingerprint-newcontrol' });
expect(next.status).toBe(200);
});
});
@@ -10,6 +10,7 @@ const {
mockGetLocalScanPolicies,
mockGetLocalCveSuppressions,
mockReplaceReplicatedScanPolicies,
mockReplaceReplicatedCveSuppressions,
mockRecordFleetSyncSuccess,
mockRecordFleetSyncFailure,
mockGetSystemState,
@@ -23,6 +24,7 @@ const {
mockGetLocalScanPolicies: vi.fn().mockReturnValue([]),
mockGetLocalCveSuppressions: vi.fn().mockReturnValue([]),
mockReplaceReplicatedScanPolicies: vi.fn(),
mockReplaceReplicatedCveSuppressions: vi.fn(),
mockRecordFleetSyncSuccess: vi.fn(),
mockRecordFleetSyncFailure: vi.fn(),
mockGetSystemState: vi.fn().mockReturnValue(null),
@@ -39,6 +41,7 @@ vi.mock('../services/DatabaseService', () => ({
getLocalScanPolicies: mockGetLocalScanPolicies,
getLocalCveSuppressions: mockGetLocalCveSuppressions,
replaceReplicatedScanPolicies: mockReplaceReplicatedScanPolicies,
replaceReplicatedCveSuppressions: mockReplaceReplicatedCveSuppressions,
recordFleetSyncSuccess: mockRecordFleetSyncSuccess,
recordFleetSyncFailure: mockRecordFleetSyncFailure,
getSystemState: mockGetSystemState,
@@ -372,3 +375,123 @@ describe('FleetSyncService.applyIncomingSync transactional', () => {
expect(watermarkWrites).toHaveLength(0);
});
});
/** Reset the static fingerprint cache between tests; the static field is private. */
function resetControlIdentityCache(): void {
(FleetSyncService as unknown as { cachedControlIdentity: string | null }).cachedControlIdentity = null;
}
describe('FleetSyncService control anchor', () => {
it('persists controlIdentity on first sync (no cached fingerprint)', () => {
mockGetSystemState.mockImplementation((key: string) => {
if (key === 'fleet_control_identity') return null;
return null;
});
FleetSyncService.getInstance().applyIncomingSync(
'scan_policies',
[],
'https://me.example',
undefined,
'fingerprint-aaa',
);
expect(mockSetSystemState).toHaveBeenCalledWith('fleet_control_identity', 'fingerprint-aaa');
});
it('rejects with ControlIdentityMismatchError when cached fingerprint differs', async () => {
const { ControlIdentityMismatchError } = await import('../services/FleetSyncService');
mockGetSystemState.mockImplementation((key: string) => {
if (key === 'fleet_control_identity') return 'fingerprint-original';
return null;
});
expect(() => {
FleetSyncService.getInstance().applyIncomingSync(
'scan_policies',
[],
'https://me.example',
undefined,
'fingerprint-different',
);
}).toThrow(ControlIdentityMismatchError);
});
it('accepts subsequent push when controlIdentity matches the cached fingerprint', () => {
mockGetSystemState.mockImplementation((key: string) => {
if (key === 'fleet_control_identity') return 'fingerprint-aaa';
return null;
});
expect(() => {
FleetSyncService.getInstance().applyIncomingSync(
'scan_policies',
[],
'https://me.example',
undefined,
'fingerprint-aaa',
);
}).not.toThrow();
// Cached identity is not re-written when matching to avoid noisy churn.
const writes = mockSetSystemState.mock.calls.filter((c) => c[0] === 'fleet_control_identity');
expect(writes).toHaveLength(0);
});
it('treats empty incoming controlIdentity as legacy and accepts (back-compat)', () => {
mockGetSystemState.mockImplementation((key: string) => {
if (key === 'fleet_control_identity') return 'fingerprint-aaa';
return null;
});
expect(() => {
FleetSyncService.getInstance().applyIncomingSync(
'scan_policies',
[],
'https://me.example',
undefined,
'',
);
}).not.toThrow();
});
it('treats empty cached fingerprint (post-reanchor) as un-anchored', () => {
mockGetSystemState.mockImplementation((key: string) => {
if (key === 'fleet_control_identity') return '';
return null;
});
expect(() => {
FleetSyncService.getInstance().applyIncomingSync(
'scan_policies',
[],
'https://me.example',
undefined,
'fingerprint-new-control',
);
}).not.toThrow();
expect(mockSetSystemState).toHaveBeenCalledWith('fleet_control_identity', 'fingerprint-new-control');
});
});
describe('FleetSyncService.reanchor', () => {
it('clears cached fingerprint, watermarks, and replicated rows in one transaction', () => {
FleetSyncService.getInstance().reanchor();
expect(mockTransaction).toHaveBeenCalled();
expect(mockSetSystemState).toHaveBeenCalledWith('fleet_control_identity', '');
expect(mockSetSystemState).toHaveBeenCalledWith('received_pushed_at:scan_policies', '');
expect(mockSetSystemState).toHaveBeenCalledWith('received_pushed_at:cve_suppressions', '');
expect(mockReplaceReplicatedScanPolicies).toHaveBeenCalledWith([]);
});
});
describe('FleetSyncService.getControlIdentity', () => {
it('returns a stable 16-hex-char fingerprint derived from instance_id', () => {
mockGetSystemState.mockImplementation((key: string) => (key === 'instance_id' ? 'uuid-abc-def' : null));
resetControlIdentityCache();
const fp1 = FleetSyncService.getControlIdentity();
expect(fp1).toMatch(/^[0-9a-f]{16}$/);
resetControlIdentityCache();
const fp2 = FleetSyncService.getControlIdentity();
expect(fp2).toBe(fp1);
});
it('returns empty string when instance_id is missing', () => {
resetControlIdentityCache();
mockGetSystemState.mockImplementation(() => null);
expect(FleetSyncService.getControlIdentity()).toBe('');
});
});
+37 -1
View File
@@ -4,7 +4,7 @@ import semver from 'semver';
import si from 'systeminformation';
import type Dockerode from 'dockerode';
import { DatabaseService, type Node } from '../services/DatabaseService';
import { FleetSyncService, StaleSyncPushError } from '../services/FleetSyncService';
import { ControlIdentityMismatchError, FleetSyncService, StaleSyncPushError } from '../services/FleetSyncService';
import { MAX_SYNC_ROWS, SYNC_ERROR_CODES } from '../services/fleetSyncConstants';
import { FleetUpdateTrackerService } from '../services/FleetUpdateTrackerService';
import { NodeRegistry } from '../services/NodeRegistry';
@@ -321,6 +321,9 @@ fleetRouter.post('/sync/:resource', authMiddleware, (req: Request, res: Response
const pushedAt = typeof body.pushedAt === 'number' && Number.isFinite(body.pushedAt) && body.pushedAt > 0
? body.pushedAt
: null;
// controlIdentity is optional for back-compat. The receiver anchors to the
// first non-empty fingerprint it sees and rejects mismatches afterward.
const controlIdentity = typeof body.controlIdentity === 'string' ? body.controlIdentity : '';
if (!rows) {
res.status(400).json({ error: 'rows array is required' });
return;
@@ -343,6 +346,7 @@ fleetRouter.post('/sync/:resource', authMiddleware, (req: Request, res: Response
rows,
targetIdentity,
pushedAt ?? undefined,
controlIdentity || undefined,
);
res.json({ success: true, applied: rows.length });
} catch (error) {
@@ -353,11 +357,43 @@ fleetRouter.post('/sync/:resource', authMiddleware, (req: Request, res: Response
});
return;
}
if (error instanceof ControlIdentityMismatchError) {
res.status(409).json({
error: error.message,
code: SYNC_ERROR_CODES.controlIdentityMismatch,
expected: error.expected,
got: error.got,
});
return;
}
console.error('[FleetSync] Failed to apply incoming sync:', error);
res.status(500).json({ error: 'Failed to apply sync' });
}
});
// Reset the control anchor on this replica. An admin must opt in explicitly
// with `{override: true}` because reanchor wipes all replicated rows; the
// next push from a different control will re-populate them. Used when a
// control is permanently rebuilt or replaced and must be re-bound to its
// existing replicas.
fleetRouter.post('/role/reanchor', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
const body = req.body ?? {};
if (body.override !== true) {
res.status(400).json({
error: 'Reanchor requires explicit override. Send { "override": true } to confirm.',
});
return;
}
try {
FleetSyncService.getInstance().reanchor();
res.json({ success: true });
} catch (error) {
console.error('[FleetSync] Reanchor failed:', error);
res.status(500).json({ error: 'Failed to reset control anchor' });
}
});
fleetRouter.get('/sync-status', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requirePaid(req, res)) return;
+92 -13
View File
@@ -1,4 +1,5 @@
import axios, { AxiosError } from 'axios';
import { createHash } from 'crypto';
import { CveSuppression, DatabaseService, Node, ScanPolicy } from './DatabaseService';
import { NodeRegistry } from './NodeRegistry';
import { NotificationService } from './NotificationService';
@@ -50,6 +51,21 @@ export class StaleSyncPushError extends Error {
}
}
/**
* Thrown by `applyIncomingSync` when the incoming `controlIdentity` does not
* match the fingerprint cached on first sync. The replica is anchored to a
* specific control; an operator must re-anchor explicitly via
* POST /api/fleet/role/reanchor before a different control can write. The
* route handler catches this specifically and returns 409
* CONTROL_IDENTITY_MISMATCH.
*/
export class ControlIdentityMismatchError extends Error {
constructor(public readonly expected: string, public readonly got: string) {
super(`Control identity mismatch: replica is anchored to "${expected}", push from "${got}"`);
this.name = 'ControlIdentityMismatchError';
}
}
/**
* FleetSyncService replicates security configuration from a control Sencho
* instance to every managed remote node. Security rules live on the control's
@@ -131,14 +147,34 @@ export class FleetSyncService {
}
/**
* Returns the empty string. The receiver treats empty `controlIdentity`
* as legacy and accepts. A future change will derive a stable fingerprint
* from a system_state key for control-anchor enforcement.
* Stable fingerprint that identifies this control instance to its
* replicas. Derived by SHA-256-truncating `system_state.instance_id`
* (the local UUID generated once on first boot by LicenseService.initialize).
*
* Anchored on the URL path is fragile: an operator who switches a
* control's public hostname would falsely flag drift. Anchoring on the
* persisted instance UUID survives hostname rotations; only a full
* SQLite reset (or explicit reanchor) breaks the binding.
*
* Returns the empty string when `instance_id` is missing (very early
* boot, before LicenseService.initialize has run). The receiver treats
* empty as legacy and accepts.
*/
public static getControlIdentity(): string {
return '';
if (FleetSyncService.cachedControlIdentity !== null) {
return FleetSyncService.cachedControlIdentity;
}
const instanceId = DatabaseService.getInstance().getSystemState('instance_id');
if (!instanceId) {
return '';
}
const fingerprint = createHash('sha256').update(instanceId).digest('hex').slice(0, 16);
FleetSyncService.cachedControlIdentity = fingerprint;
return fingerprint;
}
private static cachedControlIdentity: string | null = null;
private nextPushedAt(): number {
const now = Date.now();
const next = now > FleetSyncService.lastPushedAt ? now : FleetSyncService.lastPushedAt + 1;
@@ -189,26 +225,45 @@ export class FleetSyncService {
}
/**
* Apply a received sync payload on a replica. Runs role flip, identity
* cache, row replacement, watermark write, AND staleness comparison inside
* a single SQLite transaction so a partial-write window cannot leave the
* watermark behind the row state.
* Apply a received sync payload on a replica. Runs control-anchor check,
* staleness comparison, role flip, identity cache, row replacement, and
* watermark write inside a single SQLite transaction so a partial-write
* window cannot leave the watermark behind the row state.
*
* Throws `StaleSyncPushError` (rolling back the transaction) when the
* incoming `pushedAt` is older than the persisted watermark; the route
* handler translates that into 409 STALE_SYNC_PUSH.
* Throws (rolling back the transaction):
* - `ControlIdentityMismatchError` when the cached fingerprint differs
* from the incoming non-empty fingerprint. Route translates to 409
* CONTROL_IDENTITY_MISMATCH; an admin must reanchor explicitly.
* - `StaleSyncPushError` when the incoming pushedAt is older than the
* persisted watermark. Route translates to 409 STALE_SYNC_PUSH.
*
* `pushedAt` is optional for back-compat with legacy controls; when absent,
* the staleness check is skipped and no watermark is written.
* Both `pushedAt` and `controlIdentity` are optional for back-compat with
* legacy controls; absent values skip the corresponding check.
*/
public applyIncomingSync(
resource: FleetResource,
rows: ScanPolicy[] | Array<Omit<CveSuppression, 'id'>>,
targetIdentity: string,
pushedAt?: number,
controlIdentity?: string,
): void {
const db = DatabaseService.getInstance();
db.transaction(() => {
if (controlIdentity && controlIdentity.length > 0) {
// The cached fingerprint has three possible states:
// null fresh install, never received a push.
// '' post-reanchor, explicitly cleared by an admin.
// '<fingerprint>' anchored to a specific control.
// The first two are treated identically as "un-anchored": persist
// the incoming fingerprint. Only a non-empty mismatch rejects.
const cached = db.getSystemState(SYNC_STATE_KEYS.fleetControlIdentity);
if (cached && cached !== controlIdentity) {
throw new ControlIdentityMismatchError(cached, controlIdentity);
}
if (!cached) {
db.setSystemState(SYNC_STATE_KEYS.fleetControlIdentity, controlIdentity);
}
}
if (pushedAt !== undefined && Number.isFinite(pushedAt)) {
const watermarkKey = SYNC_STATE_KEYS.receivedPushedAt(resource);
const previousRaw = db.getSystemState(watermarkKey);
@@ -230,6 +285,30 @@ export class FleetSyncService {
});
}
/**
* Reset the control anchor on this replica so a different control may
* push to it. Clears the cached fingerprint and all replicated rows so
* stale state from the prior control does not leak forward. Watermarks
* also reset to allow the next control's first push to succeed.
*
* Intentionally leaves `fleet_role = 'replica'` and `fleet_self_identity`
* untouched: the node remains a passive receiver, and the next push
* overwrites the cached identity. The static `cachedControlIdentity` is
* also flushed defensively in case this process previously acted as a
* control before being demoted into a replica role.
*/
public reanchor(): void {
const db = DatabaseService.getInstance();
db.transaction(() => {
db.setSystemState(SYNC_STATE_KEYS.fleetControlIdentity, '');
db.setSystemState(SYNC_STATE_KEYS.receivedPushedAt('scan_policies'), '');
db.setSystemState(SYNC_STATE_KEYS.receivedPushedAt('cve_suppressions'), '');
db.replaceReplicatedScanPolicies([]);
db.replaceReplicatedCveSuppressions([]);
});
FleetSyncService.cachedControlIdentity = null;
}
/**
* Chain a push behind any in-flight push for the same node. Different
* nodes still run in parallel via the outer Promise.all.
@@ -59,6 +59,7 @@ export type FleetResource = 'scan_policies' | 'cve_suppressions';
export const SYNC_STATE_KEYS = {
fleetRole: 'fleet_role',
fleetSelfIdentity: 'fleet_self_identity',
fleetControlIdentity: 'fleet_control_identity',
receivedPushedAt: (resource: FleetResource): string => `received_pushed_at:${resource}`,
truncationAlertAt: (resource: FleetResource): string => `fleet_sync_truncation_alert_at:${resource}`,
} as const;
@@ -67,4 +68,5 @@ export const SYNC_STATE_KEYS = {
export const SYNC_ERROR_CODES = {
staleSyncPush: 'STALE_SYNC_PUSH',
payloadTooLarge: 'SYNC_PAYLOAD_TOO_LARGE',
controlIdentityMismatch: 'CONTROL_IDENTITY_MISMATCH',
} as const;
+1
View File
@@ -87,6 +87,7 @@ export const AUDIT_ROUTE_SUMMARIES: Record<string, string> = {
'POST /fleet/snapshots/*/restore': 'Restored fleet backup',
'POST /fleet/nodes/*/update': 'Triggered fleet node update',
'POST /fleet/update-all': 'Triggered fleet-wide update',
'POST /fleet/role/reanchor': 'Re-anchored fleet replica',
// Cloud backup
'PUT /cloud-backup/config': 'Updated cloud backup config',