diff --git a/backend/src/__tests__/database-fleet-sync-sticky.test.ts b/backend/src/__tests__/database-fleet-sync-sticky.test.ts new file mode 100644 index 00000000..154efb4d --- /dev/null +++ b/backend/src/__tests__/database-fleet-sync-sticky.test.ts @@ -0,0 +1,129 @@ +/** + * Pins the DatabaseService sticky-error wiring used by the F-16 fix: + * - setFleetSyncSticky writes the code + expected + got fingerprints. + * - getFleetSyncStickyCode reads them back. + * - getFailedSyncTargets excludes sticky rows (the retry loop must not pick them up). + * - recordFleetSyncSuccess clears the sticky on success (operator reset → push resumes). + * - clearFleetSyncStickyForNode clears every resource for one node id (used by the reset endpoint). + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let nodeId: number; +let siblingId: number; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + const db = DatabaseService.getInstance(); + nodeId = db.addNode({ + name: 'sticky-target', + type: 'remote', + compose_dir: '/app/compose', + is_default: false, + api_url: 'https://sticky.example', + api_token: 'tok', + mode: 'proxy', + }); + siblingId = db.addNode({ + name: 'sticky-sibling', + type: 'remote', + compose_dir: '/app/compose', + is_default: false, + api_url: 'https://sibling-sticky.example', + api_token: 'tok', + mode: 'proxy', + }); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + const db = DatabaseService.getInstance(); + // Wipe stale rows from prior tests in this file so each case starts clean. + db.getDb().prepare('DELETE FROM fleet_sync_status WHERE node_id IN (?, ?)').run(nodeId, siblingId); +}); + +describe('fleet_sync_status sticky-error column', () => { + it('setFleetSyncSticky persists the code and fingerprints', () => { + const db = DatabaseService.getInstance(); + db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', 'aaa111', 'bbb222'); + + const row = db.getFleetSyncStatuses().find( + (s) => s.node_id === nodeId && s.resource === 'scan_policies', + ); + expect(row).toBeDefined(); + expect(row!.sticky_error_code).toBe('CONTROL_IDENTITY_MISMATCH'); + expect(row!.sticky_error_expected).toBe('aaa111'); + expect(row!.sticky_error_got).toBe('bbb222'); + expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBe('CONTROL_IDENTITY_MISMATCH'); + }); + + it('setFleetSyncSticky upserts when no row exists yet', () => { + const db = DatabaseService.getInstance(); + // Pre-state: no row. + expect(db.getFleetSyncStickyCode(nodeId, 'cve_suppressions')).toBeNull(); + + db.setFleetSyncSticky(nodeId, 'cve_suppressions', 'CONTROL_IDENTITY_MISMATCH', null, null); + + expect(db.getFleetSyncStickyCode(nodeId, 'cve_suppressions')).toBe('CONTROL_IDENTITY_MISMATCH'); + }); + + it('getFailedSyncTargets excludes rows where sticky_error_code is set', () => { + const db = DatabaseService.getInstance(); + db.recordFleetSyncFailure(nodeId, 'scan_policies', 'timeout'); + db.recordFleetSyncFailure(siblingId, 'scan_policies', 'connection refused'); + // Mark only `nodeId` as sticky; the sibling stays retriable. + db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', null, null); + + const retriable = db.getFailedSyncTargets('scan_policies', 24 * 60 * 60_000); + const retriableIds = retriable.map((r) => r.node_id); + expect(retriableIds).toContain(siblingId); + expect(retriableIds).not.toContain(nodeId); + }); + + it('recordFleetSyncSuccess clears the sticky flag (operator-reset round-trip)', () => { + const db = DatabaseService.getInstance(); + db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', 'aaa', 'bbb'); + expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBe('CONTROL_IDENTITY_MISMATCH'); + + db.recordFleetSyncSuccess(nodeId, 'scan_policies'); + + expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBeNull(); + const row = db.getFleetSyncStatuses().find( + (s) => s.node_id === nodeId && s.resource === 'scan_policies', + ); + expect(row!.sticky_error_expected).toBeNull(); + expect(row!.sticky_error_got).toBeNull(); + }); + + it('clearFleetSyncStickyForNode clears every resource for one node, leaves siblings untouched', () => { + const db = DatabaseService.getInstance(); + db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', null, null); + db.setFleetSyncSticky(nodeId, 'cve_suppressions', 'CONTROL_IDENTITY_MISMATCH', null, null); + db.setFleetSyncSticky(siblingId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', null, null); + + db.clearFleetSyncStickyForNode(nodeId); + + expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBeNull(); + expect(db.getFleetSyncStickyCode(nodeId, 'cve_suppressions')).toBeNull(); + expect(db.getFleetSyncStickyCode(siblingId, 'scan_policies')).toBe('CONTROL_IDENTITY_MISMATCH'); + }); + + it('migrateFleetSyncStickyError is idempotent (running twice does not error)', () => { + // The constructor already runs the migration once at boot. Manually + // invoke the private method twice via index access to confirm + // tryAddColumn's idempotency contract holds for this migration. + const db = DatabaseService.getInstance() as unknown as { + migrateFleetSyncStickyError: () => void; + }; + expect(() => { + db.migrateFleetSyncStickyError(); + db.migrateFleetSyncStickyError(); + }).not.toThrow(); + }); +}); diff --git a/backend/src/__tests__/fleet-sync-service.test.ts b/backend/src/__tests__/fleet-sync-service.test.ts index 1615fe86..3fc30ac1 100644 --- a/backend/src/__tests__/fleet-sync-service.test.ts +++ b/backend/src/__tests__/fleet-sync-service.test.ts @@ -16,6 +16,8 @@ const { mockInsertAuditLog, mockRecordFleetSyncSuccess, mockRecordFleetSyncFailure, + mockSetFleetSyncSticky, + mockGetFleetSyncStickyCode, mockGetSystemState, mockSetSystemState, mockTransaction, @@ -33,6 +35,8 @@ const { mockInsertAuditLog: vi.fn(), mockRecordFleetSyncSuccess: vi.fn(), mockRecordFleetSyncFailure: vi.fn(), + mockSetFleetSyncSticky: vi.fn(), + mockGetFleetSyncStickyCode: vi.fn().mockReturnValue(null), mockGetSystemState: vi.fn().mockReturnValue(null), mockSetSystemState: vi.fn(), mockTransaction: vi.fn().mockImplementation((fn: () => unknown) => fn()), @@ -53,6 +57,8 @@ vi.mock('../services/DatabaseService', () => ({ insertAuditLog: mockInsertAuditLog, recordFleetSyncSuccess: mockRecordFleetSyncSuccess, recordFleetSyncFailure: mockRecordFleetSyncFailure, + setFleetSyncSticky: mockSetFleetSyncSticky, + getFleetSyncStickyCode: mockGetFleetSyncStickyCode, getSystemState: mockGetSystemState, setSystemState: mockSetSystemState, transaction: mockTransaction, @@ -90,6 +96,7 @@ import { FleetSyncService, LOCAL_IDENTITY_SENTINEL } from '../services/FleetSync beforeEach(() => { vi.clearAllMocks(); mockGetSystemState.mockReturnValue(null); + mockGetFleetSyncStickyCode.mockReturnValue(null); }); describe('FleetSyncService.getRole', () => { @@ -594,3 +601,102 @@ describe('FleetSyncService.formatError redaction', () => { expect(failure[2]).toContain('[redacted-jwt]'); }); }); + +describe('FleetSyncService CONTROL_IDENTITY_MISMATCH sticky handling', () => { + function makeMismatchError(expected: string, got: string) { + return async () => { + const { AxiosError } = await import('axios'); + const err = new AxiosError('Request failed with status code 409'); + (err as unknown as { response: unknown }).response = { + status: 409, + statusText: 'Conflict', + data: { + error: `Control identity mismatch: replica is anchored to "${expected}", push from "${got}"`, + code: 'CONTROL_IDENTITY_MISMATCH', + expected, + got, + }, + }; + throw err; + }; + } + + it('sets the sticky flag carrying the expected/got fingerprints on first mismatch', async () => { + mockGetNodes.mockReturnValue([ + { id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' }, + ]); + mockGetLocalScanPolicies.mockReturnValue([]); + mockGetFleetSyncStickyCode.mockReturnValue(null); + mockAxiosPost.mockImplementation(makeMismatchError('cb45a2eff9db81d8', '555f8d1f7e7e71e3')); + + await FleetSyncService.getInstance().pushResource('scan_policies'); + + expect(mockRecordFleetSyncFailure).toHaveBeenCalledTimes(1); + expect(mockSetFleetSyncSticky).toHaveBeenCalledTimes(1); + expect(mockSetFleetSyncSticky).toHaveBeenCalledWith( + 7, + 'scan_policies', + 'CONTROL_IDENTITY_MISMATCH', + 'cb45a2eff9db81d8', + '555f8d1f7e7e71e3', + ); + }); + + it('short-circuits subsequent pushes when sticky is already set; no HTTP call, no failure record', async () => { + mockGetNodes.mockReturnValue([ + { id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' }, + ]); + mockGetLocalScanPolicies.mockReturnValue([]); + // Sticky already set from a prior push. + mockGetFleetSyncStickyCode.mockReturnValue('CONTROL_IDENTITY_MISMATCH'); + + await FleetSyncService.getInstance().pushResource('scan_policies'); + + expect(mockAxiosPost).not.toHaveBeenCalled(); + expect(mockRecordFleetSyncFailure).not.toHaveBeenCalled(); + expect(mockRecordFleetSyncSuccess).not.toHaveBeenCalled(); + expect(mockSetFleetSyncSticky).not.toHaveBeenCalled(); + }); + + it('tolerates a missing expected/got payload (passes null through)', async () => { + mockGetNodes.mockReturnValue([ + { id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' }, + ]); + mockGetLocalScanPolicies.mockReturnValue([]); + mockGetFleetSyncStickyCode.mockReturnValue(null); + mockAxiosPost.mockImplementation(async () => { + const { AxiosError } = await import('axios'); + const err = new AxiosError('Request failed with status code 409'); + (err as unknown as { response: unknown }).response = { + status: 409, + statusText: 'Conflict', + data: { error: 'mismatch', code: 'CONTROL_IDENTITY_MISMATCH' }, + }; + throw err; + }); + + await FleetSyncService.getInstance().pushResource('scan_policies'); + + expect(mockSetFleetSyncSticky).toHaveBeenCalledWith( + 7, + 'scan_policies', + 'CONTROL_IDENTITY_MISMATCH', + null, + null, + ); + }); + + it('does not set sticky for non-mismatch failures (network errors, 500s)', async () => { + mockGetNodes.mockReturnValue([ + { id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' }, + ]); + mockGetLocalScanPolicies.mockReturnValue([]); + mockGetFleetSyncStickyCode.mockReturnValue(null); + mockAxiosPost.mockRejectedValue(new Error('ECONNREFUSED')); + + await FleetSyncService.getInstance().pushResource('scan_policies'); + + expect(mockRecordFleetSyncFailure).toHaveBeenCalledTimes(1); + expect(mockSetFleetSyncSticky).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/__tests__/nodes-fleet-sync-reset-anchor.test.ts b/backend/src/__tests__/nodes-fleet-sync-reset-anchor.test.ts new file mode 100644 index 00000000..b0934853 --- /dev/null +++ b/backend/src/__tests__/nodes-fleet-sync-reset-anchor.test.ts @@ -0,0 +1,158 @@ +/** + * Tests for POST /api/nodes/:id/fleet-sync/reset-anchor (F-16 fix). + * + * The endpoint proxies the peer's reanchor endpoint and clears every + * sticky-error row for the node on success. Covers: + * - happy path: 200 from peer → sticky rows cleared, 200 returned. + * - peer 401/403 → 502 with helpful message. + * - peer unreachable → 504. + * - missing/non-proxy node → 400. + * - non-paid tier → 403. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; + +let tmpDir: string; +let app: import('express').Express; +let authHeader: string; +let peerNodeId: number; + +const originalFetch = globalThis.fetch; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); + authHeader = `Bearer ${token}`; + + // Seed a proxy-mode remote node and a sticky row for it. Tests then drive + // the route handler and assert side effects on the test DB. + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + peerNodeId = db.addNode({ + name: 'sticky-peer', + type: 'remote', + compose_dir: '/app/compose', + is_default: false, + api_url: 'http://192.168.1.99:1852', + api_token: 'peer-token', + mode: 'proxy', + }); + db.setFleetSyncSticky( + peerNodeId, + 'scan_policies', + 'CONTROL_IDENTITY_MISMATCH', + 'cb45a2eff9db81d8', + '555f8d1f7e7e71e3', + ); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + cleanupTestDb(tmpDir); +}); + +beforeEach(async () => { + vi.restoreAllMocks(); + globalThis.fetch = originalFetch; + // Re-establish the paid-tier spy after restoreAllMocks. Individual tests + // can override with `mockReturnValue('community')` to exercise the tier + // gate's deny path. + const { LicenseService } = await import('../services/LicenseService'); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); +}); + +describe('POST /api/nodes/:id/fleet-sync/reset-anchor', () => { + it('proxies to the peer reanchor, clears sticky rows, returns 200', async () => { + const fetchSpy = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ success: true }), { status: 200 }), + ); + globalThis.fetch = fetchSpy as unknown as typeof fetch; + + const res = await request(app) + .post(`/api/nodes/${peerNodeId}/fleet-sync/reset-anchor`) + .set('Authorization', authHeader) + .send({}); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(fetchSpy).toHaveBeenCalledTimes(1); + const [url, init] = fetchSpy.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('http://192.168.1.99:1852/api/fleet/role/reanchor'); + expect(init.method).toBe('POST'); + expect((init.headers as Record).Authorization).toBe('Bearer peer-token'); + expect(JSON.parse(init.body as string)).toEqual({ override: true }); + + const { DatabaseService } = await import('../services/DatabaseService'); + const sticky = DatabaseService.getInstance().getFleetSyncStickyCode(peerNodeId, 'scan_policies'); + expect(sticky).toBeNull(); + }); + + it('returns 502 with a helpful message when peer responds 401', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + DatabaseService.getInstance().setFleetSyncSticky( + peerNodeId, 'cve_suppressions', 'CONTROL_IDENTITY_MISMATCH', 'aaa', 'bbb', + ); + const fetchSpy = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ error: 'Admin access required.' }), { status: 401 }), + ); + globalThis.fetch = fetchSpy as unknown as typeof fetch; + + const res = await request(app) + .post(`/api/nodes/${peerNodeId}/fleet-sync/reset-anchor`) + .set('Authorization', authHeader) + .send({}); + + expect(res.status).toBe(502); + expect(res.body.error).toMatch(/Admin access required/); + // Sticky rows must remain set so the operator can retry. + const sticky = DatabaseService.getInstance().getFleetSyncStickyCode(peerNodeId, 'cve_suppressions'); + expect(sticky).toBe('CONTROL_IDENTITY_MISMATCH'); + }); + + it('returns 504 when the peer is unreachable', async () => { + const fetchSpy = vi.fn().mockRejectedValue(new Error('connect ECONNREFUSED')); + globalThis.fetch = fetchSpy as unknown as typeof fetch; + + const res = await request(app) + .post(`/api/nodes/${peerNodeId}/fleet-sync/reset-anchor`) + .set('Authorization', authHeader) + .send({}); + + expect(res.status).toBe(504); + expect(res.body.error).toMatch(/unreachable/i); + }); + + it('returns 400 for a local node', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const local = DatabaseService.getInstance().getNodes().find((n) => n.type === 'local'); + expect(local).toBeTruthy(); + const res = await request(app) + .post(`/api/nodes/${local!.id}/fleet-sync/reset-anchor`) + .set('Authorization', authHeader) + .send({}); + expect(res.status).toBe(400); + }); + + it('returns 404 for an unknown node id', async () => { + const res = await request(app) + .post('/api/nodes/9999/fleet-sync/reset-anchor') + .set('Authorization', authHeader) + .send({}); + expect(res.status).toBe(404); + }); + + it('returns 403 (PAID_REQUIRED) when the license is community-tier', async () => { + const { LicenseService } = await import('../services/LicenseService'); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + + const res = await request(app) + .post(`/api/nodes/${peerNodeId}/fleet-sync/reset-anchor`) + .set('Authorization', authHeader) + .send({}); + expect(res.status).toBe(403); + expect(res.body.code).toBe('PAID_REQUIRED'); + }); +}); diff --git a/backend/src/routes/nodes.ts b/backend/src/routes/nodes.ts index e051a4dc..77adc7b9 100644 --- a/backend/src/routes/nodes.ts +++ b/backend/src/routes/nodes.ts @@ -4,7 +4,7 @@ import crypto from 'crypto'; import { authMiddleware } from '../middleware/auth'; import { requirePermission } from '../middleware/permissions'; import { rejectApiTokenScope } from '../middleware/apiTokenScope'; -import { requireAdmiral } from '../middleware/tierGates'; +import { requireAdmin, requireAdmiral, requirePaid } from '../middleware/tierGates'; import { enrollmentLimiter } from '../middleware/rateLimiters'; import { DatabaseService } from '../services/DatabaseService'; import { NodeRegistry } from '../services/NodeRegistry'; @@ -354,6 +354,93 @@ nodesRouter.post('/:id/uncordon', (req: Request, res: Response) => { } }); +/** + * Reset the FleetSync control anchor on a remote peer. + * + * Proxies POST /api/fleet/role/reanchor to the peer using its stored + * Bearer token. A successful reanchor clears every sticky-error row for + * this node so the next push (event-driven or via the 5-minute retry + * service) re-attempts cleanly and the peer accepts the central's + * fingerprint as the new anchor. + * + * Surfaces UI affordance for the F-16 audit (mesh-e2e-2026-05-17.md): + * when a peer was previously enrolled by a different central, FleetSync + * keeps 409'ing every reconcile tick; the sticky flag halts retries and + * this endpoint is the single one-click recovery for the operator. + */ +nodesRouter.post('/:id/fleet-sync/reset-anchor', async (req: Request, res: Response) => { + if (rejectApiTokenScope(req, res, NODE_SCOPE_MESSAGE)) return; + const nodeIdParam = req.params.id as string; + if (!requirePermission(req, res, 'node:manage', 'node', nodeIdParam)) return; + if (!requirePaid(req, res)) return; + // Reset-anchor is symmetric with `/api/fleet/sync-status` (admin-only). + // Keeping read and write gated at the same role avoids a banner-invisible-to-the-actor + // gap where a node-admin could call reset without ever seeing why. + if (!requireAdmin(req, res)) return; + try { + const id = parseInt(nodeIdParam, 10); + if (!Number.isFinite(id) || id <= 0) { + res.status(400).json({ error: 'Invalid node id' }); + return; + } + const node = DatabaseService.getInstance().getNode(id); + if (!node) { + res.status(404).json({ error: 'Node not found' }); + return; + } + if (node.type !== 'remote' || node.mode !== 'proxy') { + res.status(400).json({ error: 'Reset anchor only applies to proxy-mode remote nodes' }); + return; + } + if (!node.api_url || !node.api_token) { + res.status(400).json({ error: 'Node is missing api_url or api_token' }); + return; + } + + const baseUrl = node.api_url.replace(/\/$/, ''); + let peerResponse: globalThis.Response; + try { + peerResponse = await fetch(`${baseUrl}/api/fleet/role/reanchor`, { + method: 'POST', + headers: { + Authorization: `Bearer ${node.api_token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ override: true }), + signal: AbortSignal.timeout(15_000), + }); + } catch (networkErr) { + const message = getErrorMessage(networkErr, 'Failed to reach peer'); + console.warn(`[Nodes] Reset anchor unreachable for node ${id}: ${message}`); + res.status(504).json({ error: `Peer unreachable: ${message}` }); + return; + } + + if (!peerResponse.ok) { + const status = peerResponse.status; + const body = await peerResponse.json().catch(() => ({})); + const peerError = (body as { error?: string })?.error + ?? `Peer returned HTTP ${status}`; + if (status === 401 || status === 403) { + console.warn(`[Nodes] Reset anchor rejected by peer ${id}: ${peerError}`); + res.status(502).json({ + error: `Peer rejected the reanchor request: ${peerError}. The node's API token may need to be regenerated.`, + }); + return; + } + res.status(502).json({ error: peerError }); + return; + } + + DatabaseService.getInstance().clearFleetSyncStickyForNode(id); + console.log(`[Nodes] Fleet-sync anchor reset on node ${id} ("${node.name}")`); + res.json({ success: true }); + } catch (error: unknown) { + console.error('Failed to reset fleet-sync anchor:', error); + res.status(500).json({ error: getErrorMessage(error, 'Failed to reset fleet-sync anchor') }); + } +}); + nodesRouter.post('/:id/test', async (req: Request, res: Response) => { try { const id = parseInt(req.params.id as string); diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 9c1e2ab6..6af63855 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -555,6 +555,9 @@ export interface FleetSyncStatus { last_success_at: number | null; last_failure_at: number | null; last_error: string | null; + sticky_error_code: string | null; + sticky_error_expected: string | null; + sticky_error_got: string | null; } export interface CveSuppression { @@ -659,6 +662,7 @@ export class DatabaseService { this.migrateAddNodeCordonFields(); this.migrateAddBlueprintPinnedNode(); this.migrateAutoHealNodeId(); + this.migrateFleetSyncStickyError(); // Reset the cache once at end of constructor in case any migration // populated it via getGlobalSettings() and a subsequent migration @@ -1581,6 +1585,12 @@ export class DatabaseService { this.tryAddColumn('blueprints', 'pinned_node_id', 'INTEGER'); } + private migrateFleetSyncStickyError(): void { + this.tryAddColumn('fleet_sync_status', 'sticky_error_code', 'TEXT'); + this.tryAddColumn('fleet_sync_status', 'sticky_error_expected', 'TEXT'); + this.tryAddColumn('fleet_sync_status', 'sticky_error_got', 'TEXT'); + } + private migrateAutoHealNodeId(): void { const markerKey = 'migration_auto_heal_node_scope_v1'; const markerDone = this.getGlobalSettings()[markerKey] === '1'; @@ -3931,11 +3941,15 @@ export class DatabaseService { const now = Date.now(); this.db .prepare( - `INSERT INTO fleet_sync_status (node_id, resource, last_success_at, last_failure_at, last_error) - VALUES (?, ?, ?, NULL, NULL) + `INSERT INTO fleet_sync_status (node_id, resource, last_success_at, last_failure_at, last_error, + sticky_error_code, sticky_error_expected, sticky_error_got) + VALUES (?, ?, ?, NULL, NULL, NULL, NULL, NULL) ON CONFLICT(node_id, resource) DO UPDATE SET last_success_at = excluded.last_success_at, - last_error = NULL`, + last_error = NULL, + sticky_error_code = NULL, + sticky_error_expected = NULL, + sticky_error_got = NULL`, ) .run(nodeId, resource, now); } @@ -3953,6 +3967,64 @@ export class DatabaseService { .run(nodeId, resource, now, error); } + /** + * Mark a (node, resource) pair as having hit a non-retriable failure. The + * retry service skips sticky rows and the push paths short-circuit before + * any HTTP call. The first such failure still records `last_failure_at` + + * `last_error` via `recordFleetSyncFailure`; the sticky write is additive. + * + * `expected` and `got` carry the fingerprints from a 409 + * CONTROL_IDENTITY_MISMATCH response so the UI can render + * "anchored to , this central is " without parsing the + * error string. + */ + public setFleetSyncSticky( + nodeId: number, + resource: string, + code: string, + expected: string | null, + got: string | null, + ): void { + this.db + .prepare( + `INSERT INTO fleet_sync_status (node_id, resource, sticky_error_code, + sticky_error_expected, sticky_error_got) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(node_id, resource) DO UPDATE SET + sticky_error_code = excluded.sticky_error_code, + sticky_error_expected = excluded.sticky_error_expected, + sticky_error_got = excluded.sticky_error_got`, + ) + .run(nodeId, resource, code, expected, got); + } + + public getFleetSyncStickyCode(nodeId: number, resource: string): string | null { + const row = this.db + .prepare( + `SELECT sticky_error_code FROM fleet_sync_status + WHERE node_id = ? AND resource = ?`, + ) + .get(nodeId, resource) as { sticky_error_code: string | null } | undefined; + return row?.sticky_error_code ?? null; + } + + /** + * Clear every sticky-error row for one node id. Used by the + * reset-anchor endpoint after the peer has acknowledged a reanchor; the + * next push attempt re-tries normally. + */ + public clearFleetSyncStickyForNode(nodeId: number): void { + this.db + .prepare( + `UPDATE fleet_sync_status + SET sticky_error_code = NULL, + sticky_error_expected = NULL, + sticky_error_got = NULL + WHERE node_id = ?`, + ) + .run(nodeId); + } + public getFailedSyncTargets(resource: string, maxAgeMs: number): FleetSyncStatus[] { const cutoff = Date.now() - maxAgeMs; return this.db @@ -3960,7 +4032,8 @@ export class DatabaseService { `SELECT * FROM fleet_sync_status WHERE resource = ? AND (last_failure_at IS NOT NULL AND last_failure_at > ?) - AND (last_success_at IS NULL OR last_success_at < last_failure_at)`, + AND (last_success_at IS NULL OR last_success_at < last_failure_at) + AND sticky_error_code IS NULL`, ) .all(resource, cutoff) as FleetSyncStatus[]; } diff --git a/backend/src/services/FleetSyncService.ts b/backend/src/services/FleetSyncService.ts index eb103bf3..32cd8506 100644 --- a/backend/src/services/FleetSyncService.ts +++ b/backend/src/services/FleetSyncService.ts @@ -449,12 +449,38 @@ export class FleetSyncService { return next; } + /** + * Concurrency note: the sticky-set in the CONTROL_IDENTITY_MISMATCH catch + * branch is best-effort against an operator-initiated reset that lands + * during a push's HTTP round-trip. The reset endpoint clears + * `sticky_error_code` to NULL; if this push's 409 arrives after that + * clear, it will re-pin the row. The operator clicks Reset again. The + * window is bounded by one HTTP round-trip per resource; no lock or + * generation counter is justified. + */ private async executePushToNode( node: Node & { id: number }, resource: FleetResource, partial: Omit, ): Promise { const db = DatabaseService.getInstance(); + + // Sticky-error short-circuit. When a previous push hit a non-retriable + // failure (today: CONTROL_IDENTITY_MISMATCH), every subsequent push + // would re-issue the same 409 and re-spam the log every 5 minutes. The + // sticky flag is cleared by either (a) the operator resetting the + // anchor via POST /api/nodes/:id/fleet-sync/reset-anchor, or (b) a + // successful push to the same node (recordFleetSyncSuccess clears + // it). Until then, skip the HTTP call entirely. + if (db.getFleetSyncStickyCode(node.id, resource)) { + if (isDebugEnabled()) { + console.debug( + `[FleetSync:debug] Skipping ${resource} push to "${node.name}": sticky error blocks retries.`, + ); + } + return; + } + const apiUrl = node.api_url ?? ''; const baseUrl = apiUrl.replace(/\/$/, ''); const payload: FleetSyncPayload = { ...partial, targetIdentity: apiUrl }; @@ -479,7 +505,7 @@ export class FleetSyncService { // healthy; suppress the failure record so it does not surface as // an alert in the sync-status panel. if (err instanceof AxiosError && err.response?.status === 409) { - const data = err.response.data as { code?: string } | undefined; + const data = err.response.data as { code?: string; expected?: string; got?: string } | undefined; if (data?.code === SYNC_ERROR_CODES.staleSyncPush) { if (isDebugEnabled()) { console.debug( @@ -488,6 +514,26 @@ export class FleetSyncService { } return; } + // CONTROL_IDENTITY_MISMATCH is permanent until the operator + // explicitly resets the peer anchor. Record the failure once + // (last_error + last_failure_at) plus a sticky flag so the + // retry service and event-driven push path skip subsequent + // attempts. + if (data?.code === SYNC_ERROR_CODES.controlIdentityMismatch) { + const message = this.formatError(err); + console.warn( + `[FleetSync] Failed to push ${resource} to "${node.name}" (${baseUrl}): ${message}`, + ); + db.recordFleetSyncFailure(node.id, resource, message); + db.setFleetSyncSticky( + node.id, + resource, + SYNC_ERROR_CODES.controlIdentityMismatch, + typeof data.expected === 'string' ? data.expected : null, + typeof data.got === 'string' ? data.got : null, + ); + return; + } } const message = this.formatError(err); console.warn( diff --git a/frontend/src/components/NodeManager.tsx b/frontend/src/components/NodeManager.tsx index fc7e301f..93f65d35 100644 --- a/frontend/src/components/NodeManager.tsx +++ b/frontend/src/components/NodeManager.tsx @@ -9,7 +9,7 @@ import { Badge } from './ui/badge'; import { Separator } from './ui/separator'; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from './ui/table'; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from './ui/tooltip'; -import { Plus, Trash2, Wifi, WifiOff, Star, Pencil, Monitor, Globe, Copy, KeyRound, Check, Calendar, RefreshCw, Terminal } from 'lucide-react'; +import { AlertTriangle, Plus, Trash2, Wifi, WifiOff, Star, Pencil, Monitor, Globe, Copy, KeyRound, Check, Calendar, RefreshCw, Terminal } from 'lucide-react'; import { formatTimeUntil, formatTimeAgo } from '@/lib/relativeTime'; import { SettingsPrimaryButton } from './settings/SettingsActions'; import { useMastheadStats } from './settings/MastheadStatsContext'; @@ -17,6 +17,8 @@ import { NodeLabelPicker } from './blueprints/NodeLabelPicker'; import { useLicense } from '@/context/LicenseContext'; import { useAuth } from '@/context/AuthContext'; import { useNodeActions, type NodeTestInfo } from './nodes/useNodeActions'; +import { useFleetSyncStatus } from '@/hooks/useFleetSyncStatus'; +import { resetFleetSyncAnchor, STICKY_CONTROL_IDENTITY_MISMATCH } from '@/lib/fleetSyncApi'; interface NodeSchedulingSummary { active_tasks: number; @@ -60,6 +62,48 @@ export function NodeManager() { onTestResult: (result) => setTestResult(result), }); + const { statuses: syncStatuses, refresh: refreshSyncStatuses } = useFleetSyncStatus(); + const [resettingAnchor, setResettingAnchor] = useState(null); + + // Per-node aggregate of CONTROL_IDENTITY_MISMATCH sticky errors. All resources + // for one peer share the same root cause (the peer's cached fingerprint), so + // collapse to one entry per node id and surface a single banner. + const anchorMismatches = useMemo(() => { + const byNode = new Map(); + for (const row of syncStatuses) { + if (row.sticky_error_code !== STICKY_CONTROL_IDENTITY_MISMATCH) continue; + const existing = byNode.get(row.node_id); + if (existing) { + existing.resources.push(row.resource); + if (!existing.expected && row.sticky_error_expected) existing.expected = row.sticky_error_expected; + if (!existing.got && row.sticky_error_got) existing.got = row.sticky_error_got; + } else { + byNode.set(row.node_id, { + expected: row.sticky_error_expected, + got: row.sticky_error_got, + resources: [row.resource], + }); + } + } + return Array.from(byNode.entries()).map(([nodeId, agg]) => { + const node = nodes.find((n) => n.id === nodeId); + return { nodeId, node, ...agg }; + }).filter((entry) => entry.node !== undefined); + }, [syncStatuses, nodes]); + + const handleResetAnchor = async (nodeId: number) => { + setResettingAnchor(nodeId); + try { + await resetFleetSyncAnchor(nodeId); + toast.success('Anchor reset. Security policy sync will resume on the next push.'); + refreshSyncStatuses(); + } catch (error) { + toast.error((error as Error).message || 'Failed to reset anchor on peer'); + } finally { + setResettingAnchor(null); + } + }; + const fetchSchedulingSummary = useCallback(async () => { try { const res = await apiFetch('/nodes/scheduling-summary', { localOnly: true }); @@ -188,6 +232,51 @@ export function NodeManager() { )} + {/* Sync issues: surfaces FleetSync sticky errors (currently CONTROL_IDENTITY_MISMATCH). */} + {anchorMismatches.length > 0 && ( +
+ {anchorMismatches.map(({ nodeId, node, expected, got, resources }) => ( +
+ +
+
+ Node "{node?.name ?? `id ${nodeId}`}" is anchored to another central +
+
+ Security policy sync is paused for {resources.join(', ')}. + {expected && got && ( + <> This peer is anchored to {expected}; this central is {got}. + )} + {' '}Reset the anchor on the peer to resume sync, or remove the node from this fleet. +
+
+ + {node && !node.is_default && ( + + )} +
+
+
+ ))} +
+ )} + {/* Nodes Table */}
diff --git a/frontend/src/components/fleet/FleetConfiguration.tsx b/frontend/src/components/fleet/FleetConfiguration.tsx index 62d0a504..912fa10b 100644 --- a/frontend/src/components/fleet/FleetConfiguration.tsx +++ b/frontend/src/components/fleet/FleetConfiguration.tsx @@ -1,12 +1,15 @@ -import { useState, useEffect, useCallback } from 'react'; +import { useState, useEffect, useCallback, useMemo } from 'react'; import { apiFetch } from '@/lib/api'; import { formatCount } from '@/lib/utils'; import { Card, CardContent } from '@/components/ui/card'; import { Badge } from '@/components/ui/badge'; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip'; import { - Bell, Zap, Shield, HardDrive, WifiOff, CheckCircle2, + Bell, Zap, Shield, HardDrive, WifiOff, CheckCircle2, RefreshCw, } from 'lucide-react'; import { useLicense } from '@/context/LicenseContext'; +import { useFleetSyncStatus } from '@/hooks/useFleetSyncStatus'; +import { STICKY_CONTROL_IDENTITY_MISMATCH, type FleetSyncStatus } from '@/lib/fleetSyncApi'; import type { ConfigurationStatusPayload } from '@/components/dashboard'; interface FleetNodeConfiguration { @@ -31,7 +34,69 @@ function SummaryRow({ icon: Icon, label, value }: { ); } -function NodeCard({ node, isPaid }: { node: FleetNodeConfiguration; isPaid: boolean }) { +type PolicySyncState = + | { kind: 'in_sync' } + | { kind: 'degraded'; lastError: string | null } + | { kind: 'paused' }; + +function derivePolicySyncState(rows: FleetSyncStatus[]): PolicySyncState | null { + if (rows.length === 0) return null; + for (const row of rows) { + if (row.sticky_error_code === STICKY_CONTROL_IDENTITY_MISMATCH) { + return { kind: 'paused' }; + } + } + let degradedError: string | null = null; + let hasSuccess = false; + for (const row of rows) { + if (row.last_success_at !== null) hasSuccess = true; + if ( + row.last_failure_at !== null + && (row.last_success_at === null || row.last_failure_at > row.last_success_at) + ) { + degradedError = row.last_error; + } + } + if (degradedError !== null) return { kind: 'degraded', lastError: degradedError }; + if (hasSuccess) return { kind: 'in_sync' }; + return null; +} + +function PolicySyncRow({ state }: { state: PolicySyncState }) { + if (state.kind === 'in_sync') { + return ; + } + const tooltip = state.kind === 'paused' + ? 'Anchored to another central. Open Settings → Nodes to reset the anchor or remove the node.' + : (state.lastError ?? 'Last push to this node failed.'); + return ( +
+ + Policy sync + + + + + {state.kind === 'paused' ? 'paused' : 'degraded'} + + + {tooltip} + + +
+ ); +} + +function NodeCard({ node, isPaid, policySyncState }: { + node: FleetNodeConfiguration; + isPaid: boolean; + policySyncState: PolicySyncState | null; +}) { const isRemote = node.type === 'remote'; if (!node.configuration) { return ( @@ -102,6 +167,7 @@ function NodeCard({ node, isPaid }: { node: FleetNodeConfiguration; isPaid: bool )} + {policySyncState && } @@ -110,10 +176,26 @@ function NodeCard({ node, isPaid }: { node: FleetNodeConfiguration; isPaid: bool export function FleetConfiguration() { const { isPaid } = useLicense(); + const { statuses: syncStatuses } = useFleetSyncStatus(); const [nodes, setNodes] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); + const syncStateByNode = useMemo(() => { + const byNode = new Map(); + for (const row of syncStatuses) { + const list = byNode.get(row.node_id); + if (list) list.push(row); + else byNode.set(row.node_id, [row]); + } + const out = new Map(); + for (const [nodeId, rows] of byNode) { + const state = derivePolicySyncState(rows); + if (state) out.set(nodeId, state); + } + return out; + }, [syncStatuses]); + const fetchData = useCallback(async () => { try { const res = await apiFetch('/fleet/configuration', { localOnly: true }); @@ -170,7 +252,14 @@ export function FleetConfiguration() { return (
- {nodes.map(node => )} + {nodes.map(node => ( + + ))}
); } diff --git a/frontend/src/hooks/useFleetSyncStatus.ts b/frontend/src/hooks/useFleetSyncStatus.ts new file mode 100644 index 00000000..e6835b93 --- /dev/null +++ b/frontend/src/hooks/useFleetSyncStatus.ts @@ -0,0 +1,45 @@ +import { useCallback, useEffect, useState } from 'react'; +import { useLicense } from '@/context/LicenseContext'; +import { visibilityInterval } from '@/lib/utils'; +import { fetchFleetSyncStatuses, type FleetSyncStatus } from '@/lib/fleetSyncApi'; + +const REFRESH_INTERVAL_MS = 30_000; + +/** + * Polls `/api/fleet/sync-status` and exposes the rows plus a manual + * `refresh()`. Skips fetching for community-tier users since the endpoint is + * paid-tier-gated; the hook returns an empty array in that case so consumers + * can render the `!isPaid` branch without conditionals. + */ +export function useFleetSyncStatus(): { + statuses: FleetSyncStatus[]; + loading: boolean; + refresh: () => void; +} { + const { isPaid } = useLicense(); + const [statuses, setStatuses] = useState([]); + const [loading, setLoading] = useState(true); + + const refresh = useCallback(() => { + if (!isPaid) { + setStatuses([]); + setLoading(false); + return; + } + fetchFleetSyncStatuses() + .then((rows) => setStatuses(rows)) + .catch((err) => { + // Stale data stays visible to avoid flicker; log so the failure isn't completely silent. + console.warn('[FleetSync] sync-status fetch failed:', err); + }) + .finally(() => setLoading(false)); + }, [isPaid]); + + useEffect(() => { + refresh(); + if (!isPaid) return undefined; + return visibilityInterval(refresh, REFRESH_INTERVAL_MS); + }, [isPaid, refresh]); + + return { statuses, loading, refresh }; +} diff --git a/frontend/src/lib/fleetSyncApi.ts b/frontend/src/lib/fleetSyncApi.ts new file mode 100644 index 00000000..e7a7d88f --- /dev/null +++ b/frontend/src/lib/fleetSyncApi.ts @@ -0,0 +1,43 @@ +import { apiFetch } from '@/lib/api'; + +/** Wire shape of `GET /api/fleet/sync-status`. Mirrors backend `FleetSyncStatus`. */ +export interface FleetSyncStatus { + node_id: number; + resource: string; + last_success_at: number | null; + last_failure_at: number | null; + last_error: string | null; + /** Non-null when retries are paused (today: 'CONTROL_IDENTITY_MISMATCH'). */ + sticky_error_code: string | null; + /** Fingerprint the peer is anchored to (from 409 body); null when not applicable. */ + sticky_error_expected: string | null; + /** Fingerprint this central pushed (from 409 body); null when not applicable. */ + sticky_error_got: string | null; +} + +export const STICKY_CONTROL_IDENTITY_MISMATCH = 'CONTROL_IDENTITY_MISMATCH'; + +export async function fetchFleetSyncStatuses(): Promise { + const res = await apiFetch('/fleet/sync-status', { localOnly: true }); + if (!res.ok) { + throw new Error(`Failed to fetch fleet sync status (HTTP ${res.status})`); + } + return (await res.json()) as FleetSyncStatus[]; +} + +/** + * Proxy the peer's reanchor endpoint so the peer drops its cached control + * fingerprint. Central clears every sticky-error row for the node on a 200, + * so the next push (event-driven or via the 5-minute retry tick) re-tries + * cleanly. + */ +export async function resetFleetSyncAnchor(nodeId: number): Promise { + const res = await apiFetch(`/nodes/${nodeId}/fleet-sync/reset-anchor`, { + method: 'POST', + localOnly: true, + }); + if (!res.ok) { + const body = (await res.json().catch(() => ({}))) as { error?: string }; + throw new Error(body.error ?? `Reset anchor failed (HTTP ${res.status})`); + } +}