fix(fleet-sync): make control-identity-mismatch sticky and surface in UI (#1117)

Treat 409 CONTROL_IDENTITY_MISMATCH from a replica as a non-retriable
failure instead of looping the same 409 through the 5-minute retry
service forever and silently writing identical failure rows.

Backend
- DatabaseService: add `sticky_error_code`, `sticky_error_expected`,
  `sticky_error_got` columns to `fleet_sync_status` via an idempotent
  migration. New methods setFleetSyncSticky, getFleetSyncStickyCode,
  clearFleetSyncStickyForNode. recordFleetSyncSuccess clears the sticky
  flag on a clean push. getFailedSyncTargets SQL adds
  `AND sticky_error_code IS NULL` so the retry loop skips sticky rows.
- FleetSyncService.executePushToNode: short-circuits at the top when
  sticky is set (covers event-driven pushResourceAsync calls). On a 409
  with code CONTROL_IDENTITY_MISMATCH, records the failure once and
  pins sticky with the expected/got fingerprints carried in the 409 body.
- routes/nodes.ts: new POST /api/nodes/:id/fleet-sync/reset-anchor.
  Admin + paid + node:manage. Proxies POST /api/fleet/role/reanchor to
  the peer with `{override:true}` using the stored Bearer node_proxy
  token. On peer 200, clears every sticky row for the node so the next
  push re-anchors and resumes replication. Distinct 502 / 504 responses
  for peer-rejected / peer-unreachable so the UI can show a useful toast.

Frontend
- New lib/fleetSyncApi.ts + hooks/useFleetSyncStatus.ts. Polling hook
  (30s visibilityInterval) skips fetch when !isPaid.
- NodeManager.tsx: destructive banner per affected node listing both
  fingerprints, with `Reset anchor on peer` and `Remove node` buttons.
  Hidden for community-tier users via empty hook data.
- FleetConfiguration.tsx (Fleet -> Status): read-only `Policy sync`
  SummaryRow per remote node card. In sync / degraded / paused with
  a tooltip; no action buttons (the action lives in NodeManager).

Tests
- fleet-sync-service.test.ts: 4 new cases for sticky-set on first
  mismatch, short-circuit on subsequent pushes, null fingerprints,
  and non-mismatch failures not setting sticky.
- database-fleet-sync-sticky.test.ts (new): 6 cases pinning the DB
  contract incl. retry-loop SQL filter and migration idempotency.
- nodes-fleet-sync-reset-anchor.test.ts (new): 6 cases covering
  happy path, peer 401 -> 502, peer unreachable -> 504, local-node
  rejection, unknown node id, and community-tier 403.

Gate parity (Directive 30): the new POST .../reset-anchor enforces
requireAdmin + requirePaid + node:manage (matches the existing read at
GET /api/fleet/sync-status). UI banner + SummaryRow only render when the
hook returns data, which it only does for paid-tier authed users. No
existing tier-gate file moved; this is greenfield parity.

Auth audit: the peer's POST /api/fleet/role/reanchor route already uses
requireAdmin, which accepts the central's stored node_proxy Bearer
token because authMiddleware maps `scope === 'node_proxy'` to
`req.user = { username: 'node-proxy', role: 'admin', userId: 0 }`.
No widening required.

Backend tsc clean. Frontend tsc -b clean. 59 fleet-sync tests pass; full
backend suite green minus the pre-existing Windows-only file-lock flake
on filesystem-backup.test.ts that reproduces unchanged on main.

backend/src/__tests__/database-fleet-sync-sticky.test.ts

@@ -0,0 +1,129 @@
+/**
+ * Pins the DatabaseService sticky-error wiring used by the F-16 fix:
+ *   - setFleetSyncSticky writes the code + expected + got fingerprints.
+ *   - getFleetSyncStickyCode reads them back.
+ *   - getFailedSyncTargets excludes sticky rows (the retry loop must not pick them up).
+ *   - recordFleetSyncSuccess clears the sticky on success (operator reset → push resumes).
+ *   - clearFleetSyncStickyForNode clears every resource for one node id (used by the reset endpoint).
+ */
+import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
+import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
+
+let tmpDir: string;
+let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
+let nodeId: number;
+let siblingId: number;
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ DatabaseService } = await import('../services/DatabaseService'));
+  const db = DatabaseService.getInstance();
+  nodeId = db.addNode({
+    name: 'sticky-target',
+    type: 'remote',
+    compose_dir: '/app/compose',
+    is_default: false,
+    api_url: 'https://sticky.example',
+    api_token: 'tok',
+    mode: 'proxy',
+  });
+  siblingId = db.addNode({
+    name: 'sticky-sibling',
+    type: 'remote',
+    compose_dir: '/app/compose',
+    is_default: false,
+    api_url: 'https://sibling-sticky.example',
+    api_token: 'tok',
+    mode: 'proxy',
+  });
+});
+
+afterAll(() => {
+  cleanupTestDb(tmpDir);
+});
+
+beforeEach(() => {
+  const db = DatabaseService.getInstance();
+  // Wipe stale rows from prior tests in this file so each case starts clean.
+  db.getDb().prepare('DELETE FROM fleet_sync_status WHERE node_id IN (?, ?)').run(nodeId, siblingId);
+});
+
+describe('fleet_sync_status sticky-error column', () => {
+  it('setFleetSyncSticky persists the code and fingerprints', () => {
+    const db = DatabaseService.getInstance();
+    db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', 'aaa111', 'bbb222');
+
+    const row = db.getFleetSyncStatuses().find(
+      (s) => s.node_id === nodeId && s.resource === 'scan_policies',
+    );
+    expect(row).toBeDefined();
+    expect(row!.sticky_error_code).toBe('CONTROL_IDENTITY_MISMATCH');
+    expect(row!.sticky_error_expected).toBe('aaa111');
+    expect(row!.sticky_error_got).toBe('bbb222');
+    expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBe('CONTROL_IDENTITY_MISMATCH');
+  });
+
+  it('setFleetSyncSticky upserts when no row exists yet', () => {
+    const db = DatabaseService.getInstance();
+    // Pre-state: no row.
+    expect(db.getFleetSyncStickyCode(nodeId, 'cve_suppressions')).toBeNull();
+
+    db.setFleetSyncSticky(nodeId, 'cve_suppressions', 'CONTROL_IDENTITY_MISMATCH', null, null);
+
+    expect(db.getFleetSyncStickyCode(nodeId, 'cve_suppressions')).toBe('CONTROL_IDENTITY_MISMATCH');
+  });
+
+  it('getFailedSyncTargets excludes rows where sticky_error_code is set', () => {
+    const db = DatabaseService.getInstance();
+    db.recordFleetSyncFailure(nodeId, 'scan_policies', 'timeout');
+    db.recordFleetSyncFailure(siblingId, 'scan_policies', 'connection refused');
+    // Mark only `nodeId` as sticky; the sibling stays retriable.
+    db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', null, null);
+
+    const retriable = db.getFailedSyncTargets('scan_policies', 24 * 60 * 60_000);
+    const retriableIds = retriable.map((r) => r.node_id);
+    expect(retriableIds).toContain(siblingId);
+    expect(retriableIds).not.toContain(nodeId);
+  });
+
+  it('recordFleetSyncSuccess clears the sticky flag (operator-reset round-trip)', () => {
+    const db = DatabaseService.getInstance();
+    db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', 'aaa', 'bbb');
+    expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBe('CONTROL_IDENTITY_MISMATCH');
+
+    db.recordFleetSyncSuccess(nodeId, 'scan_policies');
+
+    expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBeNull();
+    const row = db.getFleetSyncStatuses().find(
+      (s) => s.node_id === nodeId && s.resource === 'scan_policies',
+    );
+    expect(row!.sticky_error_expected).toBeNull();
+    expect(row!.sticky_error_got).toBeNull();
+  });
+
+  it('clearFleetSyncStickyForNode clears every resource for one node, leaves siblings untouched', () => {
+    const db = DatabaseService.getInstance();
+    db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', null, null);
+    db.setFleetSyncSticky(nodeId, 'cve_suppressions', 'CONTROL_IDENTITY_MISMATCH', null, null);
+    db.setFleetSyncSticky(siblingId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', null, null);
+
+    db.clearFleetSyncStickyForNode(nodeId);
+
+    expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBeNull();
+    expect(db.getFleetSyncStickyCode(nodeId, 'cve_suppressions')).toBeNull();
+    expect(db.getFleetSyncStickyCode(siblingId, 'scan_policies')).toBe('CONTROL_IDENTITY_MISMATCH');
+  });
+
+  it('migrateFleetSyncStickyError is idempotent (running twice does not error)', () => {
+    // The constructor already runs the migration once at boot. Manually
+    // invoke the private method twice via index access to confirm
+    // tryAddColumn's idempotency contract holds for this migration.
+    const db = DatabaseService.getInstance() as unknown as {
+      migrateFleetSyncStickyError: () => void;
+    };
+    expect(() => {
+      db.migrateFleetSyncStickyError();
+      db.migrateFleetSyncStickyError();
+    }).not.toThrow();
+  });
+});

backend/src/__tests__/fleet-sync-service.test.ts

@@ -16,6 +16,8 @@ const {
   mockInsertAuditLog,
   mockRecordFleetSyncSuccess,
   mockRecordFleetSyncFailure,
+  mockSetFleetSyncSticky,
+  mockGetFleetSyncStickyCode,
   mockGetSystemState,
   mockSetSystemState,
   mockTransaction,
@@ -33,6 +35,8 @@ const {
   mockInsertAuditLog: vi.fn(),
   mockRecordFleetSyncSuccess: vi.fn(),
   mockRecordFleetSyncFailure: vi.fn(),
+  mockSetFleetSyncSticky: vi.fn(),
+  mockGetFleetSyncStickyCode: vi.fn().mockReturnValue(null),
   mockGetSystemState: vi.fn().mockReturnValue(null),
   mockSetSystemState: vi.fn(),
   mockTransaction: vi.fn().mockImplementation((fn: () => unknown) => fn()),
@@ -53,6 +57,8 @@ vi.mock('../services/DatabaseService', () => ({
       insertAuditLog: mockInsertAuditLog,
       recordFleetSyncSuccess: mockRecordFleetSyncSuccess,
       recordFleetSyncFailure: mockRecordFleetSyncFailure,
+      setFleetSyncSticky: mockSetFleetSyncSticky,
+      getFleetSyncStickyCode: mockGetFleetSyncStickyCode,
       getSystemState: mockGetSystemState,
       setSystemState: mockSetSystemState,
       transaction: mockTransaction,
@@ -90,6 +96,7 @@ import { FleetSyncService, LOCAL_IDENTITY_SENTINEL } from '../services/FleetSync
 beforeEach(() => {
   vi.clearAllMocks();
   mockGetSystemState.mockReturnValue(null);
+  mockGetFleetSyncStickyCode.mockReturnValue(null);
 });
 
 describe('FleetSyncService.getRole', () => {
@@ -594,3 +601,102 @@ describe('FleetSyncService.formatError redaction', () => {
     expect(failure[2]).toContain('[redacted-jwt]');
   });
 });
+
+describe('FleetSyncService CONTROL_IDENTITY_MISMATCH sticky handling', () => {
+  function makeMismatchError(expected: string, got: string) {
+    return async () => {
+      const { AxiosError } = await import('axios');
+      const err = new AxiosError('Request failed with status code 409');
+      (err as unknown as { response: unknown }).response = {
+        status: 409,
+        statusText: 'Conflict',
+        data: {
+          error: `Control identity mismatch: replica is anchored to "${expected}", push from "${got}"`,
+          code: 'CONTROL_IDENTITY_MISMATCH',
+          expected,
+          got,
+        },
+      };
+      throw err;
+    };
+  }
+
+  it('sets the sticky flag carrying the expected/got fingerprints on first mismatch', async () => {
+    mockGetNodes.mockReturnValue([
+      { id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' },
+    ]);
+    mockGetLocalScanPolicies.mockReturnValue([]);
+    mockGetFleetSyncStickyCode.mockReturnValue(null);
+    mockAxiosPost.mockImplementation(makeMismatchError('cb45a2eff9db81d8', '555f8d1f7e7e71e3'));
+
+    await FleetSyncService.getInstance().pushResource('scan_policies');
+
+    expect(mockRecordFleetSyncFailure).toHaveBeenCalledTimes(1);
+    expect(mockSetFleetSyncSticky).toHaveBeenCalledTimes(1);
+    expect(mockSetFleetSyncSticky).toHaveBeenCalledWith(
+      7,
+      'scan_policies',
+      'CONTROL_IDENTITY_MISMATCH',
+      'cb45a2eff9db81d8',
+      '555f8d1f7e7e71e3',
+    );
+  });
+
+  it('short-circuits subsequent pushes when sticky is already set; no HTTP call, no failure record', async () => {
+    mockGetNodes.mockReturnValue([
+      { id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' },
+    ]);
+    mockGetLocalScanPolicies.mockReturnValue([]);
+    // Sticky already set from a prior push.
+    mockGetFleetSyncStickyCode.mockReturnValue('CONTROL_IDENTITY_MISMATCH');
+
+    await FleetSyncService.getInstance().pushResource('scan_policies');
+
+    expect(mockAxiosPost).not.toHaveBeenCalled();
+    expect(mockRecordFleetSyncFailure).not.toHaveBeenCalled();
+    expect(mockRecordFleetSyncSuccess).not.toHaveBeenCalled();
+    expect(mockSetFleetSyncSticky).not.toHaveBeenCalled();
+  });
+
+  it('tolerates a missing expected/got payload (passes null through)', async () => {
+    mockGetNodes.mockReturnValue([
+      { id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' },
+    ]);
+    mockGetLocalScanPolicies.mockReturnValue([]);
+    mockGetFleetSyncStickyCode.mockReturnValue(null);
+    mockAxiosPost.mockImplementation(async () => {
+      const { AxiosError } = await import('axios');
+      const err = new AxiosError('Request failed with status code 409');
+      (err as unknown as { response: unknown }).response = {
+        status: 409,
+        statusText: 'Conflict',
+        data: { error: 'mismatch', code: 'CONTROL_IDENTITY_MISMATCH' },
+      };
+      throw err;
+    });
+
+    await FleetSyncService.getInstance().pushResource('scan_policies');
+
+    expect(mockSetFleetSyncSticky).toHaveBeenCalledWith(
+      7,
+      'scan_policies',
+      'CONTROL_IDENTITY_MISMATCH',
+      null,
+      null,
+    );
+  });
+
+  it('does not set sticky for non-mismatch failures (network errors, 500s)', async () => {
+    mockGetNodes.mockReturnValue([
+      { id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' },
+    ]);
+    mockGetLocalScanPolicies.mockReturnValue([]);
+    mockGetFleetSyncStickyCode.mockReturnValue(null);
+    mockAxiosPost.mockRejectedValue(new Error('ECONNREFUSED'));
+
+    await FleetSyncService.getInstance().pushResource('scan_policies');
+
+    expect(mockRecordFleetSyncFailure).toHaveBeenCalledTimes(1);
+    expect(mockSetFleetSyncSticky).not.toHaveBeenCalled();
+  });
+});

backend/src/__tests__/nodes-fleet-sync-reset-anchor.test.ts

@@ -0,0 +1,158 @@
+/**
+ * Tests for POST /api/nodes/:id/fleet-sync/reset-anchor (F-16 fix).
+ *
+ * The endpoint proxies the peer's reanchor endpoint and clears every
+ * sticky-error row for the node on success. Covers:
+ *   - happy path: 200 from peer → sticky rows cleared, 200 returned.
+ *   - peer 401/403 → 502 with helpful message.
+ *   - peer unreachable → 504.
+ *   - missing/non-proxy node → 400.
+ *   - non-paid tier → 403.
+ */
+import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
+import request from 'supertest';
+import jwt from 'jsonwebtoken';
+import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
+
+let tmpDir: string;
+let app: import('express').Express;
+let authHeader: string;
+let peerNodeId: number;
+
+const originalFetch = globalThis.fetch;
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ app } = await import('../index'));
+  const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
+  authHeader = `Bearer ${token}`;
+
+  // Seed a proxy-mode remote node and a sticky row for it. Tests then drive
+  // the route handler and assert side effects on the test DB.
+  const { DatabaseService } = await import('../services/DatabaseService');
+  const db = DatabaseService.getInstance();
+  peerNodeId = db.addNode({
+    name: 'sticky-peer',
+    type: 'remote',
+    compose_dir: '/app/compose',
+    is_default: false,
+    api_url: 'http://192.168.1.99:1852',
+    api_token: 'peer-token',
+    mode: 'proxy',
+  });
+  db.setFleetSyncSticky(
+    peerNodeId,
+    'scan_policies',
+    'CONTROL_IDENTITY_MISMATCH',
+    'cb45a2eff9db81d8',
+    '555f8d1f7e7e71e3',
+  );
+});
+
+afterAll(() => {
+  globalThis.fetch = originalFetch;
+  cleanupTestDb(tmpDir);
+});
+
+beforeEach(async () => {
+  vi.restoreAllMocks();
+  globalThis.fetch = originalFetch;
+  // Re-establish the paid-tier spy after restoreAllMocks. Individual tests
+  // can override with `mockReturnValue('community')` to exercise the tier
+  // gate's deny path.
+  const { LicenseService } = await import('../services/LicenseService');
+  vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+});
+
+describe('POST /api/nodes/:id/fleet-sync/reset-anchor', () => {
+  it('proxies to the peer reanchor, clears sticky rows, returns 200', async () => {
+    const fetchSpy = vi.fn().mockResolvedValue(
+      new Response(JSON.stringify({ success: true }), { status: 200 }),
+    );
+    globalThis.fetch = fetchSpy as unknown as typeof fetch;
+
+    const res = await request(app)
+      .post(`/api/nodes/${peerNodeId}/fleet-sync/reset-anchor`)
+      .set('Authorization', authHeader)
+      .send({});
+
+    expect(res.status).toBe(200);
+    expect(res.body.success).toBe(true);
+    expect(fetchSpy).toHaveBeenCalledTimes(1);
+    const [url, init] = fetchSpy.mock.calls[0] as [string, RequestInit];
+    expect(url).toBe('http://192.168.1.99:1852/api/fleet/role/reanchor');
+    expect(init.method).toBe('POST');
+    expect((init.headers as Record<string, string>).Authorization).toBe('Bearer peer-token');
+    expect(JSON.parse(init.body as string)).toEqual({ override: true });
+
+    const { DatabaseService } = await import('../services/DatabaseService');
+    const sticky = DatabaseService.getInstance().getFleetSyncStickyCode(peerNodeId, 'scan_policies');
+    expect(sticky).toBeNull();
+  });
+
+  it('returns 502 with a helpful message when peer responds 401', async () => {
+    const { DatabaseService } = await import('../services/DatabaseService');
+    DatabaseService.getInstance().setFleetSyncSticky(
+      peerNodeId, 'cve_suppressions', 'CONTROL_IDENTITY_MISMATCH', 'aaa', 'bbb',
+    );
+    const fetchSpy = vi.fn().mockResolvedValue(
+      new Response(JSON.stringify({ error: 'Admin access required.' }), { status: 401 }),
+    );
+    globalThis.fetch = fetchSpy as unknown as typeof fetch;
+
+    const res = await request(app)
+      .post(`/api/nodes/${peerNodeId}/fleet-sync/reset-anchor`)
+      .set('Authorization', authHeader)
+      .send({});
+
+    expect(res.status).toBe(502);
+    expect(res.body.error).toMatch(/Admin access required/);
+    // Sticky rows must remain set so the operator can retry.
+    const sticky = DatabaseService.getInstance().getFleetSyncStickyCode(peerNodeId, 'cve_suppressions');
+    expect(sticky).toBe('CONTROL_IDENTITY_MISMATCH');
+  });
+
+  it('returns 504 when the peer is unreachable', async () => {
+    const fetchSpy = vi.fn().mockRejectedValue(new Error('connect ECONNREFUSED'));
+    globalThis.fetch = fetchSpy as unknown as typeof fetch;
+
+    const res = await request(app)
+      .post(`/api/nodes/${peerNodeId}/fleet-sync/reset-anchor`)
+      .set('Authorization', authHeader)
+      .send({});
+
+    expect(res.status).toBe(504);
+    expect(res.body.error).toMatch(/unreachable/i);
+  });
+
+  it('returns 400 for a local node', async () => {
+    const { DatabaseService } = await import('../services/DatabaseService');
+    const local = DatabaseService.getInstance().getNodes().find((n) => n.type === 'local');
+    expect(local).toBeTruthy();
+    const res = await request(app)
+      .post(`/api/nodes/${local!.id}/fleet-sync/reset-anchor`)
+      .set('Authorization', authHeader)
+      .send({});
+    expect(res.status).toBe(400);
+  });
+
+  it('returns 404 for an unknown node id', async () => {
+    const res = await request(app)
+      .post('/api/nodes/9999/fleet-sync/reset-anchor')
+      .set('Authorization', authHeader)
+      .send({});
+    expect(res.status).toBe(404);
+  });
+
+  it('returns 403 (PAID_REQUIRED) when the license is community-tier', async () => {
+    const { LicenseService } = await import('../services/LicenseService');
+    vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
+
+    const res = await request(app)
+      .post(`/api/nodes/${peerNodeId}/fleet-sync/reset-anchor`)
+      .set('Authorization', authHeader)
+      .send({});
+    expect(res.status).toBe(403);
+    expect(res.body.code).toBe('PAID_REQUIRED');
+  });
+});