mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-30 20:29:15 +00:00
fix(fleet-sync): make control-identity-mismatch sticky and surface in UI (#1117)
Treat 409 CONTROL_IDENTITY_MISMATCH from a replica as a non-retriable
failure instead of looping the same 409 through the 5-minute retry
service forever and silently writing identical failure rows.
Backend
- DatabaseService: add `sticky_error_code`, `sticky_error_expected`,
`sticky_error_got` columns to `fleet_sync_status` via an idempotent
migration. New methods setFleetSyncSticky, getFleetSyncStickyCode,
clearFleetSyncStickyForNode. recordFleetSyncSuccess clears the sticky
flag on a clean push. getFailedSyncTargets SQL adds
`AND sticky_error_code IS NULL` so the retry loop skips sticky rows.
- FleetSyncService.executePushToNode: short-circuits at the top when
sticky is set (covers event-driven pushResourceAsync calls). On a 409
with code CONTROL_IDENTITY_MISMATCH, records the failure once and
pins sticky with the expected/got fingerprints carried in the 409 body.
- routes/nodes.ts: new POST /api/nodes/:id/fleet-sync/reset-anchor.
Admin + paid + node:manage. Proxies POST /api/fleet/role/reanchor to
the peer with `{override:true}` using the stored Bearer node_proxy
token. On peer 200, clears every sticky row for the node so the next
push re-anchors and resumes replication. Distinct 502 / 504 responses
for peer-rejected / peer-unreachable so the UI can show a useful toast.
Frontend
- New lib/fleetSyncApi.ts + hooks/useFleetSyncStatus.ts. Polling hook
(30s visibilityInterval) skips fetch when !isPaid.
- NodeManager.tsx: destructive banner per affected node listing both
fingerprints, with `Reset anchor on peer` and `Remove node` buttons.
Hidden for community-tier users via empty hook data.
- FleetConfiguration.tsx (Fleet -> Status): read-only `Policy sync`
SummaryRow per remote node card. In sync / degraded / paused with
a tooltip; no action buttons (the action lives in NodeManager).
Tests
- fleet-sync-service.test.ts: 4 new cases for sticky-set on first
mismatch, short-circuit on subsequent pushes, null fingerprints,
and non-mismatch failures not setting sticky.
- database-fleet-sync-sticky.test.ts (new): 6 cases pinning the DB
contract incl. retry-loop SQL filter and migration idempotency.
- nodes-fleet-sync-reset-anchor.test.ts (new): 6 cases covering
happy path, peer 401 -> 502, peer unreachable -> 504, local-node
rejection, unknown node id, and community-tier 403.
Gate parity (Directive 30): the new POST .../reset-anchor enforces
requireAdmin + requirePaid + node:manage (matches the existing read at
GET /api/fleet/sync-status). UI banner + SummaryRow only render when the
hook returns data, which it only does for paid-tier authed users. No
existing tier-gate file moved; this is greenfield parity.
Auth audit: the peer's POST /api/fleet/role/reanchor route already uses
requireAdmin, which accepts the central's stored node_proxy Bearer
token because authMiddleware maps `scope === 'node_proxy'` to
`req.user = { username: 'node-proxy', role: 'admin', userId: 0 }`.
No widening required.
Backend tsc clean. Frontend tsc -b clean. 59 fleet-sync tests pass; full
backend suite green minus the pre-existing Windows-only file-lock flake
on filesystem-backup.test.ts that reproduces unchanged on main.
This commit is contained in:
@@ -16,6 +16,8 @@ const {
|
||||
mockInsertAuditLog,
|
||||
mockRecordFleetSyncSuccess,
|
||||
mockRecordFleetSyncFailure,
|
||||
mockSetFleetSyncSticky,
|
||||
mockGetFleetSyncStickyCode,
|
||||
mockGetSystemState,
|
||||
mockSetSystemState,
|
||||
mockTransaction,
|
||||
@@ -33,6 +35,8 @@ const {
|
||||
mockInsertAuditLog: vi.fn(),
|
||||
mockRecordFleetSyncSuccess: vi.fn(),
|
||||
mockRecordFleetSyncFailure: vi.fn(),
|
||||
mockSetFleetSyncSticky: vi.fn(),
|
||||
mockGetFleetSyncStickyCode: vi.fn().mockReturnValue(null),
|
||||
mockGetSystemState: vi.fn().mockReturnValue(null),
|
||||
mockSetSystemState: vi.fn(),
|
||||
mockTransaction: vi.fn().mockImplementation((fn: () => unknown) => fn()),
|
||||
@@ -53,6 +57,8 @@ vi.mock('../services/DatabaseService', () => ({
|
||||
insertAuditLog: mockInsertAuditLog,
|
||||
recordFleetSyncSuccess: mockRecordFleetSyncSuccess,
|
||||
recordFleetSyncFailure: mockRecordFleetSyncFailure,
|
||||
setFleetSyncSticky: mockSetFleetSyncSticky,
|
||||
getFleetSyncStickyCode: mockGetFleetSyncStickyCode,
|
||||
getSystemState: mockGetSystemState,
|
||||
setSystemState: mockSetSystemState,
|
||||
transaction: mockTransaction,
|
||||
@@ -90,6 +96,7 @@ import { FleetSyncService, LOCAL_IDENTITY_SENTINEL } from '../services/FleetSync
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockGetSystemState.mockReturnValue(null);
|
||||
mockGetFleetSyncStickyCode.mockReturnValue(null);
|
||||
});
|
||||
|
||||
describe('FleetSyncService.getRole', () => {
|
||||
@@ -594,3 +601,102 @@ describe('FleetSyncService.formatError redaction', () => {
|
||||
expect(failure[2]).toContain('[redacted-jwt]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('FleetSyncService CONTROL_IDENTITY_MISMATCH sticky handling', () => {
|
||||
function makeMismatchError(expected: string, got: string) {
|
||||
return async () => {
|
||||
const { AxiosError } = await import('axios');
|
||||
const err = new AxiosError('Request failed with status code 409');
|
||||
(err as unknown as { response: unknown }).response = {
|
||||
status: 409,
|
||||
statusText: 'Conflict',
|
||||
data: {
|
||||
error: `Control identity mismatch: replica is anchored to "${expected}", push from "${got}"`,
|
||||
code: 'CONTROL_IDENTITY_MISMATCH',
|
||||
expected,
|
||||
got,
|
||||
},
|
||||
};
|
||||
throw err;
|
||||
};
|
||||
}
|
||||
|
||||
it('sets the sticky flag carrying the expected/got fingerprints on first mismatch', async () => {
|
||||
mockGetNodes.mockReturnValue([
|
||||
{ id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' },
|
||||
]);
|
||||
mockGetLocalScanPolicies.mockReturnValue([]);
|
||||
mockGetFleetSyncStickyCode.mockReturnValue(null);
|
||||
mockAxiosPost.mockImplementation(makeMismatchError('cb45a2eff9db81d8', '555f8d1f7e7e71e3'));
|
||||
|
||||
await FleetSyncService.getInstance().pushResource('scan_policies');
|
||||
|
||||
expect(mockRecordFleetSyncFailure).toHaveBeenCalledTimes(1);
|
||||
expect(mockSetFleetSyncSticky).toHaveBeenCalledTimes(1);
|
||||
expect(mockSetFleetSyncSticky).toHaveBeenCalledWith(
|
||||
7,
|
||||
'scan_policies',
|
||||
'CONTROL_IDENTITY_MISMATCH',
|
||||
'cb45a2eff9db81d8',
|
||||
'555f8d1f7e7e71e3',
|
||||
);
|
||||
});
|
||||
|
||||
it('short-circuits subsequent pushes when sticky is already set; no HTTP call, no failure record', async () => {
|
||||
mockGetNodes.mockReturnValue([
|
||||
{ id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' },
|
||||
]);
|
||||
mockGetLocalScanPolicies.mockReturnValue([]);
|
||||
// Sticky already set from a prior push.
|
||||
mockGetFleetSyncStickyCode.mockReturnValue('CONTROL_IDENTITY_MISMATCH');
|
||||
|
||||
await FleetSyncService.getInstance().pushResource('scan_policies');
|
||||
|
||||
expect(mockAxiosPost).not.toHaveBeenCalled();
|
||||
expect(mockRecordFleetSyncFailure).not.toHaveBeenCalled();
|
||||
expect(mockRecordFleetSyncSuccess).not.toHaveBeenCalled();
|
||||
expect(mockSetFleetSyncSticky).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('tolerates a missing expected/got payload (passes null through)', async () => {
|
||||
mockGetNodes.mockReturnValue([
|
||||
{ id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' },
|
||||
]);
|
||||
mockGetLocalScanPolicies.mockReturnValue([]);
|
||||
mockGetFleetSyncStickyCode.mockReturnValue(null);
|
||||
mockAxiosPost.mockImplementation(async () => {
|
||||
const { AxiosError } = await import('axios');
|
||||
const err = new AxiosError('Request failed with status code 409');
|
||||
(err as unknown as { response: unknown }).response = {
|
||||
status: 409,
|
||||
statusText: 'Conflict',
|
||||
data: { error: 'mismatch', code: 'CONTROL_IDENTITY_MISMATCH' },
|
||||
};
|
||||
throw err;
|
||||
});
|
||||
|
||||
await FleetSyncService.getInstance().pushResource('scan_policies');
|
||||
|
||||
expect(mockSetFleetSyncSticky).toHaveBeenCalledWith(
|
||||
7,
|
||||
'scan_policies',
|
||||
'CONTROL_IDENTITY_MISMATCH',
|
||||
null,
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
it('does not set sticky for non-mismatch failures (network errors, 500s)', async () => {
|
||||
mockGetNodes.mockReturnValue([
|
||||
{ id: 7, type: 'remote', api_url: 'https://peer.example', api_token: 'tok', name: 'peer', mode: 'proxy' },
|
||||
]);
|
||||
mockGetLocalScanPolicies.mockReturnValue([]);
|
||||
mockGetFleetSyncStickyCode.mockReturnValue(null);
|
||||
mockAxiosPost.mockRejectedValue(new Error('ECONNREFUSED'));
|
||||
|
||||
await FleetSyncService.getInstance().pushResource('scan_policies');
|
||||
|
||||
expect(mockRecordFleetSyncFailure).toHaveBeenCalledTimes(1);
|
||||
expect(mockSetFleetSyncSticky).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user