mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-08 09:54:26 +00:00
e05099f2a1
Treat 409 CONTROL_IDENTITY_MISMATCH from a replica as a non-retriable
failure instead of looping the same 409 through the 5-minute retry
service forever and silently writing identical failure rows.
Backend
- DatabaseService: add `sticky_error_code`, `sticky_error_expected`,
`sticky_error_got` columns to `fleet_sync_status` via an idempotent
migration. New methods setFleetSyncSticky, getFleetSyncStickyCode,
clearFleetSyncStickyForNode. recordFleetSyncSuccess clears the sticky
flag on a clean push. getFailedSyncTargets SQL adds
`AND sticky_error_code IS NULL` so the retry loop skips sticky rows.
- FleetSyncService.executePushToNode: short-circuits at the top when
sticky is set (covers event-driven pushResourceAsync calls). On a 409
with code CONTROL_IDENTITY_MISMATCH, records the failure once and
pins sticky with the expected/got fingerprints carried in the 409 body.
- routes/nodes.ts: new POST /api/nodes/:id/fleet-sync/reset-anchor.
Admin + paid + node:manage. Proxies POST /api/fleet/role/reanchor to
the peer with `{override:true}` using the stored Bearer node_proxy
token. On peer 200, clears every sticky row for the node so the next
push re-anchors and resumes replication. Distinct 502 / 504 responses
for peer-rejected / peer-unreachable so the UI can show a useful toast.
Frontend
- New lib/fleetSyncApi.ts + hooks/useFleetSyncStatus.ts. Polling hook
(30s visibilityInterval) skips fetch when !isPaid.
- NodeManager.tsx: destructive banner per affected node listing both
fingerprints, with `Reset anchor on peer` and `Remove node` buttons.
Hidden for community-tier users via empty hook data.
- FleetConfiguration.tsx (Fleet -> Status): read-only `Policy sync`
SummaryRow per remote node card. In sync / degraded / paused with
a tooltip; no action buttons (the action lives in NodeManager).
Tests
- fleet-sync-service.test.ts: 4 new cases for sticky-set on first
mismatch, short-circuit on subsequent pushes, null fingerprints,
and non-mismatch failures not setting sticky.
- database-fleet-sync-sticky.test.ts (new): 6 cases pinning the DB
contract incl. retry-loop SQL filter and migration idempotency.
- nodes-fleet-sync-reset-anchor.test.ts (new): 6 cases covering
happy path, peer 401 -> 502, peer unreachable -> 504, local-node
rejection, unknown node id, and community-tier 403.
Gate parity (Directive 30): the new POST .../reset-anchor enforces
requireAdmin + requirePaid + node:manage (matches the existing read at
GET /api/fleet/sync-status). UI banner + SummaryRow only render when the
hook returns data, which it only does for paid-tier authed users. No
existing tier-gate file moved; this is greenfield parity.
Auth audit: the peer's POST /api/fleet/role/reanchor route already uses
requireAdmin, which accepts the central's stored node_proxy Bearer
token because authMiddleware maps `scope === 'node_proxy'` to
`req.user = { username: 'node-proxy', role: 'admin', userId: 0 }`.
No widening required.
Backend tsc clean. Frontend tsc -b clean. 59 fleet-sync tests pass; full
backend suite green minus the pre-existing Windows-only file-lock flake
on filesystem-backup.test.ts that reproduces unchanged on main.
130 lines
5.3 KiB
TypeScript
130 lines
5.3 KiB
TypeScript
/**
|
|
* Pins the DatabaseService sticky-error wiring used by the F-16 fix:
|
|
* - setFleetSyncSticky writes the code + expected + got fingerprints.
|
|
* - getFleetSyncStickyCode reads them back.
|
|
* - getFailedSyncTargets excludes sticky rows (the retry loop must not pick them up).
|
|
* - recordFleetSyncSuccess clears the sticky on success (operator reset → push resumes).
|
|
* - clearFleetSyncStickyForNode clears every resource for one node id (used by the reset endpoint).
|
|
*/
|
|
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
|
|
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
|
|
|
let tmpDir: string;
|
|
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
|
let nodeId: number;
|
|
let siblingId: number;
|
|
|
|
beforeAll(async () => {
|
|
tmpDir = await setupTestDb();
|
|
({ DatabaseService } = await import('../services/DatabaseService'));
|
|
const db = DatabaseService.getInstance();
|
|
nodeId = db.addNode({
|
|
name: 'sticky-target',
|
|
type: 'remote',
|
|
compose_dir: '/app/compose',
|
|
is_default: false,
|
|
api_url: 'https://sticky.example',
|
|
api_token: 'tok',
|
|
mode: 'proxy',
|
|
});
|
|
siblingId = db.addNode({
|
|
name: 'sticky-sibling',
|
|
type: 'remote',
|
|
compose_dir: '/app/compose',
|
|
is_default: false,
|
|
api_url: 'https://sibling-sticky.example',
|
|
api_token: 'tok',
|
|
mode: 'proxy',
|
|
});
|
|
});
|
|
|
|
afterAll(() => {
|
|
cleanupTestDb(tmpDir);
|
|
});
|
|
|
|
beforeEach(() => {
|
|
const db = DatabaseService.getInstance();
|
|
// Wipe stale rows from prior tests in this file so each case starts clean.
|
|
db.getDb().prepare('DELETE FROM fleet_sync_status WHERE node_id IN (?, ?)').run(nodeId, siblingId);
|
|
});
|
|
|
|
describe('fleet_sync_status sticky-error column', () => {
|
|
it('setFleetSyncSticky persists the code and fingerprints', () => {
|
|
const db = DatabaseService.getInstance();
|
|
db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', 'aaa111', 'bbb222');
|
|
|
|
const row = db.getFleetSyncStatuses().find(
|
|
(s) => s.node_id === nodeId && s.resource === 'scan_policies',
|
|
);
|
|
expect(row).toBeDefined();
|
|
expect(row!.sticky_error_code).toBe('CONTROL_IDENTITY_MISMATCH');
|
|
expect(row!.sticky_error_expected).toBe('aaa111');
|
|
expect(row!.sticky_error_got).toBe('bbb222');
|
|
expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBe('CONTROL_IDENTITY_MISMATCH');
|
|
});
|
|
|
|
it('setFleetSyncSticky upserts when no row exists yet', () => {
|
|
const db = DatabaseService.getInstance();
|
|
// Pre-state: no row.
|
|
expect(db.getFleetSyncStickyCode(nodeId, 'cve_suppressions')).toBeNull();
|
|
|
|
db.setFleetSyncSticky(nodeId, 'cve_suppressions', 'CONTROL_IDENTITY_MISMATCH', null, null);
|
|
|
|
expect(db.getFleetSyncStickyCode(nodeId, 'cve_suppressions')).toBe('CONTROL_IDENTITY_MISMATCH');
|
|
});
|
|
|
|
it('getFailedSyncTargets excludes rows where sticky_error_code is set', () => {
|
|
const db = DatabaseService.getInstance();
|
|
db.recordFleetSyncFailure(nodeId, 'scan_policies', 'timeout');
|
|
db.recordFleetSyncFailure(siblingId, 'scan_policies', 'connection refused');
|
|
// Mark only `nodeId` as sticky; the sibling stays retriable.
|
|
db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', null, null);
|
|
|
|
const retriable = db.getFailedSyncTargets('scan_policies', 24 * 60 * 60_000);
|
|
const retriableIds = retriable.map((r) => r.node_id);
|
|
expect(retriableIds).toContain(siblingId);
|
|
expect(retriableIds).not.toContain(nodeId);
|
|
});
|
|
|
|
it('recordFleetSyncSuccess clears the sticky flag (operator-reset round-trip)', () => {
|
|
const db = DatabaseService.getInstance();
|
|
db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', 'aaa', 'bbb');
|
|
expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBe('CONTROL_IDENTITY_MISMATCH');
|
|
|
|
db.recordFleetSyncSuccess(nodeId, 'scan_policies');
|
|
|
|
expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBeNull();
|
|
const row = db.getFleetSyncStatuses().find(
|
|
(s) => s.node_id === nodeId && s.resource === 'scan_policies',
|
|
);
|
|
expect(row!.sticky_error_expected).toBeNull();
|
|
expect(row!.sticky_error_got).toBeNull();
|
|
});
|
|
|
|
it('clearFleetSyncStickyForNode clears every resource for one node, leaves siblings untouched', () => {
|
|
const db = DatabaseService.getInstance();
|
|
db.setFleetSyncSticky(nodeId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', null, null);
|
|
db.setFleetSyncSticky(nodeId, 'cve_suppressions', 'CONTROL_IDENTITY_MISMATCH', null, null);
|
|
db.setFleetSyncSticky(siblingId, 'scan_policies', 'CONTROL_IDENTITY_MISMATCH', null, null);
|
|
|
|
db.clearFleetSyncStickyForNode(nodeId);
|
|
|
|
expect(db.getFleetSyncStickyCode(nodeId, 'scan_policies')).toBeNull();
|
|
expect(db.getFleetSyncStickyCode(nodeId, 'cve_suppressions')).toBeNull();
|
|
expect(db.getFleetSyncStickyCode(siblingId, 'scan_policies')).toBe('CONTROL_IDENTITY_MISMATCH');
|
|
});
|
|
|
|
it('migrateFleetSyncStickyError is idempotent (running twice does not error)', () => {
|
|
// The constructor already runs the migration once at boot. Manually
|
|
// invoke the private method twice via index access to confirm
|
|
// tryAddColumn's idempotency contract holds for this migration.
|
|
const db = DatabaseService.getInstance() as unknown as {
|
|
migrateFleetSyncStickyError: () => void;
|
|
};
|
|
expect(() => {
|
|
db.migrateFleetSyncStickyError();
|
|
db.migrateFleetSyncStickyError();
|
|
}).not.toThrow();
|
|
});
|
|
});
|