mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-27 10:46:51 +00:00
fix(fleet-sync): make control-identity-mismatch sticky and surface in UI (#1117)
Treat 409 CONTROL_IDENTITY_MISMATCH from a replica as a non-retriable
failure instead of looping the same 409 through the 5-minute retry
service forever and silently writing identical failure rows.
Backend
- DatabaseService: add `sticky_error_code`, `sticky_error_expected`,
`sticky_error_got` columns to `fleet_sync_status` via an idempotent
migration. New methods setFleetSyncSticky, getFleetSyncStickyCode,
clearFleetSyncStickyForNode. recordFleetSyncSuccess clears the sticky
flag on a clean push. getFailedSyncTargets SQL adds
`AND sticky_error_code IS NULL` so the retry loop skips sticky rows.
- FleetSyncService.executePushToNode: short-circuits at the top when
sticky is set (covers event-driven pushResourceAsync calls). On a 409
with code CONTROL_IDENTITY_MISMATCH, records the failure once and
pins sticky with the expected/got fingerprints carried in the 409 body.
- routes/nodes.ts: new POST /api/nodes/:id/fleet-sync/reset-anchor.
Admin + paid + node:manage. Proxies POST /api/fleet/role/reanchor to
the peer with `{override:true}` using the stored Bearer node_proxy
token. On peer 200, clears every sticky row for the node so the next
push re-anchors and resumes replication. Distinct 502 / 504 responses
for peer-rejected / peer-unreachable so the UI can show a useful toast.
Frontend
- New lib/fleetSyncApi.ts + hooks/useFleetSyncStatus.ts. Polling hook
(30s visibilityInterval) skips fetch when !isPaid.
- NodeManager.tsx: destructive banner per affected node listing both
fingerprints, with `Reset anchor on peer` and `Remove node` buttons.
Hidden for community-tier users via empty hook data.
- FleetConfiguration.tsx (Fleet -> Status): read-only `Policy sync`
SummaryRow per remote node card. In sync / degraded / paused with
a tooltip; no action buttons (the action lives in NodeManager).
Tests
- fleet-sync-service.test.ts: 4 new cases for sticky-set on first
mismatch, short-circuit on subsequent pushes, null fingerprints,
and non-mismatch failures not setting sticky.
- database-fleet-sync-sticky.test.ts (new): 6 cases pinning the DB
contract incl. retry-loop SQL filter and migration idempotency.
- nodes-fleet-sync-reset-anchor.test.ts (new): 6 cases covering
happy path, peer 401 -> 502, peer unreachable -> 504, local-node
rejection, unknown node id, and community-tier 403.
Gate parity (Directive 30): the new POST .../reset-anchor enforces
requireAdmin + requirePaid + node:manage (matches the existing read at
GET /api/fleet/sync-status). UI banner + SummaryRow only render when the
hook returns data, which it only does for paid-tier authed users. No
existing tier-gate file moved; this is greenfield parity.
Auth audit: the peer's POST /api/fleet/role/reanchor route already uses
requireAdmin, which accepts the central's stored node_proxy Bearer
token because authMiddleware maps `scope === 'node_proxy'` to
`req.user = { username: 'node-proxy', role: 'admin', userId: 0 }`.
No widening required.
Backend tsc clean. Frontend tsc -b clean. 59 fleet-sync tests pass; full
backend suite green minus the pre-existing Windows-only file-lock flake
on filesystem-backup.test.ts that reproduces unchanged on main.
This commit is contained in:
@@ -4,7 +4,7 @@ import crypto from 'crypto';
|
||||
import { authMiddleware } from '../middleware/auth';
|
||||
import { requirePermission } from '../middleware/permissions';
|
||||
import { rejectApiTokenScope } from '../middleware/apiTokenScope';
|
||||
import { requireAdmiral } from '../middleware/tierGates';
|
||||
import { requireAdmin, requireAdmiral, requirePaid } from '../middleware/tierGates';
|
||||
import { enrollmentLimiter } from '../middleware/rateLimiters';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
@@ -354,6 +354,93 @@ nodesRouter.post('/:id/uncordon', (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Reset the FleetSync control anchor on a remote peer.
|
||||
*
|
||||
* Proxies POST /api/fleet/role/reanchor to the peer using its stored
|
||||
* Bearer token. A successful reanchor clears every sticky-error row for
|
||||
* this node so the next push (event-driven or via the 5-minute retry
|
||||
* service) re-attempts cleanly and the peer accepts the central's
|
||||
* fingerprint as the new anchor.
|
||||
*
|
||||
* Surfaces UI affordance for the F-16 audit (mesh-e2e-2026-05-17.md):
|
||||
* when a peer was previously enrolled by a different central, FleetSync
|
||||
* keeps 409'ing every reconcile tick; the sticky flag halts retries and
|
||||
* this endpoint is the single one-click recovery for the operator.
|
||||
*/
|
||||
nodesRouter.post('/:id/fleet-sync/reset-anchor', async (req: Request, res: Response) => {
|
||||
if (rejectApiTokenScope(req, res, NODE_SCOPE_MESSAGE)) return;
|
||||
const nodeIdParam = req.params.id as string;
|
||||
if (!requirePermission(req, res, 'node:manage', 'node', nodeIdParam)) return;
|
||||
if (!requirePaid(req, res)) return;
|
||||
// Reset-anchor is symmetric with `/api/fleet/sync-status` (admin-only).
|
||||
// Keeping read and write gated at the same role avoids a banner-invisible-to-the-actor
|
||||
// gap where a node-admin could call reset without ever seeing why.
|
||||
if (!requireAdmin(req, res)) return;
|
||||
try {
|
||||
const id = parseInt(nodeIdParam, 10);
|
||||
if (!Number.isFinite(id) || id <= 0) {
|
||||
res.status(400).json({ error: 'Invalid node id' });
|
||||
return;
|
||||
}
|
||||
const node = DatabaseService.getInstance().getNode(id);
|
||||
if (!node) {
|
||||
res.status(404).json({ error: 'Node not found' });
|
||||
return;
|
||||
}
|
||||
if (node.type !== 'remote' || node.mode !== 'proxy') {
|
||||
res.status(400).json({ error: 'Reset anchor only applies to proxy-mode remote nodes' });
|
||||
return;
|
||||
}
|
||||
if (!node.api_url || !node.api_token) {
|
||||
res.status(400).json({ error: 'Node is missing api_url or api_token' });
|
||||
return;
|
||||
}
|
||||
|
||||
const baseUrl = node.api_url.replace(/\/$/, '');
|
||||
let peerResponse: globalThis.Response;
|
||||
try {
|
||||
peerResponse = await fetch(`${baseUrl}/api/fleet/role/reanchor`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${node.api_token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ override: true }),
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
} catch (networkErr) {
|
||||
const message = getErrorMessage(networkErr, 'Failed to reach peer');
|
||||
console.warn(`[Nodes] Reset anchor unreachable for node ${id}: ${message}`);
|
||||
res.status(504).json({ error: `Peer unreachable: ${message}` });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!peerResponse.ok) {
|
||||
const status = peerResponse.status;
|
||||
const body = await peerResponse.json().catch(() => ({}));
|
||||
const peerError = (body as { error?: string })?.error
|
||||
?? `Peer returned HTTP ${status}`;
|
||||
if (status === 401 || status === 403) {
|
||||
console.warn(`[Nodes] Reset anchor rejected by peer ${id}: ${peerError}`);
|
||||
res.status(502).json({
|
||||
error: `Peer rejected the reanchor request: ${peerError}. The node's API token may need to be regenerated.`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
res.status(502).json({ error: peerError });
|
||||
return;
|
||||
}
|
||||
|
||||
DatabaseService.getInstance().clearFleetSyncStickyForNode(id);
|
||||
console.log(`[Nodes] Fleet-sync anchor reset on node ${id} ("${node.name}")`);
|
||||
res.json({ success: true });
|
||||
} catch (error: unknown) {
|
||||
console.error('Failed to reset fleet-sync anchor:', error);
|
||||
res.status(500).json({ error: getErrorMessage(error, 'Failed to reset fleet-sync anchor') });
|
||||
}
|
||||
});
|
||||
|
||||
nodesRouter.post('/:id/test', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const id = parseInt(req.params.id as string);
|
||||
|
||||
Reference in New Issue
Block a user