mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-18 14:33:19 +00:00
fix(fleet-sync): make control-identity-mismatch sticky and surface in UI (#1117)
Treat 409 CONTROL_IDENTITY_MISMATCH from a replica as a non-retriable
failure instead of looping the same 409 through the 5-minute retry
service forever and silently writing identical failure rows.
Backend
- DatabaseService: add `sticky_error_code`, `sticky_error_expected`,
`sticky_error_got` columns to `fleet_sync_status` via an idempotent
migration. New methods setFleetSyncSticky, getFleetSyncStickyCode,
clearFleetSyncStickyForNode. recordFleetSyncSuccess clears the sticky
flag on a clean push. getFailedSyncTargets SQL adds
`AND sticky_error_code IS NULL` so the retry loop skips sticky rows.
- FleetSyncService.executePushToNode: short-circuits at the top when
sticky is set (covers event-driven pushResourceAsync calls). On a 409
with code CONTROL_IDENTITY_MISMATCH, records the failure once and
pins sticky with the expected/got fingerprints carried in the 409 body.
- routes/nodes.ts: new POST /api/nodes/:id/fleet-sync/reset-anchor.
Admin + paid + node:manage. Proxies POST /api/fleet/role/reanchor to
the peer with `{override:true}` using the stored Bearer node_proxy
token. On peer 200, clears every sticky row for the node so the next
push re-anchors and resumes replication. Distinct 502 / 504 responses
for peer-rejected / peer-unreachable so the UI can show a useful toast.
Frontend
- New lib/fleetSyncApi.ts + hooks/useFleetSyncStatus.ts. Polling hook
(30s visibilityInterval) skips fetch when !isPaid.
- NodeManager.tsx: destructive banner per affected node listing both
fingerprints, with `Reset anchor on peer` and `Remove node` buttons.
Hidden for community-tier users via empty hook data.
- FleetConfiguration.tsx (Fleet -> Status): read-only `Policy sync`
SummaryRow per remote node card. In sync / degraded / paused with
a tooltip; no action buttons (the action lives in NodeManager).
Tests
- fleet-sync-service.test.ts: 4 new cases for sticky-set on first
mismatch, short-circuit on subsequent pushes, null fingerprints,
and non-mismatch failures not setting sticky.
- database-fleet-sync-sticky.test.ts (new): 6 cases pinning the DB
contract incl. retry-loop SQL filter and migration idempotency.
- nodes-fleet-sync-reset-anchor.test.ts (new): 6 cases covering
happy path, peer 401 -> 502, peer unreachable -> 504, local-node
rejection, unknown node id, and community-tier 403.
Gate parity (Directive 30): the new POST .../reset-anchor enforces
requireAdmin + requirePaid + node:manage (matches the existing read at
GET /api/fleet/sync-status). UI banner + SummaryRow only render when the
hook returns data, which it only does for paid-tier authed users. No
existing tier-gate file moved; this is greenfield parity.
Auth audit: the peer's POST /api/fleet/role/reanchor route already uses
requireAdmin, which accepts the central's stored node_proxy Bearer
token because authMiddleware maps `scope === 'node_proxy'` to
`req.user = { username: 'node-proxy', role: 'admin', userId: 0 }`.
No widening required.
Backend tsc clean. Frontend tsc -b clean. 59 fleet-sync tests pass; full
backend suite green minus the pre-existing Windows-only file-lock flake
on filesystem-backup.test.ts that reproduces unchanged on main.
This commit is contained in:
@@ -9,7 +9,7 @@ import { Badge } from './ui/badge';
|
||||
import { Separator } from './ui/separator';
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from './ui/table';
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from './ui/tooltip';
|
||||
import { Plus, Trash2, Wifi, WifiOff, Star, Pencil, Monitor, Globe, Copy, KeyRound, Check, Calendar, RefreshCw, Terminal } from 'lucide-react';
|
||||
import { AlertTriangle, Plus, Trash2, Wifi, WifiOff, Star, Pencil, Monitor, Globe, Copy, KeyRound, Check, Calendar, RefreshCw, Terminal } from 'lucide-react';
|
||||
import { formatTimeUntil, formatTimeAgo } from '@/lib/relativeTime';
|
||||
import { SettingsPrimaryButton } from './settings/SettingsActions';
|
||||
import { useMastheadStats } from './settings/MastheadStatsContext';
|
||||
@@ -17,6 +17,8 @@ import { NodeLabelPicker } from './blueprints/NodeLabelPicker';
|
||||
import { useLicense } from '@/context/LicenseContext';
|
||||
import { useAuth } from '@/context/AuthContext';
|
||||
import { useNodeActions, type NodeTestInfo } from './nodes/useNodeActions';
|
||||
import { useFleetSyncStatus } from '@/hooks/useFleetSyncStatus';
|
||||
import { resetFleetSyncAnchor, STICKY_CONTROL_IDENTITY_MISMATCH } from '@/lib/fleetSyncApi';
|
||||
|
||||
interface NodeSchedulingSummary {
|
||||
active_tasks: number;
|
||||
@@ -60,6 +62,48 @@ export function NodeManager() {
|
||||
onTestResult: (result) => setTestResult(result),
|
||||
});
|
||||
|
||||
const { statuses: syncStatuses, refresh: refreshSyncStatuses } = useFleetSyncStatus();
|
||||
const [resettingAnchor, setResettingAnchor] = useState<number | null>(null);
|
||||
|
||||
// Per-node aggregate of CONTROL_IDENTITY_MISMATCH sticky errors. All resources
|
||||
// for one peer share the same root cause (the peer's cached fingerprint), so
|
||||
// collapse to one entry per node id and surface a single banner.
|
||||
const anchorMismatches = useMemo(() => {
|
||||
const byNode = new Map<number, { expected: string | null; got: string | null; resources: string[] }>();
|
||||
for (const row of syncStatuses) {
|
||||
if (row.sticky_error_code !== STICKY_CONTROL_IDENTITY_MISMATCH) continue;
|
||||
const existing = byNode.get(row.node_id);
|
||||
if (existing) {
|
||||
existing.resources.push(row.resource);
|
||||
if (!existing.expected && row.sticky_error_expected) existing.expected = row.sticky_error_expected;
|
||||
if (!existing.got && row.sticky_error_got) existing.got = row.sticky_error_got;
|
||||
} else {
|
||||
byNode.set(row.node_id, {
|
||||
expected: row.sticky_error_expected,
|
||||
got: row.sticky_error_got,
|
||||
resources: [row.resource],
|
||||
});
|
||||
}
|
||||
}
|
||||
return Array.from(byNode.entries()).map(([nodeId, agg]) => {
|
||||
const node = nodes.find((n) => n.id === nodeId);
|
||||
return { nodeId, node, ...agg };
|
||||
}).filter((entry) => entry.node !== undefined);
|
||||
}, [syncStatuses, nodes]);
|
||||
|
||||
const handleResetAnchor = async (nodeId: number) => {
|
||||
setResettingAnchor(nodeId);
|
||||
try {
|
||||
await resetFleetSyncAnchor(nodeId);
|
||||
toast.success('Anchor reset. Security policy sync will resume on the next push.');
|
||||
refreshSyncStatuses();
|
||||
} catch (error) {
|
||||
toast.error((error as Error).message || 'Failed to reset anchor on peer');
|
||||
} finally {
|
||||
setResettingAnchor(null);
|
||||
}
|
||||
};
|
||||
|
||||
const fetchSchedulingSummary = useCallback(async () => {
|
||||
try {
|
||||
const res = await apiFetch('/nodes/scheduling-summary', { localOnly: true });
|
||||
@@ -188,6 +232,51 @@ export function NodeManager() {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Sync issues: surfaces FleetSync sticky errors (currently CONTROL_IDENTITY_MISMATCH). */}
|
||||
{anchorMismatches.length > 0 && (
|
||||
<div className="space-y-3">
|
||||
{anchorMismatches.map(({ nodeId, node, expected, got, resources }) => (
|
||||
<div
|
||||
key={nodeId}
|
||||
className="rounded-lg border border-destructive/40 bg-destructive/10 p-3 text-sm text-destructive flex items-start gap-3 shadow-card-bevel"
|
||||
>
|
||||
<AlertTriangle className="h-4 w-4 mt-0.5 shrink-0" strokeWidth={1.75} />
|
||||
<div className="min-w-0 flex-1 space-y-2">
|
||||
<div className="font-medium">
|
||||
Node "{node?.name ?? `id ${nodeId}`}" is anchored to another central
|
||||
</div>
|
||||
<div className="text-xs leading-relaxed text-destructive/90">
|
||||
Security policy sync is paused for {resources.join(', ')}.
|
||||
{expected && got && (
|
||||
<> This peer is anchored to <span className="font-mono">{expected}</span>; this central is <span className="font-mono">{got}</span>.</>
|
||||
)}
|
||||
{' '}Reset the anchor on the peer to resume sync, or remove the node from this fleet.
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2 pt-1">
|
||||
<Button
|
||||
size="sm"
|
||||
variant="destructive"
|
||||
onClick={() => handleResetAnchor(nodeId)}
|
||||
disabled={resettingAnchor === nodeId}
|
||||
>
|
||||
{resettingAnchor === nodeId ? 'Resetting...' : 'Reset anchor on peer'}
|
||||
</Button>
|
||||
{node && !node.is_default && (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => openDelete(node)}
|
||||
>
|
||||
Remove node
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Nodes Table */}
|
||||
<div className="rounded-md border overflow-x-auto w-full">
|
||||
<Table>
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { useState, useEffect, useCallback, useMemo } from 'react';
|
||||
import { apiFetch } from '@/lib/api';
|
||||
import { formatCount } from '@/lib/utils';
|
||||
import { Card, CardContent } from '@/components/ui/card';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip';
|
||||
import {
|
||||
Bell, Zap, Shield, HardDrive, WifiOff, CheckCircle2,
|
||||
Bell, Zap, Shield, HardDrive, WifiOff, CheckCircle2, RefreshCw,
|
||||
} from 'lucide-react';
|
||||
import { useLicense } from '@/context/LicenseContext';
|
||||
import { useFleetSyncStatus } from '@/hooks/useFleetSyncStatus';
|
||||
import { STICKY_CONTROL_IDENTITY_MISMATCH, type FleetSyncStatus } from '@/lib/fleetSyncApi';
|
||||
import type { ConfigurationStatusPayload } from '@/components/dashboard';
|
||||
|
||||
interface FleetNodeConfiguration {
|
||||
@@ -31,7 +34,69 @@ function SummaryRow({ icon: Icon, label, value }: {
|
||||
);
|
||||
}
|
||||
|
||||
function NodeCard({ node, isPaid }: { node: FleetNodeConfiguration; isPaid: boolean }) {
|
||||
type PolicySyncState =
|
||||
| { kind: 'in_sync' }
|
||||
| { kind: 'degraded'; lastError: string | null }
|
||||
| { kind: 'paused' };
|
||||
|
||||
function derivePolicySyncState(rows: FleetSyncStatus[]): PolicySyncState | null {
|
||||
if (rows.length === 0) return null;
|
||||
for (const row of rows) {
|
||||
if (row.sticky_error_code === STICKY_CONTROL_IDENTITY_MISMATCH) {
|
||||
return { kind: 'paused' };
|
||||
}
|
||||
}
|
||||
let degradedError: string | null = null;
|
||||
let hasSuccess = false;
|
||||
for (const row of rows) {
|
||||
if (row.last_success_at !== null) hasSuccess = true;
|
||||
if (
|
||||
row.last_failure_at !== null
|
||||
&& (row.last_success_at === null || row.last_failure_at > row.last_success_at)
|
||||
) {
|
||||
degradedError = row.last_error;
|
||||
}
|
||||
}
|
||||
if (degradedError !== null) return { kind: 'degraded', lastError: degradedError };
|
||||
if (hasSuccess) return { kind: 'in_sync' };
|
||||
return null;
|
||||
}
|
||||
|
||||
function PolicySyncRow({ state }: { state: PolicySyncState }) {
|
||||
if (state.kind === 'in_sync') {
|
||||
return <SummaryRow icon={RefreshCw} label="Policy sync" value="In sync" />;
|
||||
}
|
||||
const tooltip = state.kind === 'paused'
|
||||
? 'Anchored to another central. Open Settings → Nodes to reset the anchor or remove the node.'
|
||||
: (state.lastError ?? 'Last push to this node failed.');
|
||||
return (
|
||||
<div className="flex items-center gap-2 py-0.5">
|
||||
<RefreshCw className="h-3 w-3 shrink-0 text-stat-icon" strokeWidth={1.5} />
|
||||
<span className="text-xs flex-1 text-stat-subtitle">Policy sync</span>
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger>
|
||||
<span
|
||||
className={
|
||||
'inline-flex items-center rounded border px-1.5 py-0 text-[10px] leading-3 font-mono uppercase '
|
||||
+ 'border-amber-500/40 bg-amber-500/10 text-amber-600 dark:text-amber-400'
|
||||
}
|
||||
>
|
||||
{state.kind === 'paused' ? 'paused' : 'degraded'}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{tooltip}</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function NodeCard({ node, isPaid, policySyncState }: {
|
||||
node: FleetNodeConfiguration;
|
||||
isPaid: boolean;
|
||||
policySyncState: PolicySyncState | null;
|
||||
}) {
|
||||
const isRemote = node.type === 'remote';
|
||||
if (!node.configuration) {
|
||||
return (
|
||||
@@ -102,6 +167,7 @@ function NodeCard({ node, isPaid }: { node: FleetNodeConfiguration; isPaid: bool
|
||||
)}
|
||||
<SummaryRow icon={HardDrive} label="Crash detect"
|
||||
value={thresholds.globalCrash ? 'On' : 'Off'} />
|
||||
{policySyncState && <PolicySyncRow state={policySyncState} />}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
@@ -110,10 +176,26 @@ function NodeCard({ node, isPaid }: { node: FleetNodeConfiguration; isPaid: bool
|
||||
|
||||
export function FleetConfiguration() {
|
||||
const { isPaid } = useLicense();
|
||||
const { statuses: syncStatuses } = useFleetSyncStatus();
|
||||
const [nodes, setNodes] = useState<FleetNodeConfiguration[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const syncStateByNode = useMemo(() => {
|
||||
const byNode = new Map<number, FleetSyncStatus[]>();
|
||||
for (const row of syncStatuses) {
|
||||
const list = byNode.get(row.node_id);
|
||||
if (list) list.push(row);
|
||||
else byNode.set(row.node_id, [row]);
|
||||
}
|
||||
const out = new Map<number, PolicySyncState>();
|
||||
for (const [nodeId, rows] of byNode) {
|
||||
const state = derivePolicySyncState(rows);
|
||||
if (state) out.set(nodeId, state);
|
||||
}
|
||||
return out;
|
||||
}, [syncStatuses]);
|
||||
|
||||
const fetchData = useCallback(async () => {
|
||||
try {
|
||||
const res = await apiFetch('/fleet/configuration', { localOnly: true });
|
||||
@@ -170,7 +252,14 @@ export function FleetConfiguration() {
|
||||
|
||||
return (
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-3 gap-4 p-1">
|
||||
{nodes.map(node => <NodeCard key={node.id} node={node} isPaid={isPaid} />)}
|
||||
{nodes.map(node => (
|
||||
<NodeCard
|
||||
key={node.id}
|
||||
node={node}
|
||||
isPaid={isPaid}
|
||||
policySyncState={syncStateByNode.get(node.id) ?? null}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user