mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-12 03:36:59 +00:00
fix(fleet): resolve Stop-by-label stack labels across all nodes (#1382)
* fix(fleet): resolve Stop-by-label stack labels across all nodes The Fleet Actions "Stop by label" card only ever saw the control node's own stack labels. The stack-label routes are proxied, so each node stores its labels in its own database and the control holds no mirror for remote nodes. The suggestions, match-preview, and fleet-stop endpoints all read that nonexistent mirror, so remote-only labels were invisible and remote stacks were skipped before the remote was ever asked. Make all three authoritative across the fleet with a new collectFleetLabelSummaries helper: the local node reads its own database, and each remote is queried live through its labels and label-assignments endpoints over the proxy, with fail-closed parsing and per-node reachability. fleet-stop drops the mirror pre-check and always calls each reachable remote's local-stop receiver, reporting unreachable nodes at the node level so they never block the reachable ones. The picker now surfaces remote-only labels, aggregates shared names once with combined counts and the carrying node names, and flags incomplete coverage when a node is unreachable. The preview groups matches per node, lists unreachable nodes separately, and distinguishes no matching stacks from "label exists but no stacks" from "remote unavailable". * fix(fleet): harden Stop-by-label card against malformed responses Guard the match-preview and fleet-stop response bodies so a malformed but 200 reply degrades instead of crashing or misreporting. match-preview now validates the per-node shape before rendering (the preview reads it outside any try/catch) and logs a malformed body; fleet-stop distinguishes a non-array results body (a server bug, now logged and surfaced as an unexpected-response error) from a genuine empty fleet, and guards per-node stackResults. Docs: an unreachable or errored remote is reported once per node, not as a per-stack error row.
This commit is contained in:
Anso
@@ -109,6 +109,32 @@ async function createAssignedLabel(name: string, stacks: string[]) {
|
||||
return created.body as { id: number; node_id: number; name: string };
|
||||
}
|
||||
|
||||
// Stub the control's server-side fan-out to a remote node. The remote's
|
||||
// authoritative stack-label state is read live via /api/labels +
|
||||
// /api/labels/assignments; this returns canned bodies for those two URLs and
|
||||
// passes any other URL (e.g. the fleet-stop local-stop receiver) to `other`.
|
||||
function mockRemoteLabelFetch(opts: {
|
||||
labels: { name: string; color?: string }[];
|
||||
assignments: Record<string, string[]>; // stackName -> label names
|
||||
other?: (url: string) => Response;
|
||||
}) {
|
||||
return vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: Parameters<typeof fetch>[0]) => {
|
||||
const url = String(input);
|
||||
if (url.endsWith('/api/labels')) {
|
||||
const body = opts.labels.map((l, i) => ({ id: i + 1, node_id: 9, name: l.name, color: l.color ?? 'teal' }));
|
||||
return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
if (url.endsWith('/api/labels/assignments')) {
|
||||
const body: Record<string, { name: string; color: string }[]> = {};
|
||||
for (const [stack, names] of Object.entries(opts.assignments)) {
|
||||
body[stack] = names.map(n => ({ name: n, color: 'teal' }));
|
||||
}
|
||||
return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
return opts.other ? opts.other(url) : new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
});
|
||||
}
|
||||
|
||||
describe('POST /api/fleet/labels/match-preview', () => {
|
||||
it('returns 401 without auth', async () => {
|
||||
const res = await request(app)
|
||||
@@ -164,6 +190,99 @@ describe('POST /api/fleet/labels/match-preview', () => {
|
||||
expect(res.body.matchedNodes).toBe(0);
|
||||
expect(res.body.matchedStacks).toBe(0);
|
||||
});
|
||||
|
||||
it('includes remote stacks grouped per node, drops stale local stacks, and flags reachability', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const local = db.createLabel(localId, 'media', 'teal');
|
||||
db.setStackLabels('alpha', localId, [local.id]);
|
||||
// A stale local assignment to a stack that no longer exists on disk
|
||||
// (not in mockFsStacks) must be filtered out, not counted.
|
||||
db.setStackLabels('media-ghost', localId, [local.id]);
|
||||
const remoteId = db.addNode({
|
||||
name: 'media-remote', type: 'remote', api_url: 'http://media.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
// The control DB holds no 'media' row for the remote; its stacks come live.
|
||||
mockRemoteLabelFetch({ labels: [{ name: 'media' }], assignments: { sonarr: ['media'], radarr: ['media'] } });
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/match-preview')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: 'media' });
|
||||
expect(res.status).toBe(200);
|
||||
// 1 live local stack (the ghost is dropped) + 2 remote stacks.
|
||||
expect(res.body.matchedStacks).toBe(3);
|
||||
expect(res.body.matchedNodes).toBe(2);
|
||||
expect(res.body.unreachableNodes).toBe(0);
|
||||
const localRow = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === localId);
|
||||
expect(localRow.stackCount).toBe(1);
|
||||
expect(localRow.stackNames).toEqual(['alpha']);
|
||||
const remote = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remote.reachable).toBe(true);
|
||||
expect(remote.labelExists).toBe(true);
|
||||
expect(remote.stackNames.sort()).toEqual(['radarr', 'sonarr']);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('distinguishes label-exists-no-stacks from an unreachable remote', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
// Local carries the label but assigns no stacks to it.
|
||||
db.createLabel(localId, 'empty-label', 'teal');
|
||||
const remoteId = db.addNode({
|
||||
name: 'unreach-remote', type: 'remote', api_url: 'http://unreach.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('connect ECONNREFUSED'));
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/match-preview')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: 'empty-label' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.unreachableNodes).toBe(1);
|
||||
const localRow = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === localId);
|
||||
expect(localRow.reachable).toBe(true);
|
||||
expect(localRow.labelExists).toBe(true);
|
||||
expect(localRow.stackCount).toBe(0);
|
||||
const remoteRow = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remoteRow.reachable).toBe(false);
|
||||
expect(remoteRow.error).toMatch(/ECONNREFUSED|reach/i);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('surfaces the remote error body when it serves labels but rejects assignments', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
db.setStackLabels('alpha', localId, [db.createLabel(localId, 'ok-label', 'teal').id]);
|
||||
const remoteId = db.addNode({
|
||||
name: 'split-remote', type: 'remote', api_url: 'http://split.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: Parameters<typeof fetch>[0]) => {
|
||||
const url = String(input);
|
||||
if (url.endsWith('/api/labels')) {
|
||||
return new Response(JSON.stringify([]), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
// /api/labels/assignments rejects with a real error body.
|
||||
return new Response(JSON.stringify({ error: 'token expired' }), { status: 403, headers: { 'content-type': 'application/json' } });
|
||||
});
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/match-preview')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: 'ok-label' });
|
||||
expect(res.status).toBe(200);
|
||||
const remoteRow = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remoteRow.reachable).toBe(false);
|
||||
// The remote's own message is surfaced, not a bare status number.
|
||||
expect(remoteRow.error).toBe('token expired');
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/fleet/labels/suggestions', () => {
|
||||
@@ -192,22 +311,23 @@ describe('GET /api/fleet/labels/suggestions', () => {
|
||||
expect(Array.isArray(res.body.suggestions)).toBe(true);
|
||||
});
|
||||
|
||||
it('aggregates stack labels across nodes, includes unassigned ones, and excludes node labels', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
it('surfaces remote-only stack labels and aggregates shared names across nodes', async () => {
|
||||
const localNode = db.getNodes().find(n => n.is_default)!;
|
||||
const remoteId = db.addNode({
|
||||
name: 'sugg-remote', type: 'remote', api_url: 'http://sugg.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
// Same-named stack label on both nodes, each with one assigned stack.
|
||||
const local = db.createLabel(localId, 'shared-prod', 'teal');
|
||||
db.setStackLabels('alpha', localId, [local.id]);
|
||||
const remote = db.createLabel(remoteId, 'shared-prod', 'teal');
|
||||
db.setStackLabels('beta', remoteId, [remote.id]);
|
||||
// A stack label with no assignments still belongs to the picker.
|
||||
db.createLabel(localId, 'unused-stack-label', 'sky');
|
||||
// A node-only label that must never surface as a stop target.
|
||||
db.addNodeLabel(remoteId, 'edge-only');
|
||||
// Local node: 'shared-prod' on stack 'alpha' (alpha is in mockFsStacks).
|
||||
const local = db.createLabel(localNode.id, 'shared-prod', 'teal');
|
||||
db.setStackLabels('alpha', localNode.id, [local.id]);
|
||||
// Remote node: queried live. It carries 'shared-prod' on TWO stacks plus a
|
||||
// remote-only 'edge'. NONE of this is mirrored in the control DB. The
|
||||
// asymmetric remote count (2) makes a sum-vs-concatenate bug observable.
|
||||
mockRemoteLabelFetch({
|
||||
labels: [{ name: 'shared-prod' }, { name: 'edge' }],
|
||||
assignments: { sonarr: ['shared-prod'], lidarr: ['shared-prod'], radarr: ['edge'] },
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
@@ -216,36 +336,42 @@ describe('GET /api/fleet/labels/suggestions', () => {
|
||||
|
||||
const names = res.body.suggestions.map((s: { name: string }) => s.name);
|
||||
expect(names).toContain('shared-prod');
|
||||
expect(names).toContain('unused-stack-label');
|
||||
expect(names).not.toContain('edge-only');
|
||||
// Remote-only label is visible despite no control-DB row.
|
||||
expect(names).toContain('edge');
|
||||
// Sorted by name.
|
||||
expect(names).toEqual([...names].sort((a: string, b: string) => a.localeCompare(b)));
|
||||
expect(res.body.partial).toBe(false);
|
||||
expect(res.body.unreachableNodes).toBe(0);
|
||||
|
||||
const shared = res.body.suggestions.find((s: { name: string }) => s.name === 'shared-prod');
|
||||
expect(shared.scope).toBe('stack');
|
||||
expect(shared.nodeCount).toBe(2);
|
||||
expect(shared.stackCount).toBe(2);
|
||||
// 1 local stack + 2 remote stacks, summed (not deduped to the node count).
|
||||
expect(shared.stackCount).toBe(3);
|
||||
expect([...shared.nodes].sort()).toEqual([localNode.name, 'sugg-remote'].sort());
|
||||
|
||||
const unused = res.body.suggestions.find((s: { name: string }) => s.name === 'unused-stack-label');
|
||||
expect(unused.nodeCount).toBe(1);
|
||||
expect(unused.stackCount).toBe(0);
|
||||
const edge = res.body.suggestions.find((s: { name: string }) => s.name === 'edge');
|
||||
expect(edge.nodeCount).toBe(1);
|
||||
expect(edge.stackCount).toBe(1);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('counts only the stack-label side when a node label shares the name', async () => {
|
||||
it('never folds node labels into the picker even when one shares a stack-label name', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const remoteId = db.addNode({
|
||||
name: 'collision-remote', type: 'remote', api_url: 'http://collision.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
// A stack label 'prod' lives on the local node only...
|
||||
// A stack label 'prod' lives on the local node...
|
||||
const stackLabel = db.createLabel(localId, 'prod', 'teal');
|
||||
db.setStackLabels('alpha', localId, [stackLabel.id]);
|
||||
// ...while a node label of the same name lives on the remote node.
|
||||
// ...while a node label of the same name lives on the remote node. The
|
||||
// remote's stack-label endpoint (/api/labels) never returns node labels.
|
||||
db.addNodeLabel(remoteId, 'prod');
|
||||
mockRemoteLabelFetch({ labels: [], assignments: {} });
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
@@ -260,6 +386,75 @@ describe('GET /api/fleet/labels/suggestions', () => {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('excludes labels whose only assigned stacks no longer exist on disk', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const ghost = db.createLabel(localId, 'ghost-only', 'teal');
|
||||
// 'vanished-stack' is not in mockFsStacks, so the fs-present filter drops it,
|
||||
// leaving the label with zero matching stacks fleet-wide.
|
||||
db.setStackLabels('vanished-stack', localId, [ghost.id]);
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
.set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
const names = res.body.suggestions.map((s: { name: string }) => s.name);
|
||||
expect(names).not.toContain('ghost-only');
|
||||
});
|
||||
|
||||
it('reports unreachable remotes as partial without dropping reachable suggestions', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const local = db.createLabel(localId, 'live-label', 'teal');
|
||||
db.setStackLabels('alpha', localId, [local.id]);
|
||||
const remoteId = db.addNode({
|
||||
name: 'down-remote', type: 'remote', api_url: 'http://down.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('connect ECONNREFUSED'));
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
.set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.partial).toBe(true);
|
||||
expect(res.body.unreachableNodes).toBe(1);
|
||||
const names = res.body.suggestions.map((s: { name: string }) => s.name);
|
||||
expect(names).toContain('live-label');
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('fails closed: a remote returning a malformed label body is marked unreachable', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const local = db.createLabel(localId, 'safe-label', 'teal');
|
||||
db.setStackLabels('alpha', localId, [local.id]);
|
||||
const remoteId = db.addNode({
|
||||
name: 'garbage-remote', type: 'remote', api_url: 'http://garbage.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: Parameters<typeof fetch>[0]) => {
|
||||
const url = String(input);
|
||||
if (url.endsWith('/api/labels')) {
|
||||
// Not an array: the fail-closed parser must reject the whole node.
|
||||
return new Response(JSON.stringify({ not: 'an array' }), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
return new Response(JSON.stringify({}), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
});
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
.set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.partial).toBe(true);
|
||||
expect(res.body.unreachableNodes).toBe(1);
|
||||
// The garbage remote contributes nothing; the local label still surfaces.
|
||||
const names = res.body.suggestions.map((s: { name: string }) => s.name);
|
||||
expect(names).toContain('safe-label');
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/fleet/prune/estimate', () => {
|
||||
@@ -438,6 +633,64 @@ describe('POST /api/fleet/labels/fleet-stop remote leg', () => {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('calls the remote local-stop even when the control DB has no remote label row', async () => {
|
||||
const remoteId = db.addNode({
|
||||
name: 'remote-nomirror', type: 'remote', api_url: 'http://remote-nomirror.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
// Deliberately seed NO label/assignment for the remote in the control DB.
|
||||
// The old mirror pre-check would have skipped this node entirely.
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true, status: 200,
|
||||
json: async () => ({ matched: true, results: [{ stackName: 'remote-stack', success: true }] }),
|
||||
} as unknown as Response);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-stop')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: 'remote-only-label' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const urls = fetchSpy.mock.calls.map(c => String(c[0]));
|
||||
expect(urls.some(u => u.endsWith('/api/fleet-actions/labels/local-stop'))).toBe(true);
|
||||
const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
|
||||
expect(remoteRow.reachable).toBe(true);
|
||||
expect(remoteRow.matched).toBe(true);
|
||||
expect(remoteRow.stackResults).toEqual([{ stackName: 'remote-stack', success: true }]);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('reports an unreachable remote at the node level without blocking the local stop', async () => {
|
||||
const localLabel = await createAssignedLabel('local-live', ['alpha']);
|
||||
const remoteId = db.addNode({
|
||||
name: 'remote-down', type: 'remote', api_url: 'http://remote-down.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('connect ECONNREFUSED'));
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-stop')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: localLabel.name });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const localResult = res.body.results.find((r: { nodeId: number }) => r.nodeId === localLabel.node_id);
|
||||
expect(localResult.reachable).toBe(true);
|
||||
expect(localResult.matched).toBe(true);
|
||||
expect(localResult.stackResults).toEqual([{ stackName: 'alpha', success: true }]);
|
||||
const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
|
||||
expect(remoteRow.reachable).toBe(false);
|
||||
expect(remoteRow.matched).toBe(false);
|
||||
expect(remoteRow.stackResults).toEqual([]);
|
||||
expect(remoteRow.error).toMatch(/ECONNREFUSED|reach/i);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/fleet/labels/fleet-stop with dryRun: true', () => {
|
||||
|
||||
@@ -158,7 +158,50 @@ describe('POST /api/fleet/labels/fleet-stop (pilot-agent dispatch)', () => {
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const pilotResult = res.body.results.find((r: { nodeId: number }) => r.nodeId === pilotNodeId);
|
||||
expect(pilotResult.stackResults[0].error).toMatch(/pilot tunnel/i);
|
||||
// A node with no reachable target is reported at the node level; there is no
|
||||
// control-side mirror to enumerate per-stack rows for an unreachable remote.
|
||||
expect(pilotResult.reachable).toBe(false);
|
||||
expect(pilotResult.matched).toBe(false);
|
||||
expect(pilotResult.stackResults).toEqual([]);
|
||||
expect(pilotResult.error).toMatch(/pilot tunnel/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/fleet/labels/suggestions (pilot-agent summary fan-out)', () => {
|
||||
it('reads each remote label set through the proxy target with conditional Authorization', async () => {
|
||||
mockTargets();
|
||||
const calls: Array<{ url: string; auth: string | undefined }> = [];
|
||||
mockFetch((url, init) => {
|
||||
const headers = (init?.headers as Record<string, string>) ?? {};
|
||||
calls.push({ url, auth: headers.Authorization });
|
||||
if (url.endsWith('/api/labels')) {
|
||||
return new Response(JSON.stringify([{ id: 1, node_id: 0, name: 'summary-label', color: 'teal' }]), {
|
||||
status: 200, headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
}
|
||||
if (url.endsWith('/api/labels/assignments')) {
|
||||
return new Response(JSON.stringify({ svc: [{ id: 1, node_id: 0, name: 'summary-label', color: 'teal' }] }), {
|
||||
status: 200, headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
}
|
||||
return new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
.set('Authorization', authHeader);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
// Both fan-out legs (labels + assignments) must use the proxy target with the
|
||||
// right auth: pilot loopback carries no Authorization, proxy carries Bearer.
|
||||
const pilotCalls = calls.filter(c => c.url.startsWith(PILOT_LOOPBACK));
|
||||
expect(pilotCalls.map(c => c.url).sort()).toEqual([`${PILOT_LOOPBACK}/api/labels`, `${PILOT_LOOPBACK}/api/labels/assignments`]);
|
||||
expect(pilotCalls.every(c => c.auth === undefined)).toBe(true);
|
||||
const proxyCalls = calls.filter(c => c.url.startsWith(PROXY_URL));
|
||||
expect(proxyCalls.map(c => c.url).sort()).toEqual([`${PROXY_URL}/api/labels`, `${PROXY_URL}/api/labels/assignments`]);
|
||||
expect(proxyCalls.every(c => c.auth === `Bearer ${PROXY_TOKEN}`)).toBe(true);
|
||||
// The label only exists on the remotes (live fan-out), never in the control DB.
|
||||
expect(res.body.suggestions.some((s: { name: string }) => s.name === 'summary-label')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { DatabaseService, type Node } from '../services/DatabaseService';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { formatNoTargetError } from '../utils/remoteTarget';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
export interface NodeLabelSummaryEntry {
|
||||
name: string;
|
||||
color: string;
|
||||
stackNames: string[];
|
||||
}
|
||||
|
||||
export interface NodeLabelSummary {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
reachable: boolean;
|
||||
labels: NodeLabelSummaryEntry[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// Per-node fetch budget. Suggestions load on card mount and match-preview runs
|
||||
// on a 500ms debounce, so keep this short: a wedged or slow remote should fall
|
||||
// to "unreachable" fast rather than stall the whole fleet readout.
|
||||
const SUMMARY_FETCH_TIMEOUT_MS = 5000;
|
||||
|
||||
/**
|
||||
* Authoritative stack-label summary for one node's own labels.
|
||||
*
|
||||
* Stack names are filtered to those still present on disk, matching the
|
||||
* `/api/labels/assignments` cleanup (routes/labels.ts) and runLocalLabelStop's
|
||||
* validStacks filter (helpers/fleetLabelStop.ts), so the preview/suggestion
|
||||
* counts equal the set a stop would actually act on.
|
||||
*/
|
||||
export async function readLocalLabelSummary(nodeId: number): Promise<NodeLabelSummaryEntry[]> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const labels = db.getLabels(nodeId);
|
||||
if (labels.length === 0) return [];
|
||||
const fsStacks = new Set(await FileSystemService.getInstance(nodeId).getStacks());
|
||||
return labels.map(label => ({
|
||||
name: label.name,
|
||||
color: label.color,
|
||||
stackNames: db.getStacksForLabel(label.id, nodeId).filter(name => fsStacks.has(name)),
|
||||
}));
|
||||
}
|
||||
|
||||
// Fail-closed parser for a remote `/api/labels` body (Label[]). A malformed
|
||||
// shape returns null so the caller marks the node unreachable rather than
|
||||
// flowing partial data into the aggregate counts.
|
||||
function parseRemoteLabels(value: unknown): { name: string; color: string }[] | null {
|
||||
if (!Array.isArray(value)) return null;
|
||||
const out: { name: string; color: string }[] = [];
|
||||
for (const item of value) {
|
||||
if (!item || typeof item !== 'object') return null;
|
||||
const label = item as Record<string, unknown>;
|
||||
if (typeof label.name !== 'string' || typeof label.color !== 'string') return null;
|
||||
out.push({ name: label.name, color: label.color });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Fail-closed parser for a remote `/api/labels/assignments` body
|
||||
// (Record<stackName, Label[]>). Collapses to label-name -> stack-name lists.
|
||||
function parseRemoteAssignments(value: unknown): Map<string, string[]> | null {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
||||
const stacksByLabel = new Map<string, string[]>();
|
||||
for (const [stackName, labels] of Object.entries(value as Record<string, unknown>)) {
|
||||
if (!Array.isArray(labels)) return null;
|
||||
for (const label of labels) {
|
||||
if (!label || typeof label !== 'object') return null;
|
||||
const name = (label as Record<string, unknown>).name;
|
||||
if (typeof name !== 'string') return null;
|
||||
const list = stacksByLabel.get(name) ?? [];
|
||||
list.push(stackName);
|
||||
stacksByLabel.set(name, list);
|
||||
}
|
||||
}
|
||||
return stacksByLabel;
|
||||
}
|
||||
|
||||
async function summarizeRemoteNode(node: Node): Promise<NodeLabelSummary> {
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target) {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: formatNoTargetError(node) };
|
||||
}
|
||||
const base = target.apiUrl.replace(/\/$/, '');
|
||||
// Conditional Bearer only: pilot-agent targets carry an empty token and reach
|
||||
// the remote over the loopback tunnel without an Authorization header.
|
||||
const headers: Record<string, string> = {};
|
||||
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
|
||||
try {
|
||||
const [labelsRes, assignmentsRes] = await Promise.all([
|
||||
fetch(`${base}/api/labels`, { headers, signal: AbortSignal.timeout(SUMMARY_FETCH_TIMEOUT_MS) }),
|
||||
fetch(`${base}/api/labels/assignments`, { headers, signal: AbortSignal.timeout(SUMMARY_FETCH_TIMEOUT_MS) }),
|
||||
]);
|
||||
if (!labelsRes.ok || !assignmentsRes.ok) {
|
||||
// Surface the remote's own error body (e.g. a token/tier message) the same
|
||||
// way the fleet-stop leg does, so a reached-but-erroring node reports its
|
||||
// real cause rather than a bare status number.
|
||||
const failed = labelsRes.ok ? assignmentsRes : labelsRes;
|
||||
const errBody = (await failed.json().catch(() => ({}))) as { error?: string };
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: errBody.error || `Remote returned ${failed.status}` };
|
||||
}
|
||||
const labels = parseRemoteLabels(await labelsRes.json().catch(() => null));
|
||||
const stacksByLabel = parseRemoteAssignments(await assignmentsRes.json().catch(() => null));
|
||||
if (!labels || !stacksByLabel) {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: 'Invalid label response from remote node' };
|
||||
}
|
||||
const summary = labels.map(label => ({
|
||||
name: label.name,
|
||||
color: label.color,
|
||||
stackNames: stacksByLabel.get(label.name) ?? [],
|
||||
}));
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: true, labels: summary };
|
||||
} catch (err) {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: getErrorMessage(err, 'Failed to reach remote node') };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative fleet-wide stack-label state: the local node from its own DB,
|
||||
* each remote queried live through its `/api/labels` + `/api/labels/assignments`
|
||||
* endpoints via the proxy target. Replaces the old central-DB read, which only
|
||||
* ever held the local node's labels (remote label rows are never mirrored to the
|
||||
* control). A node that cannot be reached, or that answers with a non-ok or
|
||||
* unusable response, is `reachable: false` with an error (its real cause) and
|
||||
* never blocks the reachable ones; reachable nodes carry complete `labels`.
|
||||
*/
|
||||
export async function collectFleetLabelSummaries(): Promise<NodeLabelSummary[]> {
|
||||
const nodes = DatabaseService.getInstance().getNodes();
|
||||
return Promise.all(nodes.map(async (node): Promise<NodeLabelSummary> => {
|
||||
if (node.type === 'local') {
|
||||
try {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: true, labels: await readLocalLabelSummary(node.id) };
|
||||
} catch (err) {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: getErrorMessage(err, 'Failed to read local labels') };
|
||||
}
|
||||
}
|
||||
return summarizeRemoteNode(node);
|
||||
}));
|
||||
}
|
||||
+69
-75
@@ -37,6 +37,7 @@ import { NotificationService } from '../services/NotificationService';
|
||||
import { invalidateNodeCaches, invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
|
||||
import { activeBulkActions } from './labels';
|
||||
import { runLocalLabelStop, type LabelLocalStopResponse, type StackStopResult } from '../helpers/fleetLabelStop';
|
||||
import { collectFleetLabelSummaries } from '../helpers/fleetLabelSummary';
|
||||
import { buildLocalConfigurationStatus, type ConfigurationStatus } from './dashboard';
|
||||
import { buildLocalGraph, mergeFleetGraph, isLocalDependencyGraph, type FleetNodeGraphResult } from '../services/DependencyGraphService';
|
||||
import { PROXY_TIER_HEADER } from '../services/license-headers';
|
||||
@@ -1237,13 +1238,25 @@ fleetRouter.delete('/update-status', authMiddleware, async (req: Request, res: R
|
||||
// lives here so it sits behind the `/api/fleet/` proxy-exempt prefix.
|
||||
|
||||
// Attribute one error to every stack a node was supposed to act on. Used for
|
||||
// the fleet-stop failure paths (no proxy target, non-ok remote, transport
|
||||
// error, local exception) so each stack carries the same node-level cause.
|
||||
// the local-node exception path, where the control DB authoritatively knows the
|
||||
// local stack list, so each stack carries the same node-level cause.
|
||||
const failAllStacks = (stacks: string[], error: string): StackStopResult[] =>
|
||||
stacks.map(stackName => ({ stackName, success: false, error }));
|
||||
|
||||
// Fleet-wide stop by label name. Matches each node's labels by name and runs
|
||||
// container stops on each matching stack.
|
||||
type FleetStopNodeResult = {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
reachable: boolean;
|
||||
matched: boolean;
|
||||
stackResults: StackStopResult[];
|
||||
error?: string;
|
||||
};
|
||||
|
||||
// Fleet-wide stop by label name. Each node is asked authoritatively: the local
|
||||
// node matches against its own DB in-process; each remote runs the match + stop
|
||||
// on its own Docker via its local-stop receiver. Remote label rows are never
|
||||
// mirrored to the control, so there is no central pre-check; unreachable nodes
|
||||
// are reported at the node level and never block the reachable ones.
|
||||
// Tier: requireAdmin (admin-only fleet plumbing; available on every license).
|
||||
fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
@@ -1263,51 +1276,35 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodes = db.getNodes();
|
||||
if (isDebugEnabled()) console.debug('[Fleet:debug] fleet-stop:', { labelName: trimmed, dryRun: isDryRun, nodes: nodes.length });
|
||||
const results = await Promise.all(nodes.map(async (node) => {
|
||||
const results = await Promise.all(nodes.map(async (node): Promise<FleetStopNodeResult> => {
|
||||
if (node.type === 'local') {
|
||||
// Match + stop runs in-process against the control's own Docker. The
|
||||
// helper shares the per-node `bulk:<id>` lock with the per-label action
|
||||
// route so the two cannot double-stop the same containers. A control-side
|
||||
// failure (e.g. the compose dir is unreadable) degrades to a per-stack
|
||||
// error for this node only, the same way the remote leg does, so one bad
|
||||
// node never discards the rest of the fleet's results.
|
||||
// error for this node only; the node is reachable, the stop just failed.
|
||||
try {
|
||||
const outcome = await runLocalLabelStop(node.id, trimmed, isDryRun);
|
||||
return { nodeId: node.id, nodeName: node.name, matched: outcome.matched, stackResults: outcome.stackResults };
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: true, matched: outcome.matched, stackResults: outcome.stackResults };
|
||||
} catch (err) {
|
||||
const errorMsg = getErrorMessage(err, 'Failed to stop local stacks');
|
||||
const localLabel = db.getLabels(node.id).find(l => l.name === trimmed);
|
||||
const localStacks = localLabel ? db.getStacksForLabel(localLabel.id, node.id) : [];
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, matched: !!localLabel,
|
||||
nodeId: node.id, nodeName: node.name, reachable: true, matched: !!localLabel,
|
||||
stackResults: failAllStacks(localStacks, errorMsg),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Remote node. The control's mirror tells us which stacks *should* match
|
||||
// so transport errors can be attributed per stack; the authoritative stop
|
||||
// runs on the remote via its admin-only local-stop receiver, which reuses
|
||||
// the same name-matched helper under the remote's own bulk lock. This
|
||||
// replaces the previous fan-out to `POST /api/labels/:id/action`, a
|
||||
// paid-tier route that 403'd on Community remotes even though fleet-stop
|
||||
// itself is available on every license.
|
||||
const label = db.getLabels(node.id).find(l => l.name === trimmed);
|
||||
if (!label) {
|
||||
return { nodeId: node.id, nodeName: node.name, matched: false, stackResults: [] };
|
||||
}
|
||||
const stackNames = db.getStacksForLabel(label.id, node.id);
|
||||
if (stackNames.length === 0) {
|
||||
return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: [] };
|
||||
}
|
||||
|
||||
// Remote node. Ask the remote authoritatively via its admin-only local-stop
|
||||
// receiver, which name-matches under the remote's own bulk lock. There is no
|
||||
// control-side pre-check: remote label rows are never mirrored to the
|
||||
// control, so a mirror lookup would skip every remote. A node we cannot
|
||||
// reach is reported at the node level and never blocks the reachable nodes.
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target) {
|
||||
const error = formatNoTargetError(node);
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, matched: true,
|
||||
stackResults: failAllStacks(stackNames, error),
|
||||
};
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, matched: false, stackResults: [], error: formatNoTargetError(node) };
|
||||
}
|
||||
try {
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
@@ -1320,29 +1317,19 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
|
||||
});
|
||||
if (!response.ok) {
|
||||
const err = (await response.json().catch(() => ({}))) as { error?: string };
|
||||
const message = err.error || `Remote returned ${response.status}`;
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, matched: true,
|
||||
stackResults: failAllStacks(stackNames, message),
|
||||
};
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, matched: false, stackResults: [], error: err.error || `Remote returned ${response.status}` };
|
||||
}
|
||||
// Trust the remote's own matched flag over the control's mirror: a
|
||||
// mirror-skewed control could believe the label exists while the remote
|
||||
// has no such label, which the remote reports as matched:false. Guard
|
||||
// results as an array so a malformed 200 body degrades to empty rather
|
||||
// than flowing a non-array into the per-stack renderers.
|
||||
// Trust the remote's own matched flag. Guard results as an array so a
|
||||
// malformed 200 body degrades to empty rather than flowing a non-array
|
||||
// into the per-stack renderers.
|
||||
const remote = (await response.json()) as Partial<LabelLocalStopResponse>;
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name,
|
||||
nodeId: node.id, nodeName: node.name, reachable: true,
|
||||
matched: remote.matched ?? true,
|
||||
stackResults: Array.isArray(remote.results) ? remote.results : [],
|
||||
};
|
||||
} catch (err) {
|
||||
const errorMsg = getErrorMessage(err, 'Failed to reach remote node');
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, matched: true,
|
||||
stackResults: failAllStacks(stackNames, errorMsg),
|
||||
};
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, matched: false, stackResults: [], error: getErrorMessage(err, 'Failed to reach remote node') };
|
||||
}
|
||||
}));
|
||||
if (isDebugEnabled()) {
|
||||
@@ -1530,10 +1517,11 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res
|
||||
// destructive endpoints above so the surface stays uniform: an operator who
|
||||
// can fire `fleet-stop` is also the operator who can ask how big it would be.
|
||||
|
||||
// Per-label fleet preview. Walks the central node list and looks up the label
|
||||
// + assignments table for each node. No remote fan-out: stack-to-label
|
||||
// assignments live in the central DB even for remote nodes, populated by the
|
||||
// nodes' own UIs and synced via Distributed API.
|
||||
// Per-label fleet preview. Fans out to every node authoritatively (local DB +
|
||||
// live remote reads via collectFleetLabelSummaries) and reports, per node, the
|
||||
// matching stacks, whether the label exists at all, and reachability. The card
|
||||
// uses these to distinguish "0 matching stacks" from "label exists but no
|
||||
// stacks assigned" from "remote unavailable".
|
||||
fleetRouter.post('/labels/match-preview', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
const body = req.body as { labelName?: unknown } | undefined;
|
||||
@@ -1548,54 +1536,60 @@ fleetRouter.post('/labels/match-preview', authMiddleware, async (req: Request, r
|
||||
}
|
||||
const trimmed = labelName.trim();
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodes = db.getNodes();
|
||||
const summaries = await collectFleetLabelSummaries();
|
||||
let matchedStacks = 0;
|
||||
const perNode = nodes.map((node) => {
|
||||
const label = db.getLabels(node.id).find(l => l.name === trimmed);
|
||||
const stackNames = label ? db.getStacksForLabel(label.id, node.id) : [];
|
||||
const perNode = summaries.map((node) => {
|
||||
const entry = node.labels.find(l => l.name === trimmed);
|
||||
const stackNames = entry?.stackNames ?? [];
|
||||
matchedStacks += stackNames.length;
|
||||
return {
|
||||
nodeId: node.id,
|
||||
nodeName: node.name,
|
||||
nodeId: node.nodeId,
|
||||
nodeName: node.nodeName,
|
||||
reachable: node.reachable,
|
||||
labelExists: !!entry,
|
||||
stackCount: stackNames.length,
|
||||
stackNames,
|
||||
...(node.error ? { error: node.error } : {}),
|
||||
};
|
||||
});
|
||||
const matchedNodes = perNode.filter(n => n.stackCount > 0).length;
|
||||
res.json({ matchedNodes, matchedStacks, perNode });
|
||||
const unreachableNodes = perNode.filter(n => !n.reachable).length;
|
||||
res.json({ matchedNodes, matchedStacks, unreachableNodes, perNode });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] match-preview error:', error);
|
||||
res.status(500).json({ error: getErrorMessage(error, 'Failed to compute match preview') });
|
||||
}
|
||||
});
|
||||
|
||||
// Stack-label suggestions for the Stop-by-label target picker. Aggregates the
|
||||
// per-node stack-label rows (the `stack_labels` table, via getLabels) into one
|
||||
// name-keyed list with stack/node counts. This is the only source the card's
|
||||
// autocomplete consumes; node labels (the separate `/api/node-labels`
|
||||
// namespace) are deliberately never folded in, because fleet-stop targets stack
|
||||
// labels only. The `scope: 'stack'` tag and the counts make that explicit at the
|
||||
// type level and in the UI. Central-DB only, same as match-preview, so it covers
|
||||
// every configured node including offline remotes.
|
||||
// Stack-label suggestions for the Stop-by-label target picker. Fans out to every
|
||||
// node authoritatively (local DB + live remote reads via collectFleetLabelSummaries)
|
||||
// and aggregates the per-node stack labels into one name-keyed list with node and
|
||||
// stack counts plus the carrying node names. Node labels (the separate
|
||||
// `/api/node-labels` namespace) are deliberately never folded in, because
|
||||
// fleet-stop targets stack labels only; the `scope: 'stack'` tag makes that
|
||||
// explicit. Labels with zero matching stacks fleet-wide are dropped (stopping
|
||||
// them is a no-op). `unreachableNodes`/`partial` tell the card the counts cover
|
||||
// only the nodes it could reach.
|
||||
fleetRouter.get('/labels/suggestions', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const agg = new Map<string, { nodeCount: number; stackCount: number }>();
|
||||
for (const node of db.getNodes()) {
|
||||
for (const label of db.getLabels(node.id)) {
|
||||
const stackCount = db.getStacksForLabel(label.id, node.id).length;
|
||||
const entry = agg.get(label.name) ?? { nodeCount: 0, stackCount: 0 };
|
||||
const summaries = await collectFleetLabelSummaries();
|
||||
const agg = new Map<string, { nodeCount: number; stackCount: number; nodes: string[] }>();
|
||||
for (const node of summaries) {
|
||||
for (const label of node.labels) {
|
||||
const entry = agg.get(label.name) ?? { nodeCount: 0, stackCount: 0, nodes: [] };
|
||||
entry.nodeCount += 1;
|
||||
entry.stackCount += stackCount;
|
||||
entry.stackCount += label.stackNames.length;
|
||||
entry.nodes.push(node.nodeName);
|
||||
agg.set(label.name, entry);
|
||||
}
|
||||
}
|
||||
const suggestions = Array.from(agg.entries())
|
||||
.map(([name, counts]) => ({ name, scope: 'stack' as const, nodeCount: counts.nodeCount, stackCount: counts.stackCount }))
|
||||
.filter(([, counts]) => counts.stackCount > 0)
|
||||
.map(([name, counts]) => ({ name, scope: 'stack' as const, nodeCount: counts.nodeCount, stackCount: counts.stackCount, nodes: counts.nodes }))
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
res.json({ suggestions });
|
||||
const unreachableNodes = summaries.filter(n => !n.reachable).length;
|
||||
res.json({ suggestions, unreachableNodes, partial: unreachableNodes > 0 });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] label-suggestions error:', error);
|
||||
res.status(500).json({ error: getErrorMessage(error, 'Failed to load stack-label suggestions') });
|
||||
|
||||
Reference in New Issue
Block a user