mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-28 12:49:03 +00:00
fix(fleet): resolve Stop-by-label stack labels across all nodes (#1382)
* fix(fleet): resolve Stop-by-label stack labels across all nodes The Fleet Actions "Stop by label" card only ever saw the control node's own stack labels. The stack-label routes are proxied, so each node stores its labels in its own database and the control holds no mirror for remote nodes. The suggestions, match-preview, and fleet-stop endpoints all read that nonexistent mirror, so remote-only labels were invisible and remote stacks were skipped before the remote was ever asked. Make all three authoritative across the fleet with a new collectFleetLabelSummaries helper: the local node reads its own database, and each remote is queried live through its labels and label-assignments endpoints over the proxy, with fail-closed parsing and per-node reachability. fleet-stop drops the mirror pre-check and always calls each reachable remote's local-stop receiver, reporting unreachable nodes at the node level so they never block the reachable ones. The picker now surfaces remote-only labels, aggregates shared names once with combined counts and the carrying node names, and flags incomplete coverage when a node is unreachable. The preview groups matches per node, lists unreachable nodes separately, and distinguishes no matching stacks from "label exists but no stacks" from "remote unavailable". * fix(fleet): harden Stop-by-label card against malformed responses Guard the match-preview and fleet-stop response bodies so a malformed but 200 reply degrades instead of crashing or misreporting. match-preview now validates the per-node shape before rendering (the preview reads it outside any try/catch) and logs a malformed body; fleet-stop distinguishes a non-array results body (a server bug, now logged and surfaced as an unexpected-response error) from a genuine empty fleet, and guards per-node stackResults. Docs: an unreachable or errored remote is reported once per node, not as a per-stack error row.
This commit is contained in:
Anso
@@ -109,6 +109,32 @@ async function createAssignedLabel(name: string, stacks: string[]) {
|
||||
return created.body as { id: number; node_id: number; name: string };
|
||||
}
|
||||
|
||||
// Stub the control's server-side fan-out to a remote node. The remote's
|
||||
// authoritative stack-label state is read live via /api/labels +
|
||||
// /api/labels/assignments; this returns canned bodies for those two URLs and
|
||||
// passes any other URL (e.g. the fleet-stop local-stop receiver) to `other`.
|
||||
function mockRemoteLabelFetch(opts: {
|
||||
labels: { name: string; color?: string }[];
|
||||
assignments: Record<string, string[]>; // stackName -> label names
|
||||
other?: (url: string) => Response;
|
||||
}) {
|
||||
return vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: Parameters<typeof fetch>[0]) => {
|
||||
const url = String(input);
|
||||
if (url.endsWith('/api/labels')) {
|
||||
const body = opts.labels.map((l, i) => ({ id: i + 1, node_id: 9, name: l.name, color: l.color ?? 'teal' }));
|
||||
return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
if (url.endsWith('/api/labels/assignments')) {
|
||||
const body: Record<string, { name: string; color: string }[]> = {};
|
||||
for (const [stack, names] of Object.entries(opts.assignments)) {
|
||||
body[stack] = names.map(n => ({ name: n, color: 'teal' }));
|
||||
}
|
||||
return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
return opts.other ? opts.other(url) : new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
});
|
||||
}
|
||||
|
||||
describe('POST /api/fleet/labels/match-preview', () => {
|
||||
it('returns 401 without auth', async () => {
|
||||
const res = await request(app)
|
||||
@@ -164,6 +190,99 @@ describe('POST /api/fleet/labels/match-preview', () => {
|
||||
expect(res.body.matchedNodes).toBe(0);
|
||||
expect(res.body.matchedStacks).toBe(0);
|
||||
});
|
||||
|
||||
it('includes remote stacks grouped per node, drops stale local stacks, and flags reachability', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const local = db.createLabel(localId, 'media', 'teal');
|
||||
db.setStackLabels('alpha', localId, [local.id]);
|
||||
// A stale local assignment to a stack that no longer exists on disk
|
||||
// (not in mockFsStacks) must be filtered out, not counted.
|
||||
db.setStackLabels('media-ghost', localId, [local.id]);
|
||||
const remoteId = db.addNode({
|
||||
name: 'media-remote', type: 'remote', api_url: 'http://media.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
// The control DB holds no 'media' row for the remote; its stacks come live.
|
||||
mockRemoteLabelFetch({ labels: [{ name: 'media' }], assignments: { sonarr: ['media'], radarr: ['media'] } });
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/match-preview')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: 'media' });
|
||||
expect(res.status).toBe(200);
|
||||
// 1 live local stack (the ghost is dropped) + 2 remote stacks.
|
||||
expect(res.body.matchedStacks).toBe(3);
|
||||
expect(res.body.matchedNodes).toBe(2);
|
||||
expect(res.body.unreachableNodes).toBe(0);
|
||||
const localRow = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === localId);
|
||||
expect(localRow.stackCount).toBe(1);
|
||||
expect(localRow.stackNames).toEqual(['alpha']);
|
||||
const remote = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remote.reachable).toBe(true);
|
||||
expect(remote.labelExists).toBe(true);
|
||||
expect(remote.stackNames.sort()).toEqual(['radarr', 'sonarr']);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('distinguishes label-exists-no-stacks from an unreachable remote', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
// Local carries the label but assigns no stacks to it.
|
||||
db.createLabel(localId, 'empty-label', 'teal');
|
||||
const remoteId = db.addNode({
|
||||
name: 'unreach-remote', type: 'remote', api_url: 'http://unreach.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('connect ECONNREFUSED'));
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/match-preview')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: 'empty-label' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.unreachableNodes).toBe(1);
|
||||
const localRow = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === localId);
|
||||
expect(localRow.reachable).toBe(true);
|
||||
expect(localRow.labelExists).toBe(true);
|
||||
expect(localRow.stackCount).toBe(0);
|
||||
const remoteRow = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remoteRow.reachable).toBe(false);
|
||||
expect(remoteRow.error).toMatch(/ECONNREFUSED|reach/i);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('surfaces the remote error body when it serves labels but rejects assignments', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
db.setStackLabels('alpha', localId, [db.createLabel(localId, 'ok-label', 'teal').id]);
|
||||
const remoteId = db.addNode({
|
||||
name: 'split-remote', type: 'remote', api_url: 'http://split.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: Parameters<typeof fetch>[0]) => {
|
||||
const url = String(input);
|
||||
if (url.endsWith('/api/labels')) {
|
||||
return new Response(JSON.stringify([]), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
// /api/labels/assignments rejects with a real error body.
|
||||
return new Response(JSON.stringify({ error: 'token expired' }), { status: 403, headers: { 'content-type': 'application/json' } });
|
||||
});
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/match-preview')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: 'ok-label' });
|
||||
expect(res.status).toBe(200);
|
||||
const remoteRow = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remoteRow.reachable).toBe(false);
|
||||
// The remote's own message is surfaced, not a bare status number.
|
||||
expect(remoteRow.error).toBe('token expired');
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/fleet/labels/suggestions', () => {
|
||||
@@ -192,22 +311,23 @@ describe('GET /api/fleet/labels/suggestions', () => {
|
||||
expect(Array.isArray(res.body.suggestions)).toBe(true);
|
||||
});
|
||||
|
||||
it('aggregates stack labels across nodes, includes unassigned ones, and excludes node labels', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
it('surfaces remote-only stack labels and aggregates shared names across nodes', async () => {
|
||||
const localNode = db.getNodes().find(n => n.is_default)!;
|
||||
const remoteId = db.addNode({
|
||||
name: 'sugg-remote', type: 'remote', api_url: 'http://sugg.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
// Same-named stack label on both nodes, each with one assigned stack.
|
||||
const local = db.createLabel(localId, 'shared-prod', 'teal');
|
||||
db.setStackLabels('alpha', localId, [local.id]);
|
||||
const remote = db.createLabel(remoteId, 'shared-prod', 'teal');
|
||||
db.setStackLabels('beta', remoteId, [remote.id]);
|
||||
// A stack label with no assignments still belongs to the picker.
|
||||
db.createLabel(localId, 'unused-stack-label', 'sky');
|
||||
// A node-only label that must never surface as a stop target.
|
||||
db.addNodeLabel(remoteId, 'edge-only');
|
||||
// Local node: 'shared-prod' on stack 'alpha' (alpha is in mockFsStacks).
|
||||
const local = db.createLabel(localNode.id, 'shared-prod', 'teal');
|
||||
db.setStackLabels('alpha', localNode.id, [local.id]);
|
||||
// Remote node: queried live. It carries 'shared-prod' on TWO stacks plus a
|
||||
// remote-only 'edge'. NONE of this is mirrored in the control DB. The
|
||||
// asymmetric remote count (2) makes a sum-vs-concatenate bug observable.
|
||||
mockRemoteLabelFetch({
|
||||
labels: [{ name: 'shared-prod' }, { name: 'edge' }],
|
||||
assignments: { sonarr: ['shared-prod'], lidarr: ['shared-prod'], radarr: ['edge'] },
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
@@ -216,36 +336,42 @@ describe('GET /api/fleet/labels/suggestions', () => {
|
||||
|
||||
const names = res.body.suggestions.map((s: { name: string }) => s.name);
|
||||
expect(names).toContain('shared-prod');
|
||||
expect(names).toContain('unused-stack-label');
|
||||
expect(names).not.toContain('edge-only');
|
||||
// Remote-only label is visible despite no control-DB row.
|
||||
expect(names).toContain('edge');
|
||||
// Sorted by name.
|
||||
expect(names).toEqual([...names].sort((a: string, b: string) => a.localeCompare(b)));
|
||||
expect(res.body.partial).toBe(false);
|
||||
expect(res.body.unreachableNodes).toBe(0);
|
||||
|
||||
const shared = res.body.suggestions.find((s: { name: string }) => s.name === 'shared-prod');
|
||||
expect(shared.scope).toBe('stack');
|
||||
expect(shared.nodeCount).toBe(2);
|
||||
expect(shared.stackCount).toBe(2);
|
||||
// 1 local stack + 2 remote stacks, summed (not deduped to the node count).
|
||||
expect(shared.stackCount).toBe(3);
|
||||
expect([...shared.nodes].sort()).toEqual([localNode.name, 'sugg-remote'].sort());
|
||||
|
||||
const unused = res.body.suggestions.find((s: { name: string }) => s.name === 'unused-stack-label');
|
||||
expect(unused.nodeCount).toBe(1);
|
||||
expect(unused.stackCount).toBe(0);
|
||||
const edge = res.body.suggestions.find((s: { name: string }) => s.name === 'edge');
|
||||
expect(edge.nodeCount).toBe(1);
|
||||
expect(edge.stackCount).toBe(1);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('counts only the stack-label side when a node label shares the name', async () => {
|
||||
it('never folds node labels into the picker even when one shares a stack-label name', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const remoteId = db.addNode({
|
||||
name: 'collision-remote', type: 'remote', api_url: 'http://collision.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
// A stack label 'prod' lives on the local node only...
|
||||
// A stack label 'prod' lives on the local node...
|
||||
const stackLabel = db.createLabel(localId, 'prod', 'teal');
|
||||
db.setStackLabels('alpha', localId, [stackLabel.id]);
|
||||
// ...while a node label of the same name lives on the remote node.
|
||||
// ...while a node label of the same name lives on the remote node. The
|
||||
// remote's stack-label endpoint (/api/labels) never returns node labels.
|
||||
db.addNodeLabel(remoteId, 'prod');
|
||||
mockRemoteLabelFetch({ labels: [], assignments: {} });
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
@@ -260,6 +386,75 @@ describe('GET /api/fleet/labels/suggestions', () => {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('excludes labels whose only assigned stacks no longer exist on disk', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const ghost = db.createLabel(localId, 'ghost-only', 'teal');
|
||||
// 'vanished-stack' is not in mockFsStacks, so the fs-present filter drops it,
|
||||
// leaving the label with zero matching stacks fleet-wide.
|
||||
db.setStackLabels('vanished-stack', localId, [ghost.id]);
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
.set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
const names = res.body.suggestions.map((s: { name: string }) => s.name);
|
||||
expect(names).not.toContain('ghost-only');
|
||||
});
|
||||
|
||||
it('reports unreachable remotes as partial without dropping reachable suggestions', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const local = db.createLabel(localId, 'live-label', 'teal');
|
||||
db.setStackLabels('alpha', localId, [local.id]);
|
||||
const remoteId = db.addNode({
|
||||
name: 'down-remote', type: 'remote', api_url: 'http://down.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('connect ECONNREFUSED'));
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
.set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.partial).toBe(true);
|
||||
expect(res.body.unreachableNodes).toBe(1);
|
||||
const names = res.body.suggestions.map((s: { name: string }) => s.name);
|
||||
expect(names).toContain('live-label');
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('fails closed: a remote returning a malformed label body is marked unreachable', async () => {
|
||||
const localId = db.getNodes().find(n => n.is_default)!.id;
|
||||
const local = db.createLabel(localId, 'safe-label', 'teal');
|
||||
db.setStackLabels('alpha', localId, [local.id]);
|
||||
const remoteId = db.addNode({
|
||||
name: 'garbage-remote', type: 'remote', api_url: 'http://garbage.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: Parameters<typeof fetch>[0]) => {
|
||||
const url = String(input);
|
||||
if (url.endsWith('/api/labels')) {
|
||||
// Not an array: the fail-closed parser must reject the whole node.
|
||||
return new Response(JSON.stringify({ not: 'an array' }), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
return new Response(JSON.stringify({}), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
});
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
.set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.partial).toBe(true);
|
||||
expect(res.body.unreachableNodes).toBe(1);
|
||||
// The garbage remote contributes nothing; the local label still surfaces.
|
||||
const names = res.body.suggestions.map((s: { name: string }) => s.name);
|
||||
expect(names).toContain('safe-label');
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/fleet/prune/estimate', () => {
|
||||
@@ -438,6 +633,64 @@ describe('POST /api/fleet/labels/fleet-stop remote leg', () => {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('calls the remote local-stop even when the control DB has no remote label row', async () => {
|
||||
const remoteId = db.addNode({
|
||||
name: 'remote-nomirror', type: 'remote', api_url: 'http://remote-nomirror.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
// Deliberately seed NO label/assignment for the remote in the control DB.
|
||||
// The old mirror pre-check would have skipped this node entirely.
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true, status: 200,
|
||||
json: async () => ({ matched: true, results: [{ stackName: 'remote-stack', success: true }] }),
|
||||
} as unknown as Response);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-stop')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: 'remote-only-label' });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const urls = fetchSpy.mock.calls.map(c => String(c[0]));
|
||||
expect(urls.some(u => u.endsWith('/api/fleet-actions/labels/local-stop'))).toBe(true);
|
||||
const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
|
||||
expect(remoteRow.reachable).toBe(true);
|
||||
expect(remoteRow.matched).toBe(true);
|
||||
expect(remoteRow.stackResults).toEqual([{ stackName: 'remote-stack', success: true }]);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('reports an unreachable remote at the node level without blocking the local stop', async () => {
|
||||
const localLabel = await createAssignedLabel('local-live', ['alpha']);
|
||||
const remoteId = db.addNode({
|
||||
name: 'remote-down', type: 'remote', api_url: 'http://remote-down.example:1852',
|
||||
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
|
||||
});
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('connect ECONNREFUSED'));
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-stop')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ labelName: localLabel.name });
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const localResult = res.body.results.find((r: { nodeId: number }) => r.nodeId === localLabel.node_id);
|
||||
expect(localResult.reachable).toBe(true);
|
||||
expect(localResult.matched).toBe(true);
|
||||
expect(localResult.stackResults).toEqual([{ stackName: 'alpha', success: true }]);
|
||||
const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
|
||||
expect(remoteRow.reachable).toBe(false);
|
||||
expect(remoteRow.matched).toBe(false);
|
||||
expect(remoteRow.stackResults).toEqual([]);
|
||||
expect(remoteRow.error).toMatch(/ECONNREFUSED|reach/i);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/fleet/labels/fleet-stop with dryRun: true', () => {
|
||||
|
||||
@@ -158,7 +158,50 @@ describe('POST /api/fleet/labels/fleet-stop (pilot-agent dispatch)', () => {
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const pilotResult = res.body.results.find((r: { nodeId: number }) => r.nodeId === pilotNodeId);
|
||||
expect(pilotResult.stackResults[0].error).toMatch(/pilot tunnel/i);
|
||||
// A node with no reachable target is reported at the node level; there is no
|
||||
// control-side mirror to enumerate per-stack rows for an unreachable remote.
|
||||
expect(pilotResult.reachable).toBe(false);
|
||||
expect(pilotResult.matched).toBe(false);
|
||||
expect(pilotResult.stackResults).toEqual([]);
|
||||
expect(pilotResult.error).toMatch(/pilot tunnel/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/fleet/labels/suggestions (pilot-agent summary fan-out)', () => {
|
||||
it('reads each remote label set through the proxy target with conditional Authorization', async () => {
|
||||
mockTargets();
|
||||
const calls: Array<{ url: string; auth: string | undefined }> = [];
|
||||
mockFetch((url, init) => {
|
||||
const headers = (init?.headers as Record<string, string>) ?? {};
|
||||
calls.push({ url, auth: headers.Authorization });
|
||||
if (url.endsWith('/api/labels')) {
|
||||
return new Response(JSON.stringify([{ id: 1, node_id: 0, name: 'summary-label', color: 'teal' }]), {
|
||||
status: 200, headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
}
|
||||
if (url.endsWith('/api/labels/assignments')) {
|
||||
return new Response(JSON.stringify({ svc: [{ id: 1, node_id: 0, name: 'summary-label', color: 'teal' }] }), {
|
||||
status: 200, headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
}
|
||||
return new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/fleet/labels/suggestions')
|
||||
.set('Authorization', authHeader);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
// Both fan-out legs (labels + assignments) must use the proxy target with the
|
||||
// right auth: pilot loopback carries no Authorization, proxy carries Bearer.
|
||||
const pilotCalls = calls.filter(c => c.url.startsWith(PILOT_LOOPBACK));
|
||||
expect(pilotCalls.map(c => c.url).sort()).toEqual([`${PILOT_LOOPBACK}/api/labels`, `${PILOT_LOOPBACK}/api/labels/assignments`]);
|
||||
expect(pilotCalls.every(c => c.auth === undefined)).toBe(true);
|
||||
const proxyCalls = calls.filter(c => c.url.startsWith(PROXY_URL));
|
||||
expect(proxyCalls.map(c => c.url).sort()).toEqual([`${PROXY_URL}/api/labels`, `${PROXY_URL}/api/labels/assignments`]);
|
||||
expect(proxyCalls.every(c => c.auth === `Bearer ${PROXY_TOKEN}`)).toBe(true);
|
||||
// The label only exists on the remotes (live fan-out), never in the control DB.
|
||||
expect(res.body.suggestions.some((s: { name: string }) => s.name === 'summary-label')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { DatabaseService, type Node } from '../services/DatabaseService';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { formatNoTargetError } from '../utils/remoteTarget';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
export interface NodeLabelSummaryEntry {
|
||||
name: string;
|
||||
color: string;
|
||||
stackNames: string[];
|
||||
}
|
||||
|
||||
export interface NodeLabelSummary {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
reachable: boolean;
|
||||
labels: NodeLabelSummaryEntry[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// Per-node fetch budget. Suggestions load on card mount and match-preview runs
|
||||
// on a 500ms debounce, so keep this short: a wedged or slow remote should fall
|
||||
// to "unreachable" fast rather than stall the whole fleet readout.
|
||||
const SUMMARY_FETCH_TIMEOUT_MS = 5000;
|
||||
|
||||
/**
|
||||
* Authoritative stack-label summary for one node's own labels.
|
||||
*
|
||||
* Stack names are filtered to those still present on disk, matching the
|
||||
* `/api/labels/assignments` cleanup (routes/labels.ts) and runLocalLabelStop's
|
||||
* validStacks filter (helpers/fleetLabelStop.ts), so the preview/suggestion
|
||||
* counts equal the set a stop would actually act on.
|
||||
*/
|
||||
export async function readLocalLabelSummary(nodeId: number): Promise<NodeLabelSummaryEntry[]> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const labels = db.getLabels(nodeId);
|
||||
if (labels.length === 0) return [];
|
||||
const fsStacks = new Set(await FileSystemService.getInstance(nodeId).getStacks());
|
||||
return labels.map(label => ({
|
||||
name: label.name,
|
||||
color: label.color,
|
||||
stackNames: db.getStacksForLabel(label.id, nodeId).filter(name => fsStacks.has(name)),
|
||||
}));
|
||||
}
|
||||
|
||||
// Fail-closed parser for a remote `/api/labels` body (Label[]). A malformed
|
||||
// shape returns null so the caller marks the node unreachable rather than
|
||||
// flowing partial data into the aggregate counts.
|
||||
function parseRemoteLabels(value: unknown): { name: string; color: string }[] | null {
|
||||
if (!Array.isArray(value)) return null;
|
||||
const out: { name: string; color: string }[] = [];
|
||||
for (const item of value) {
|
||||
if (!item || typeof item !== 'object') return null;
|
||||
const label = item as Record<string, unknown>;
|
||||
if (typeof label.name !== 'string' || typeof label.color !== 'string') return null;
|
||||
out.push({ name: label.name, color: label.color });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Fail-closed parser for a remote `/api/labels/assignments` body
|
||||
// (Record<stackName, Label[]>). Collapses to label-name -> stack-name lists.
|
||||
function parseRemoteAssignments(value: unknown): Map<string, string[]> | null {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
||||
const stacksByLabel = new Map<string, string[]>();
|
||||
for (const [stackName, labels] of Object.entries(value as Record<string, unknown>)) {
|
||||
if (!Array.isArray(labels)) return null;
|
||||
for (const label of labels) {
|
||||
if (!label || typeof label !== 'object') return null;
|
||||
const name = (label as Record<string, unknown>).name;
|
||||
if (typeof name !== 'string') return null;
|
||||
const list = stacksByLabel.get(name) ?? [];
|
||||
list.push(stackName);
|
||||
stacksByLabel.set(name, list);
|
||||
}
|
||||
}
|
||||
return stacksByLabel;
|
||||
}
|
||||
|
||||
async function summarizeRemoteNode(node: Node): Promise<NodeLabelSummary> {
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target) {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: formatNoTargetError(node) };
|
||||
}
|
||||
const base = target.apiUrl.replace(/\/$/, '');
|
||||
// Conditional Bearer only: pilot-agent targets carry an empty token and reach
|
||||
// the remote over the loopback tunnel without an Authorization header.
|
||||
const headers: Record<string, string> = {};
|
||||
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
|
||||
try {
|
||||
const [labelsRes, assignmentsRes] = await Promise.all([
|
||||
fetch(`${base}/api/labels`, { headers, signal: AbortSignal.timeout(SUMMARY_FETCH_TIMEOUT_MS) }),
|
||||
fetch(`${base}/api/labels/assignments`, { headers, signal: AbortSignal.timeout(SUMMARY_FETCH_TIMEOUT_MS) }),
|
||||
]);
|
||||
if (!labelsRes.ok || !assignmentsRes.ok) {
|
||||
// Surface the remote's own error body (e.g. a token/tier message) the same
|
||||
// way the fleet-stop leg does, so a reached-but-erroring node reports its
|
||||
// real cause rather than a bare status number.
|
||||
const failed = labelsRes.ok ? assignmentsRes : labelsRes;
|
||||
const errBody = (await failed.json().catch(() => ({}))) as { error?: string };
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: errBody.error || `Remote returned ${failed.status}` };
|
||||
}
|
||||
const labels = parseRemoteLabels(await labelsRes.json().catch(() => null));
|
||||
const stacksByLabel = parseRemoteAssignments(await assignmentsRes.json().catch(() => null));
|
||||
if (!labels || !stacksByLabel) {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: 'Invalid label response from remote node' };
|
||||
}
|
||||
const summary = labels.map(label => ({
|
||||
name: label.name,
|
||||
color: label.color,
|
||||
stackNames: stacksByLabel.get(label.name) ?? [],
|
||||
}));
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: true, labels: summary };
|
||||
} catch (err) {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: getErrorMessage(err, 'Failed to reach remote node') };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative fleet-wide stack-label state: the local node from its own DB,
|
||||
* each remote queried live through its `/api/labels` + `/api/labels/assignments`
|
||||
* endpoints via the proxy target. Replaces the old central-DB read, which only
|
||||
* ever held the local node's labels (remote label rows are never mirrored to the
|
||||
* control). A node that cannot be reached, or that answers with a non-ok or
|
||||
* unusable response, is `reachable: false` with an error (its real cause) and
|
||||
* never blocks the reachable ones; reachable nodes carry complete `labels`.
|
||||
*/
|
||||
export async function collectFleetLabelSummaries(): Promise<NodeLabelSummary[]> {
|
||||
const nodes = DatabaseService.getInstance().getNodes();
|
||||
return Promise.all(nodes.map(async (node): Promise<NodeLabelSummary> => {
|
||||
if (node.type === 'local') {
|
||||
try {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: true, labels: await readLocalLabelSummary(node.id) };
|
||||
} catch (err) {
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, labels: [], error: getErrorMessage(err, 'Failed to read local labels') };
|
||||
}
|
||||
}
|
||||
return summarizeRemoteNode(node);
|
||||
}));
|
||||
}
|
||||
+69
-75
@@ -37,6 +37,7 @@ import { NotificationService } from '../services/NotificationService';
|
||||
import { invalidateNodeCaches, invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
|
||||
import { activeBulkActions } from './labels';
|
||||
import { runLocalLabelStop, type LabelLocalStopResponse, type StackStopResult } from '../helpers/fleetLabelStop';
|
||||
import { collectFleetLabelSummaries } from '../helpers/fleetLabelSummary';
|
||||
import { buildLocalConfigurationStatus, type ConfigurationStatus } from './dashboard';
|
||||
import { buildLocalGraph, mergeFleetGraph, isLocalDependencyGraph, type FleetNodeGraphResult } from '../services/DependencyGraphService';
|
||||
import { PROXY_TIER_HEADER } from '../services/license-headers';
|
||||
@@ -1237,13 +1238,25 @@ fleetRouter.delete('/update-status', authMiddleware, async (req: Request, res: R
|
||||
// lives here so it sits behind the `/api/fleet/` proxy-exempt prefix.
|
||||
|
||||
// Attribute one error to every stack a node was supposed to act on. Used for
|
||||
// the fleet-stop failure paths (no proxy target, non-ok remote, transport
|
||||
// error, local exception) so each stack carries the same node-level cause.
|
||||
// the local-node exception path, where the control DB authoritatively knows the
|
||||
// local stack list, so each stack carries the same node-level cause.
|
||||
const failAllStacks = (stacks: string[], error: string): StackStopResult[] =>
|
||||
stacks.map(stackName => ({ stackName, success: false, error }));
|
||||
|
||||
// Fleet-wide stop by label name. Matches each node's labels by name and runs
|
||||
// container stops on each matching stack.
|
||||
type FleetStopNodeResult = {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
reachable: boolean;
|
||||
matched: boolean;
|
||||
stackResults: StackStopResult[];
|
||||
error?: string;
|
||||
};
|
||||
|
||||
// Fleet-wide stop by label name. Each node is asked authoritatively: the local
|
||||
// node matches against its own DB in-process; each remote runs the match + stop
|
||||
// on its own Docker via its local-stop receiver. Remote label rows are never
|
||||
// mirrored to the control, so there is no central pre-check; unreachable nodes
|
||||
// are reported at the node level and never block the reachable ones.
|
||||
// Tier: requireAdmin (admin-only fleet plumbing; available on every license).
|
||||
fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
@@ -1263,51 +1276,35 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodes = db.getNodes();
|
||||
if (isDebugEnabled()) console.debug('[Fleet:debug] fleet-stop:', { labelName: trimmed, dryRun: isDryRun, nodes: nodes.length });
|
||||
const results = await Promise.all(nodes.map(async (node) => {
|
||||
const results = await Promise.all(nodes.map(async (node): Promise<FleetStopNodeResult> => {
|
||||
if (node.type === 'local') {
|
||||
// Match + stop runs in-process against the control's own Docker. The
|
||||
// helper shares the per-node `bulk:<id>` lock with the per-label action
|
||||
// route so the two cannot double-stop the same containers. A control-side
|
||||
// failure (e.g. the compose dir is unreadable) degrades to a per-stack
|
||||
// error for this node only, the same way the remote leg does, so one bad
|
||||
// node never discards the rest of the fleet's results.
|
||||
// error for this node only; the node is reachable, the stop just failed.
|
||||
try {
|
||||
const outcome = await runLocalLabelStop(node.id, trimmed, isDryRun);
|
||||
return { nodeId: node.id, nodeName: node.name, matched: outcome.matched, stackResults: outcome.stackResults };
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: true, matched: outcome.matched, stackResults: outcome.stackResults };
|
||||
} catch (err) {
|
||||
const errorMsg = getErrorMessage(err, 'Failed to stop local stacks');
|
||||
const localLabel = db.getLabels(node.id).find(l => l.name === trimmed);
|
||||
const localStacks = localLabel ? db.getStacksForLabel(localLabel.id, node.id) : [];
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, matched: !!localLabel,
|
||||
nodeId: node.id, nodeName: node.name, reachable: true, matched: !!localLabel,
|
||||
stackResults: failAllStacks(localStacks, errorMsg),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Remote node. The control's mirror tells us which stacks *should* match
|
||||
// so transport errors can be attributed per stack; the authoritative stop
|
||||
// runs on the remote via its admin-only local-stop receiver, which reuses
|
||||
// the same name-matched helper under the remote's own bulk lock. This
|
||||
// replaces the previous fan-out to `POST /api/labels/:id/action`, a
|
||||
// paid-tier route that 403'd on Community remotes even though fleet-stop
|
||||
// itself is available on every license.
|
||||
const label = db.getLabels(node.id).find(l => l.name === trimmed);
|
||||
if (!label) {
|
||||
return { nodeId: node.id, nodeName: node.name, matched: false, stackResults: [] };
|
||||
}
|
||||
const stackNames = db.getStacksForLabel(label.id, node.id);
|
||||
if (stackNames.length === 0) {
|
||||
return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: [] };
|
||||
}
|
||||
|
||||
// Remote node. Ask the remote authoritatively via its admin-only local-stop
|
||||
// receiver, which name-matches under the remote's own bulk lock. There is no
|
||||
// control-side pre-check: remote label rows are never mirrored to the
|
||||
// control, so a mirror lookup would skip every remote. A node we cannot
|
||||
// reach is reported at the node level and never blocks the reachable nodes.
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target) {
|
||||
const error = formatNoTargetError(node);
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, matched: true,
|
||||
stackResults: failAllStacks(stackNames, error),
|
||||
};
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, matched: false, stackResults: [], error: formatNoTargetError(node) };
|
||||
}
|
||||
try {
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
@@ -1320,29 +1317,19 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
|
||||
});
|
||||
if (!response.ok) {
|
||||
const err = (await response.json().catch(() => ({}))) as { error?: string };
|
||||
const message = err.error || `Remote returned ${response.status}`;
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, matched: true,
|
||||
stackResults: failAllStacks(stackNames, message),
|
||||
};
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, matched: false, stackResults: [], error: err.error || `Remote returned ${response.status}` };
|
||||
}
|
||||
// Trust the remote's own matched flag over the control's mirror: a
|
||||
// mirror-skewed control could believe the label exists while the remote
|
||||
// has no such label, which the remote reports as matched:false. Guard
|
||||
// results as an array so a malformed 200 body degrades to empty rather
|
||||
// than flowing a non-array into the per-stack renderers.
|
||||
// Trust the remote's own matched flag. Guard results as an array so a
|
||||
// malformed 200 body degrades to empty rather than flowing a non-array
|
||||
// into the per-stack renderers.
|
||||
const remote = (await response.json()) as Partial<LabelLocalStopResponse>;
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name,
|
||||
nodeId: node.id, nodeName: node.name, reachable: true,
|
||||
matched: remote.matched ?? true,
|
||||
stackResults: Array.isArray(remote.results) ? remote.results : [],
|
||||
};
|
||||
} catch (err) {
|
||||
const errorMsg = getErrorMessage(err, 'Failed to reach remote node');
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, matched: true,
|
||||
stackResults: failAllStacks(stackNames, errorMsg),
|
||||
};
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: false, matched: false, stackResults: [], error: getErrorMessage(err, 'Failed to reach remote node') };
|
||||
}
|
||||
}));
|
||||
if (isDebugEnabled()) {
|
||||
@@ -1530,10 +1517,11 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res
|
||||
// destructive endpoints above so the surface stays uniform: an operator who
|
||||
// can fire `fleet-stop` is also the operator who can ask how big it would be.
|
||||
|
||||
// Per-label fleet preview. Walks the central node list and looks up the label
|
||||
// + assignments table for each node. No remote fan-out: stack-to-label
|
||||
// assignments live in the central DB even for remote nodes, populated by the
|
||||
// nodes' own UIs and synced via Distributed API.
|
||||
// Per-label fleet preview. Fans out to every node authoritatively (local DB +
|
||||
// live remote reads via collectFleetLabelSummaries) and reports, per node, the
|
||||
// matching stacks, whether the label exists at all, and reachability. The card
|
||||
// uses these to distinguish "0 matching stacks" from "label exists but no
|
||||
// stacks assigned" from "remote unavailable".
|
||||
fleetRouter.post('/labels/match-preview', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
const body = req.body as { labelName?: unknown } | undefined;
|
||||
@@ -1548,54 +1536,60 @@ fleetRouter.post('/labels/match-preview', authMiddleware, async (req: Request, r
|
||||
}
|
||||
const trimmed = labelName.trim();
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodes = db.getNodes();
|
||||
const summaries = await collectFleetLabelSummaries();
|
||||
let matchedStacks = 0;
|
||||
const perNode = nodes.map((node) => {
|
||||
const label = db.getLabels(node.id).find(l => l.name === trimmed);
|
||||
const stackNames = label ? db.getStacksForLabel(label.id, node.id) : [];
|
||||
const perNode = summaries.map((node) => {
|
||||
const entry = node.labels.find(l => l.name === trimmed);
|
||||
const stackNames = entry?.stackNames ?? [];
|
||||
matchedStacks += stackNames.length;
|
||||
return {
|
||||
nodeId: node.id,
|
||||
nodeName: node.name,
|
||||
nodeId: node.nodeId,
|
||||
nodeName: node.nodeName,
|
||||
reachable: node.reachable,
|
||||
labelExists: !!entry,
|
||||
stackCount: stackNames.length,
|
||||
stackNames,
|
||||
...(node.error ? { error: node.error } : {}),
|
||||
};
|
||||
});
|
||||
const matchedNodes = perNode.filter(n => n.stackCount > 0).length;
|
||||
res.json({ matchedNodes, matchedStacks, perNode });
|
||||
const unreachableNodes = perNode.filter(n => !n.reachable).length;
|
||||
res.json({ matchedNodes, matchedStacks, unreachableNodes, perNode });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] match-preview error:', error);
|
||||
res.status(500).json({ error: getErrorMessage(error, 'Failed to compute match preview') });
|
||||
}
|
||||
});
|
||||
|
||||
// Stack-label suggestions for the Stop-by-label target picker. Aggregates the
|
||||
// per-node stack-label rows (the `stack_labels` table, via getLabels) into one
|
||||
// name-keyed list with stack/node counts. This is the only source the card's
|
||||
// autocomplete consumes; node labels (the separate `/api/node-labels`
|
||||
// namespace) are deliberately never folded in, because fleet-stop targets stack
|
||||
// labels only. The `scope: 'stack'` tag and the counts make that explicit at the
|
||||
// type level and in the UI. Central-DB only, same as match-preview, so it covers
|
||||
// every configured node including offline remotes.
|
||||
// Stack-label suggestions for the Stop-by-label target picker. Fans out to every
|
||||
// node authoritatively (local DB + live remote reads via collectFleetLabelSummaries)
|
||||
// and aggregates the per-node stack labels into one name-keyed list with node and
|
||||
// stack counts plus the carrying node names. Node labels (the separate
|
||||
// `/api/node-labels` namespace) are deliberately never folded in, because
|
||||
// fleet-stop targets stack labels only; the `scope: 'stack'` tag makes that
|
||||
// explicit. Labels with zero matching stacks fleet-wide are dropped (stopping
|
||||
// them is a no-op). `unreachableNodes`/`partial` tell the card the counts cover
|
||||
// only the nodes it could reach.
|
||||
fleetRouter.get('/labels/suggestions', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const agg = new Map<string, { nodeCount: number; stackCount: number }>();
|
||||
for (const node of db.getNodes()) {
|
||||
for (const label of db.getLabels(node.id)) {
|
||||
const stackCount = db.getStacksForLabel(label.id, node.id).length;
|
||||
const entry = agg.get(label.name) ?? { nodeCount: 0, stackCount: 0 };
|
||||
const summaries = await collectFleetLabelSummaries();
|
||||
const agg = new Map<string, { nodeCount: number; stackCount: number; nodes: string[] }>();
|
||||
for (const node of summaries) {
|
||||
for (const label of node.labels) {
|
||||
const entry = agg.get(label.name) ?? { nodeCount: 0, stackCount: 0, nodes: [] };
|
||||
entry.nodeCount += 1;
|
||||
entry.stackCount += stackCount;
|
||||
entry.stackCount += label.stackNames.length;
|
||||
entry.nodes.push(node.nodeName);
|
||||
agg.set(label.name, entry);
|
||||
}
|
||||
}
|
||||
const suggestions = Array.from(agg.entries())
|
||||
.map(([name, counts]) => ({ name, scope: 'stack' as const, nodeCount: counts.nodeCount, stackCount: counts.stackCount }))
|
||||
.filter(([, counts]) => counts.stackCount > 0)
|
||||
.map(([name, counts]) => ({ name, scope: 'stack' as const, nodeCount: counts.nodeCount, stackCount: counts.stackCount, nodes: counts.nodes }))
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
res.json({ suggestions });
|
||||
const unreachableNodes = summaries.filter(n => !n.reachable).length;
|
||||
res.json({ suggestions, unreachableNodes, partial: unreachableNodes > 0 });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] label-suggestions error:', error);
|
||||
res.status(500).json({ error: getErrorMessage(error, 'Failed to load stack-label suggestions') });
|
||||
|
||||
@@ -46,7 +46,7 @@ Stop every stack assigned a given **stack label** on every node where that stack
|
||||
### Step by step
|
||||
|
||||
1. Open **Fleet → Actions**.
|
||||
2. Type a name in the **Stack label** field. The picker suggests stack labels from across the fleet, each with its stack and node counts, so the scope is unmistakable. Suggestions are read from the control instance, so stack labels on offline nodes appear too. You can also type a name by hand.
|
||||
2. Type a name in the **Stack label** field. The picker queries each reachable node for its own stack labels and lists each name once with its combined stack and node counts (and the carrying node names), so the scope is unmistakable. A stack label that exists only on a remote node still appears. When a node cannot be reached, the picker notes that the suggestions may be incomplete. You can also type a name by hand.
|
||||
3. Click **Stop fleet**.
|
||||
4. A confirmation appears with the kicker **Fleet stop** and the title `Stop all stacks with the stack label "<name>"?`. Click **Stop fleet** to commit.
|
||||
|
||||
@@ -66,7 +66,7 @@ A few quirks worth knowing:
|
||||
|
||||
- A node that has no stack label by that name appears as `<node> (no matching stack label)` and is counted in the **failed** badge. This is not a transport failure, it just means the stack label was not present on that node.
|
||||
- A node where the label exists but no stacks are assigned to it appears with a matched count of zero stacks. No per-stack rows render.
|
||||
- When the control instance reaches a remote node, the per-stack result you see comes from the remote node's own response. If the remote returns a non-2xx for the whole label, every stack on that node renders with the same error message.
|
||||
- When the control instance reaches a remote node, the per-stack results you see come from the remote node's own response. A remote the control cannot reach, or that returns a non-2xx, is reported once as a single `<node> (unreachable)` row carrying the reason, not as a per-stack error, and it never blocks the stops on the reachable nodes.
|
||||
|
||||
### Behaviour and partial-failure semantics
|
||||
|
||||
@@ -130,7 +130,7 @@ Scope is a segmented control with two options:
|
||||
|
||||
| Requirement | Why it matters |
|
||||
|---|---|
|
||||
| **Configured remote nodes in Settings → Nodes** | The two fan-out cards iterate the configured node list. A node missing its `api_url` or `api_token` shows up in the results with *Remote node not configured* per stack or per target. |
|
||||
| **Configured remote nodes in Settings → Nodes** | The two fan-out cards iterate the configured node list. A node missing its `api_url` or `api_token`, or one that cannot be reached, is reported once per node as unreachable (Stop fleet by label) or per target (Prune), and never blocks the reachable nodes. |
|
||||
| **Labels you intend to target** | Stop fleet by label and the autocomplete depend on labels existing on at least one node. See [Stack Labels](/features/stack-labels) for the authoring flow. |
|
||||
|
||||
## Behaviour and lifecycle
|
||||
@@ -173,7 +173,7 @@ Run **Prune Docker resources fleet-wide** with **Images** selected and **Managed
|
||||
The fleet-stop match is by **stack label name**, not label ID. Confirm the stack label name on the affected node under **Settings → Labels**; a typo, a case mismatch, or a trailing space will leave the node out. Stack labels are scoped per node, so renaming the label on one node does not propagate to the others.
|
||||
</Accordion>
|
||||
<Accordion title="The autocomplete didn't suggest a stack label I know exists">
|
||||
The picker reads stack labels from the control instance across every configured node, including offline ones, so a name is missing only when no node has a **stack label** by that name. Node labels never appear here, because this action targets stack labels only. You can always type a name by hand; the fleet-stop request still iterates every configured node.
|
||||
The picker queries each reachable node for its own stack labels, so a name is missing when no reachable node has a **stack label** by that name, or when the node that owns it could not be reached when the picker loaded (the picker flags this case). Node labels never appear here, because this action targets stack labels only. You can always type a name by hand; the fleet-stop request still iterates every node and asks each one authoritatively.
|
||||
</Accordion>
|
||||
<Accordion title="Bulk label assign reports 'Invalid stack name' for one row">
|
||||
Stack names must match the standard validator: alphanumeric plus dash and underscore, no spaces, no path separators. The endpoint validates each assignment independently, so a single bad name does not block the rest of the batch. Fix the offending entry and re-run; the rows that already succeeded won't be re-applied.
|
||||
@@ -190,8 +190,8 @@ Run **Prune Docker resources fleet-wide** with **Images** selected and **Managed
|
||||
<Accordion title="The Apply button returns an error toast">
|
||||
Fleet Actions runs admin-only. Confirm the active user has the admin role under **Settings → Users**. Operator and viewer roles see the cards but cannot apply them.
|
||||
</Accordion>
|
||||
<Accordion title="A node returns a transport error row for every stack or every target">
|
||||
The node is in **Settings → Nodes** but its `api_url` or `api_token` is missing, expired, or unreachable. Open **Settings → Nodes** on the control instance and click **Test connection** for the remote; fix the credential or the reachability, then re-run the action.
|
||||
<Accordion title="A node is reported as unreachable (or shows a transport error per target)">
|
||||
The node is in **Settings → Nodes** but its `api_url` or `api_token` is missing, expired, or unreachable. Stop fleet by label reports it once as a single `<node> (unreachable)` row; Prune reports it per target. Open **Settings → Nodes** on the control instance and click **Test connection** for the remote; fix the credential or the reachability, then re-run the action.
|
||||
</Accordion>
|
||||
</AccordionGroup>
|
||||
|
||||
|
||||
@@ -93,9 +93,9 @@ Two cards in the **Fleet · Actions** tab use labels to drive cross-node operati
|
||||
|
||||
### Stop fleet by label
|
||||
|
||||
Type a stack label name; Sencho fans the request out to every node and stops every stack on that node assigned a stack label with the same name. This action targets stack labels only, never node labels. The picker suggests stack labels from across the fleet, read from the control instance so labels on offline nodes appear too, each with its stack and node counts so the scope is unmistakable. The result list shows a per-node breakdown with success and failure counts, and the helper line under the input states the scope: `Stops stacks assigned to this stack label across matching nodes. Node labels are not used by this action.` A confirmation modal titled `Stop all stacks with the stack label "<name>"?` with the **Stop fleet** primary action runs the action.
|
||||
Type a stack label name; Sencho fans the request out to every node and stops every stack on that node assigned a stack label with the same name. This action targets stack labels only, never node labels. The picker queries each reachable node for its own stack labels, so a label that exists only on a remote node still appears, listed once with its combined stack and node counts (and the carrying node names) so the scope is unmistakable. When a node cannot be reached, the picker notes that the suggestions may be incomplete. The result list shows a per-node breakdown with success and failure counts, and the helper line under the input states the scope: `Stops every stack assigned this stack label on every reachable node across the fleet. Node labels are not used by this action.` A confirmation modal titled `Stop all stacks with the stack label "<name>"?` with the **Stop fleet** primary action runs the action.
|
||||
|
||||
A node with no stack label by that name is reported as a failure during the run, so a partial-fleet stop is observable rather than silent.
|
||||
A node that carries no matching stack label is shown as such, and a node Sencho could not reach is reported as unreachable, so a partial-fleet stop is observable rather than silent. Unreachable nodes never block the stop on the reachable ones.
|
||||
|
||||
### Bulk label assign
|
||||
|
||||
@@ -129,7 +129,7 @@ The **Bulk label assign** card is per-node only by design. To re-tag stacks on a
|
||||
The Tags filter aggregates labels across every node in the fleet by name. If the new label only exists on one node and that node was offline at the moment the page loaded, the dropdown may not include it. Refresh **Fleet · Overview** with the **Refresh** button in the toolbar to repull node state.
|
||||
</Accordion>
|
||||
<Accordion title="`Stop fleet by label` reports `No node carries a stack label by that name`">
|
||||
Stack labels are per-node, so the fleet-stop matches by name across nodes. If the stack label you typed only exists on one node and you typed the wrong case (`prod` versus `Prod`), no node will match. The picker suggests stack labels read from the control instance across every configured node, including offline ones; node labels never appear there. Pick from the suggestion list rather than typing freehand to avoid case mistakes.
|
||||
Stack labels are per-node, so the fleet-stop matches by name across nodes. If the stack label you typed only exists on one node and you typed the wrong case (`prod` versus `Prod`), no node will match. The picker queries each reachable node for its own stack labels; node labels never appear there. A label on a node Sencho cannot currently reach will not be suggested, and the picker flags that the list may be incomplete. Pick from the suggestion list rather than typing freehand to avoid case mistakes.
|
||||
</Accordion>
|
||||
<Accordion title="`Bulk label assign` cleared every label on my stacks unexpectedly">
|
||||
The card replaces, it does not merge. Selecting no labels and clicking **Apply to N stack(s)** is the documented way to clear assignments, and the confirmation copy on the **Bulk label assign** modal restates this: `No labels selected, this will clear existing assignments on the selected stacks.` Re-pick the labels you want and run the action again to restore them.
|
||||
|
||||
@@ -146,7 +146,7 @@ it('shows a zero-stack preview when a node-only name is typed', async () => {
|
||||
render(<LabelFleetStopCard />);
|
||||
await user.type(screen.getByPlaceholderText('e.g. production'), 'edge');
|
||||
|
||||
expect(await screen.findByText('No stacks are assigned to this stack label', undefined, { timeout: 2000 })).toBeInTheDocument();
|
||||
expect(await screen.findByText('No reachable node carries a stack label by that name', undefined, { timeout: 2000 })).toBeInTheDocument();
|
||||
await waitFor(() => expect(screen.getByText('0 matching stacks')).toBeInTheDocument());
|
||||
});
|
||||
|
||||
@@ -169,7 +169,7 @@ it('dry run calls fleet-stop with dryRun:true and never opens the confirm modal'
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/fleet/labels/fleet-stop') {
|
||||
return Promise.resolve(jsonResponse(200, {
|
||||
results: [{ nodeId: 1, nodeName: 'central', matched: true, stackResults: [{ stackName: 'web', success: true, dryRun: true }] }],
|
||||
results: [{ nodeId: 1, nodeName: 'central', reachable: true, matched: true, stackResults: [{ stackName: 'web', success: true, dryRun: true }] }],
|
||||
}));
|
||||
}
|
||||
return Promise.resolve(jsonResponse(404, {}));
|
||||
@@ -193,8 +193,8 @@ it('stop fleet opens the confirm modal, and confirming runs a real stop that ren
|
||||
if (url === '/fleet/labels/fleet-stop') {
|
||||
return Promise.resolve(jsonResponse(200, {
|
||||
results: [
|
||||
{ nodeId: 1, nodeName: 'central', matched: true, stackResults: [{ stackName: 'web', success: true }] },
|
||||
{ nodeId: 2, nodeName: 'edge-1', matched: false, stackResults: [] },
|
||||
{ nodeId: 1, nodeName: 'central', reachable: true, matched: true, stackResults: [{ stackName: 'web', success: true }] },
|
||||
{ nodeId: 2, nodeName: 'edge-1', reachable: true, matched: false, stackResults: [] },
|
||||
],
|
||||
}));
|
||||
}
|
||||
@@ -238,7 +238,7 @@ it('populates the blast readout from the debounced match-preview', async () => {
|
||||
const user = userEvent.setup();
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/fleet/labels/match-preview') {
|
||||
return Promise.resolve(jsonResponse(200, { matchedNodes: 2, matchedStacks: 3, perNode: [] }));
|
||||
return Promise.resolve(jsonResponse(200, { matchedNodes: 2, matchedStacks: 3, unreachableNodes: 0, perNode: [] }));
|
||||
}
|
||||
return Promise.resolve(jsonResponse(404, {}));
|
||||
});
|
||||
@@ -246,3 +246,120 @@ it('populates the blast readout from the debounced match-preview', async () => {
|
||||
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
|
||||
await waitFor(() => expect(screen.getByText('3 stacks · 2 nodes')).toBeInTheDocument(), { timeout: 2000 });
|
||||
});
|
||||
|
||||
it('degrades to "preview unavailable" when match-preview returns a malformed 200 body', async () => {
|
||||
const user = userEvent.setup();
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/fleet/labels/match-preview') {
|
||||
// 200 but perNode is missing: the render path must degrade, not throw.
|
||||
return Promise.resolve(jsonResponse(200, { matchedStacks: 1, matchedNodes: 1 }));
|
||||
}
|
||||
return Promise.resolve(jsonResponse(404, {}));
|
||||
});
|
||||
render(<LabelFleetStopCard />);
|
||||
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
|
||||
expect(await screen.findByText('preview endpoint did not respond', undefined, { timeout: 2000 })).toBeInTheDocument();
|
||||
// The card stays interactive; no crash from the missing perNode array.
|
||||
expect(screen.getByRole('button', { name: 'Stop fleet' })).toBeEnabled();
|
||||
});
|
||||
|
||||
it('does not crash when fleet-stop returns a malformed 200 body', async () => {
|
||||
const user = userEvent.setup();
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/fleet/labels/fleet-stop') {
|
||||
// results is not an array: must degrade to a toast, not throw.
|
||||
return Promise.resolve(jsonResponse(200, { results: 'nope' }));
|
||||
}
|
||||
return Promise.resolve(jsonResponse(404, {}));
|
||||
});
|
||||
render(<LabelFleetStopCard />);
|
||||
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
|
||||
await user.click(screen.getByRole('button', { name: 'Dry run' }));
|
||||
// Reported as an unexpected response, not masqueraded as "No reachable nodes".
|
||||
await waitFor(() => expect(toastError).toHaveBeenCalledWith('Fleet stop returned an unexpected response. Check the server logs and retry.'));
|
||||
expect(toastWarning).not.toHaveBeenCalled();
|
||||
expect(screen.getByPlaceholderText('e.g. production')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows remote labels with their node spread and flags partial coverage', async () => {
|
||||
const user = userEvent.setup();
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/fleet/labels/suggestions') {
|
||||
return Promise.resolve(jsonResponse(200, {
|
||||
suggestions: [{ name: 'media', scope: 'stack', nodeCount: 2, stackCount: 3, nodes: ['central', 'edge-1'] }],
|
||||
unreachableNodes: 1,
|
||||
partial: true,
|
||||
}));
|
||||
}
|
||||
return Promise.resolve(jsonResponse(404, {}));
|
||||
});
|
||||
render(<LabelFleetStopCard />);
|
||||
expect(await screen.findByText(/1 node unreachable; suggestions may be incomplete/i)).toBeInTheDocument();
|
||||
await user.click(screen.getByPlaceholderText('e.g. production'));
|
||||
expect(await screen.findByText('media')).toBeInTheDocument();
|
||||
// The node-name spread renders as a muted detail under the label name.
|
||||
expect(screen.getByText('central, edge-1')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders unreachable nodes in the preview without dropping the matched ones', async () => {
|
||||
const user = userEvent.setup();
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/fleet/labels/match-preview') {
|
||||
return Promise.resolve(jsonResponse(200, {
|
||||
matchedNodes: 1,
|
||||
matchedStacks: 1,
|
||||
unreachableNodes: 1,
|
||||
perNode: [
|
||||
{ nodeId: 1, nodeName: 'central', reachable: true, labelExists: true, stackCount: 1, stackNames: ['web'] },
|
||||
{ nodeId: 2, nodeName: 'edge-1', reachable: false, labelExists: false, stackCount: 0, stackNames: [], error: 'Remote node not configured' },
|
||||
],
|
||||
}));
|
||||
}
|
||||
return Promise.resolve(jsonResponse(404, {}));
|
||||
});
|
||||
render(<LabelFleetStopCard />);
|
||||
await user.type(screen.getByPlaceholderText('e.g. production'), 'media');
|
||||
expect(await screen.findByText('web', undefined, { timeout: 2000 })).toBeInTheDocument();
|
||||
expect(screen.getByText('unreachable · 1')).toBeInTheDocument();
|
||||
expect(screen.getByText('edge-1')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('distinguishes "label exists, no stacks" and shows a 0-reachable blast when the rest are unreachable', async () => {
|
||||
const user = userEvent.setup();
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/fleet/labels/match-preview') {
|
||||
return Promise.resolve(jsonResponse(200, {
|
||||
matchedNodes: 0,
|
||||
matchedStacks: 0,
|
||||
unreachableNodes: 1,
|
||||
perNode: [
|
||||
{ nodeId: 1, nodeName: 'central', reachable: true, labelExists: true, stackCount: 0, stackNames: [] },
|
||||
{ nodeId: 2, nodeName: 'edge-1', reachable: false, labelExists: false, stackCount: 0, stackNames: [], error: 'Remote node not configured' },
|
||||
],
|
||||
}));
|
||||
}
|
||||
return Promise.resolve(jsonResponse(404, {}));
|
||||
});
|
||||
render(<LabelFleetStopCard />);
|
||||
await user.type(screen.getByPlaceholderText('e.g. production'), 'media');
|
||||
expect(await screen.findByText('This stack label exists but has no stacks assigned on the reachable nodes', undefined, { timeout: 2000 })).toBeInTheDocument();
|
||||
expect(screen.getByText('unreachable · 1')).toBeInTheDocument();
|
||||
await waitFor(() => expect(screen.getByText('0 reachable · 1 unreachable')).toBeInTheDocument());
|
||||
});
|
||||
|
||||
it('renders an unreachable fleet-stop node as unreachable, not "no matching label", and warns', async () => {
|
||||
const user = userEvent.setup();
|
||||
mockedFetch.mockImplementation((url: string) => {
|
||||
if (url === '/fleet/labels/fleet-stop') {
|
||||
return Promise.resolve(jsonResponse(200, {
|
||||
results: [{ nodeId: 2, nodeName: 'edge-1', reachable: false, matched: false, stackResults: [], error: 'Remote node not configured' }],
|
||||
}));
|
||||
}
|
||||
return Promise.resolve(jsonResponse(404, {}));
|
||||
});
|
||||
render(<LabelFleetStopCard />);
|
||||
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
|
||||
await user.click(screen.getByRole('button', { name: 'Dry run' }));
|
||||
expect(await screen.findByText(/edge-1 \(unreachable\)/)).toBeInTheDocument();
|
||||
await waitFor(() => expect(toastWarning).toHaveBeenCalled());
|
||||
});
|
||||
|
||||
@@ -12,17 +12,20 @@ interface NodeStackResult { stackName: string; success: boolean; error?: string;
|
||||
interface FleetStopNodeResult {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
reachable: boolean;
|
||||
matched: boolean;
|
||||
stackResults: NodeStackResult[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// Stop-by-label targets stack labels only. The `scope: 'stack'` tag keeps node
|
||||
// labels (a separate namespace) from ever being fed into this destructive card,
|
||||
// and the counts make the stack scope tangible in the picker.
|
||||
interface FleetStopLabelSuggestion { name: string; scope: 'stack'; nodeCount: number; stackCount: number }
|
||||
// and the counts make the stack scope tangible in the picker. `nodes` lists the
|
||||
// nodes that carry the label so the operator can see the spread before acting.
|
||||
interface FleetStopLabelSuggestion { name: string; scope: 'stack'; nodeCount: number; stackCount: number; nodes: string[] }
|
||||
|
||||
interface MatchPreviewNode { nodeId: number; nodeName: string; stackCount: number; stackNames: string[] }
|
||||
interface MatchPreviewResponse { matchedNodes: number; matchedStacks: number; perNode: MatchPreviewNode[] }
|
||||
interface MatchPreviewNode { nodeId: number; nodeName: string; reachable: boolean; labelExists: boolean; stackCount: number; stackNames: string[]; error?: string }
|
||||
interface MatchPreviewResponse { matchedNodes: number; matchedStacks: number; unreachableNodes: number; perNode: MatchPreviewNode[] }
|
||||
|
||||
type PreviewState =
|
||||
| { kind: 'idle' }
|
||||
@@ -36,24 +39,50 @@ const PREVIEW_ROW_LIMIT = 6;
|
||||
function isSuggestion(value: unknown): value is FleetStopLabelSuggestion {
|
||||
if (typeof value !== 'object' || value === null) return false;
|
||||
const s = value as Record<string, unknown>;
|
||||
// `nodes` is tolerated as absent (coerced to [] at render) so a suggestion is
|
||||
// never dropped over a missing optional detail field.
|
||||
const nodesOk = s.nodes === undefined
|
||||
|| (Array.isArray(s.nodes) && s.nodes.every(n => typeof n === 'string'));
|
||||
return typeof s.name === 'string' && s.scope === 'stack'
|
||||
&& typeof s.nodeCount === 'number' && typeof s.stackCount === 'number';
|
||||
&& typeof s.nodeCount === 'number' && typeof s.stackCount === 'number' && nodesOk;
|
||||
}
|
||||
|
||||
// The preview section renders `perNode` (filter/some/flatMap) outside any
|
||||
// try/catch, so a malformed success body would throw during render rather than
|
||||
// degrade. Validate the load-bearing shape before trusting it; anything off
|
||||
// falls to the "unavailable" state.
|
||||
function isMatchPreviewResponse(value: unknown): value is MatchPreviewResponse {
|
||||
if (typeof value !== 'object' || value === null) return false;
|
||||
const d = value as Record<string, unknown>;
|
||||
if (typeof d.matchedNodes !== 'number' || typeof d.matchedStacks !== 'number') return false;
|
||||
if (!Array.isArray(d.perNode)) return false;
|
||||
return d.perNode.every((n) => {
|
||||
if (typeof n !== 'object' || n === null) return false;
|
||||
const node = n as Record<string, unknown>;
|
||||
return typeof node.nodeId === 'number'
|
||||
&& typeof node.nodeName === 'string'
|
||||
&& typeof node.reachable === 'boolean'
|
||||
&& typeof node.stackCount === 'number'
|
||||
&& Array.isArray(node.stackNames);
|
||||
});
|
||||
}
|
||||
|
||||
export function LabelFleetStopCard() {
|
||||
const [labelName, setLabelName] = useState('');
|
||||
const [suggestions, setSuggestions] = useState<FleetStopLabelSuggestion[]>([]);
|
||||
const [suggestUnreachable, setSuggestUnreachable] = useState(0);
|
||||
const [confirmOpen, setConfirmOpen] = useState(false);
|
||||
const [running, setRunning] = useState(false);
|
||||
const [results, setResults] = useState<ResultRow[]>([]);
|
||||
const [preview, setPreview] = useState<PreviewState>({ kind: 'idle' });
|
||||
|
||||
// Stack-label suggestions for the target picker. The fleet endpoint aggregates
|
||||
// the stack_labels rows across every configured node (central DB), so node
|
||||
// labels can never appear here. A non-ok or malformed response leaves the list
|
||||
// empty rather than crashing: the Actions tab renders without an admin gate
|
||||
// while the endpoint is admin-only, so a viewer simply gets no suggestions and
|
||||
// can still type a name by hand.
|
||||
// Stack-label suggestions for the target picker. The fleet endpoint queries
|
||||
// every node authoritatively (local DB + live remote reads), so node labels
|
||||
// can never appear here and remote-only stack labels do. `unreachableNodes`
|
||||
// flags that the counts cover only the nodes Sencho could reach. A non-ok or
|
||||
// malformed response leaves the list empty rather than crashing: the Actions
|
||||
// tab renders without an admin gate while the endpoint is admin-only, so a
|
||||
// viewer simply gets no suggestions and can still type a name by hand.
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
async function loadSuggestions() {
|
||||
@@ -61,9 +90,12 @@ export function LabelFleetStopCard() {
|
||||
const res = await apiFetch('/fleet/labels/suggestions');
|
||||
const data = res.ok ? await res.json().catch(() => null) : null;
|
||||
const list = Array.isArray(data?.suggestions) ? data.suggestions.filter(isSuggestion) : [];
|
||||
if (!cancelled) setSuggestions(list);
|
||||
if (!cancelled) {
|
||||
setSuggestions(list);
|
||||
setSuggestUnreachable(typeof data?.unreachableNodes === 'number' ? data.unreachableNodes : 0);
|
||||
}
|
||||
} catch {
|
||||
if (!cancelled) setSuggestions([]);
|
||||
if (!cancelled) { setSuggestions([]); setSuggestUnreachable(0); }
|
||||
}
|
||||
}
|
||||
loadSuggestions();
|
||||
@@ -95,9 +127,20 @@ export function LabelFleetStopCard() {
|
||||
setPreview({ kind: 'unavailable' });
|
||||
return;
|
||||
}
|
||||
const data = (await res.json()) as MatchPreviewResponse;
|
||||
const data = await res.json().catch(() => null);
|
||||
if (!isMatchPreviewResponse(data)) {
|
||||
// A 200 with a shape we can't render is a server/contract bug, not a
|
||||
// transport miss; log it like the catch path so it's diagnosable.
|
||||
console.error('[FleetStop] match-preview returned a malformed body', data);
|
||||
setPreview({ kind: 'unavailable' });
|
||||
return;
|
||||
}
|
||||
setPreview({ kind: 'ready', data });
|
||||
} catch {
|
||||
} catch (err) {
|
||||
// Non-destructive readout: the operator can still type a name and run the
|
||||
// stop, so we degrade to "unavailable" rather than toasting, but leave a
|
||||
// console trail so a recurring preview failure is diagnosable.
|
||||
console.error('[FleetStop] match-preview failed', err);
|
||||
setPreview({ kind: 'unavailable' });
|
||||
}
|
||||
}, 500);
|
||||
@@ -112,9 +155,12 @@ export function LabelFleetStopCard() {
|
||||
if (preview.kind === 'loading') return 'resolving…';
|
||||
if (preview.kind === 'unavailable') return 'preview unavailable';
|
||||
if (preview.kind === 'ready') {
|
||||
const { matchedNodes, matchedStacks } = preview.data;
|
||||
if (matchedStacks === 0 || matchedNodes === 0) return '0 matching stacks';
|
||||
return `${matchedStacks} stacks · ${matchedNodes} nodes`;
|
||||
const { matchedNodes, matchedStacks, unreachableNodes } = preview.data;
|
||||
if (matchedStacks === 0 || matchedNodes === 0) {
|
||||
return unreachableNodes > 0 ? `0 reachable · ${unreachableNodes} unreachable` : '0 matching stacks';
|
||||
}
|
||||
const suffix = unreachableNodes > 0 ? ` · ${unreachableNodes} unreachable` : '';
|
||||
return `${matchedStacks} stacks · ${matchedNodes} nodes${suffix}`;
|
||||
}
|
||||
return 'awaiting target';
|
||||
}, [labelName, preview]);
|
||||
@@ -139,31 +185,59 @@ export function LabelFleetStopCard() {
|
||||
toast.error(body.error || 'Fleet stop failed');
|
||||
return;
|
||||
}
|
||||
const apiResults = (body.results as FleetStopNodeResult[]) ?? [];
|
||||
const rows: ResultRow[] = apiResults.map((node) => ({
|
||||
key: `node-${node.nodeId}`,
|
||||
label: node.matched
|
||||
? `${node.nodeName} · ${node.stackResults.length} stack${node.stackResults.length === 1 ? '' : 's'}${opts.dryRun ? ' (dry run)' : ''}`
|
||||
: `${node.nodeName} (no matching stack label)`,
|
||||
success: node.matched && node.stackResults.every(s => s.success),
|
||||
error: node.matched ? undefined : 'Stack label not present',
|
||||
sub: node.stackResults.map((s, i) => ({
|
||||
key: `${node.nodeId}-${s.stackName}-${i}`,
|
||||
label: s.stackName,
|
||||
success: s.success,
|
||||
error: s.error,
|
||||
})),
|
||||
}));
|
||||
// A 200 with a non-array `results` is a server/contract bug, not an empty
|
||||
// fleet: don't let it masquerade as the legitimate "No reachable nodes"
|
||||
// outcome (a valid empty array below still reports that honestly). Log it
|
||||
// and tell the operator it was unexpected. A node's `stackResults` is
|
||||
// guarded per node further down so one bad node never nukes the readout.
|
||||
if (!Array.isArray(body.results)) {
|
||||
console.error('[FleetStop] fleet-stop returned a malformed body', body);
|
||||
toast.error('Fleet stop returned an unexpected response. Check the server logs and retry.');
|
||||
return;
|
||||
}
|
||||
const apiResults = body.results as FleetStopNodeResult[];
|
||||
const rows: ResultRow[] = apiResults.map((node) => {
|
||||
if (!node.reachable) {
|
||||
return {
|
||||
key: `node-${node.nodeId}`,
|
||||
label: `${node.nodeName} (unreachable)`,
|
||||
success: false,
|
||||
error: node.error || 'Node unreachable',
|
||||
sub: [],
|
||||
};
|
||||
}
|
||||
const stackResults = Array.isArray(node.stackResults) ? node.stackResults : [];
|
||||
return {
|
||||
key: `node-${node.nodeId}`,
|
||||
label: node.matched
|
||||
? `${node.nodeName} · ${stackResults.length} stack${stackResults.length === 1 ? '' : 's'}${opts.dryRun ? ' (dry run)' : ''}`
|
||||
: `${node.nodeName} (no matching stack label)`,
|
||||
success: node.matched && stackResults.every(s => s.success),
|
||||
error: node.matched ? undefined : 'Stack label not present',
|
||||
sub: stackResults.map((s, i) => ({
|
||||
key: `${node.nodeId}-${s.stackName}-${i}`,
|
||||
label: s.stackName,
|
||||
success: s.success,
|
||||
error: s.error,
|
||||
})),
|
||||
};
|
||||
});
|
||||
setResults(rows);
|
||||
const matchedNodes = apiResults.filter(n => n.matched).length;
|
||||
const stacksTouched = apiResults.flatMap(n => n.stackResults);
|
||||
const reachable = apiResults.filter(n => n.reachable);
|
||||
const unreachableCount = apiResults.length - reachable.length;
|
||||
const matchedNodes = reachable.filter(n => n.matched).length;
|
||||
const stacksTouched = apiResults.flatMap(n => Array.isArray(n.stackResults) ? n.stackResults : []);
|
||||
const ok = stacksTouched.filter(s => s.success).length;
|
||||
const failed = stacksTouched.length - ok;
|
||||
if (matchedNodes === 0) toast.info('No node carries a stack label by that name.');
|
||||
else if (opts.dryRun) toast.success(`Dry run: would stop ${ok} stack${ok === 1 ? '' : 's'} across ${matchedNodes} node${matchedNodes === 1 ? '' : 's'}.`);
|
||||
else if (failed === 0 && ok > 0) toast.success(`Stopped ${ok} stack${ok === 1 ? '' : 's'} across ${matchedNodes} node${matchedNodes === 1 ? '' : 's'}.`);
|
||||
else if (ok === 0 && failed === 0) toast.info('Stack label matched but no stacks were assigned to it.');
|
||||
else toast.warning(`${ok} stopped, ${failed} failed. See results below.`);
|
||||
const unreachableSuffix = unreachableCount > 0 ? ` ${unreachableCount} node${unreachableCount === 1 ? '' : 's'} unreachable.` : '';
|
||||
if (matchedNodes === 0) {
|
||||
if (reachable.length === 0) toast.warning(`No reachable nodes.${unreachableSuffix}`);
|
||||
else toast.info(`No reachable node carries a stack label by that name.${unreachableSuffix}`);
|
||||
}
|
||||
else if (opts.dryRun) toast.success(`Dry run: would stop ${ok} stack${ok === 1 ? '' : 's'} across ${matchedNodes} node${matchedNodes === 1 ? '' : 's'}.${unreachableSuffix}`);
|
||||
else if (failed === 0 && ok > 0) toast.success(`Stopped ${ok} stack${ok === 1 ? '' : 's'} across ${matchedNodes} node${matchedNodes === 1 ? '' : 's'}.${unreachableSuffix}`);
|
||||
else if (ok === 0 && failed === 0) toast.info(`Stack label matched but no stacks were assigned to it.${unreachableSuffix}`);
|
||||
else toast.warning(`${ok} stopped, ${failed} failed. See results below.${unreachableSuffix}`);
|
||||
} catch (err) {
|
||||
toast.dismiss(toastId);
|
||||
toast.error(err instanceof Error ? err.message : 'Network error');
|
||||
@@ -208,8 +282,13 @@ export function LabelFleetStopCard() {
|
||||
meta={`stack labels · ${suggestions.length}`}
|
||||
>
|
||||
<p className={cn(KICKER, 'text-stat-subtitle mb-2 normal-case tracking-normal text-[11px]')}>
|
||||
Stops stacks assigned to this stack label across matching nodes. Node labels are not used by this action.
|
||||
Stops every stack assigned this stack label on every reachable node across the fleet. Node labels are not used by this action.
|
||||
</p>
|
||||
{suggestUnreachable > 0 && (
|
||||
<p className={cn(KICKER, 'text-stat-icon mb-2 normal-case tracking-normal text-[11px]')}>
|
||||
{suggestUnreachable} node{suggestUnreachable === 1 ? '' : 's'} unreachable; suggestions may be incomplete.
|
||||
</p>
|
||||
)}
|
||||
<LabelAutocomplete
|
||||
value={labelName}
|
||||
onChange={setLabelName}
|
||||
@@ -234,7 +313,7 @@ export function LabelFleetStopCard() {
|
||||
variant="destructive"
|
||||
kicker="Fleet stop"
|
||||
title={`Stop all stacks with the stack label "${trimmed}"?`}
|
||||
description="Sencho will stop every stack assigned this stack label on every node. Node labels are not used by this action. Services will be unavailable until restarted."
|
||||
description="Sencho will stop every stack assigned this stack label on every reachable node at execution time. Node labels are not used by this action. Services will be unavailable until restarted."
|
||||
confirmLabel="Stop fleet"
|
||||
confirming={running}
|
||||
onConfirm={() => run({ dryRun: false })}
|
||||
@@ -260,26 +339,55 @@ function renderPreviewSection(preview: PreviewState, trimmed: string) {
|
||||
);
|
||||
}
|
||||
if (preview.kind === 'ready') {
|
||||
const { matchedStacks, matchedNodes, perNode } = preview.data;
|
||||
const { matchedStacks, matchedNodes, unreachableNodes, perNode } = preview.data;
|
||||
const unreachable = perNode.filter(n => !n.reachable);
|
||||
const matchingNodes = perNode.filter(n => n.reachable && n.stackCount > 0);
|
||||
if (matchedStacks === 0) {
|
||||
// Distinguish "label exists but no stacks" from "no such label", and still
|
||||
// surface unreachable nodes so a 0-count never hides a node we could not ask.
|
||||
const labelExistsSomewhere = perNode.some(n => n.reachable && n.labelExists);
|
||||
const message = labelExistsSomewhere
|
||||
? 'This stack label exists but has no stacks assigned on the reachable nodes'
|
||||
: 'No reachable node carries a stack label by that name';
|
||||
return (
|
||||
<SheetSection title="Preview" meta="0 stacks">
|
||||
<div className={cn(KICKER, 'text-stat-icon')}>No stacks are assigned to this stack label</div>
|
||||
<SheetSection title="Preview" meta={unreachableNodes > 0 ? `${unreachableNodes} unreachable` : '0 stacks'}>
|
||||
<div className={cn(KICKER, 'text-stat-icon normal-case tracking-normal text-[11px]')}>{message}</div>
|
||||
{unreachable.length > 0 && <UnreachableList nodes={unreachable} />}
|
||||
</SheetSection>
|
||||
);
|
||||
}
|
||||
const meta = `across ${matchedNodes} node${matchedNodes === 1 ? '' : 's'}${unreachableNodes > 0 ? ` · ${unreachableNodes} unreachable` : ''}`;
|
||||
return (
|
||||
<SheetSection
|
||||
title={`Preview · ${matchedStacks} stack${matchedStacks === 1 ? '' : 's'}`}
|
||||
meta={`across ${matchedNodes} node${matchedNodes === 1 ? '' : 's'}`}
|
||||
meta={meta}
|
||||
>
|
||||
<PreviewWell perNode={perNode} />
|
||||
<PreviewWell perNode={matchingNodes} />
|
||||
{unreachable.length > 0 && <UnreachableList nodes={unreachable} />}
|
||||
</SheetSection>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Muted list of nodes Sencho could not query, shown beneath the preview so a
|
||||
// partial blast radius is observable rather than silently dropped.
|
||||
function UnreachableList({ nodes }: { nodes: MatchPreviewNode[] }) {
|
||||
return (
|
||||
<div className="mt-2 rounded border border-card-border/60 bg-card/40 p-2">
|
||||
<div className={cn(KICKER, 'text-stat-icon mb-1')}>unreachable · {nodes.length}</div>
|
||||
<ul className="space-y-1">
|
||||
{nodes.map(n => (
|
||||
<li key={n.nodeId} className="flex items-center gap-2">
|
||||
<span className="flex-1 min-w-0 truncate font-mono text-[11px] text-stat-subtitle">{n.nodeName}</span>
|
||||
{n.error && <span className={cn(KICKER, 'shrink-0 max-w-[55%] truncate text-stat-icon')}>{n.error}</span>}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface PreviewWellProps {
|
||||
perNode: MatchPreviewNode[];
|
||||
}
|
||||
@@ -382,22 +490,31 @@ function LabelAutocomplete({ value, onChange, suggestions, disabled, placeholder
|
||||
{open && filtered.length > 0 && (
|
||||
<div className="absolute left-0 top-full mt-1 z-50 w-full rounded-md border border-glass-border bg-popover text-popover-foreground shadow-md backdrop-blur-[10px] backdrop-saturate-[1.15]">
|
||||
<ul className="max-h-[200px] overflow-y-auto overflow-x-hidden p-1">
|
||||
{filtered.map((s) => (
|
||||
{filtered.map((s) => {
|
||||
const nodes = s.nodes ?? [];
|
||||
return (
|
||||
<li key={s.name}>
|
||||
<button
|
||||
type="button"
|
||||
// mousedown + preventDefault keeps the input focused so the
|
||||
// selection registers before any blur-driven close fires.
|
||||
onMouseDown={(e) => { e.preventDefault(); handleSelect(s.name); }}
|
||||
className="flex w-full items-center gap-2 rounded-sm px-2 py-1.5 font-mono text-xs text-stat-value hover:bg-accent hover:text-accent-foreground"
|
||||
title={nodes.join(', ')}
|
||||
className="flex w-full flex-col gap-0.5 rounded-sm px-2 py-1.5 font-mono text-xs text-stat-value hover:bg-accent hover:text-accent-foreground"
|
||||
>
|
||||
<span className="flex-1 min-w-0 truncate text-left">{s.name}</span>
|
||||
<span className="shrink-0 text-[10px] text-stat-subtitle">
|
||||
{s.stackCount} stack{s.stackCount === 1 ? '' : 's'} · {s.nodeCount} node{s.nodeCount === 1 ? '' : 's'}
|
||||
<span className="flex w-full items-center gap-2">
|
||||
<span className="flex-1 min-w-0 truncate text-left">{s.name}</span>
|
||||
<span className="shrink-0 text-[10px] text-stat-subtitle">
|
||||
{s.stackCount} stack{s.stackCount === 1 ? '' : 's'} · {s.nodeCount} node{s.nodeCount === 1 ? '' : 's'}
|
||||
</span>
|
||||
</span>
|
||||
{nodes.length > 0 && (
|
||||
<span className="w-full truncate text-left text-[10px] text-stat-icon">{nodes.join(', ')}</span>
|
||||
)}
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
|
||||
Reference in New Issue
Block a user