fix(fleet-actions): stop-by-label works on Community remote nodes (#1270)

* fix(fleet-actions): stop-by-label works on Community remote nodes

Fleet-stop's remote leg fanned out to POST /api/labels/:id/action, which
is gated to Skipper/Admiral, so on a Community fleet the control node
stopped its own stacks but every remote node returned 403. Fleet-stop
itself is admin-only and available on every license, so the remote leg
contradicted the feature's own gate.

Extract the label-match plus bulk-stop logic into a shared
runLocalLabelStop helper and add an admin-only, every-license
POST /api/fleet-actions/labels/local-stop receiver. The control now fans
out to that receiver, so remote stacks stop on every tier. Each node runs
under its own per-node bulk lock, so a fleet-stop and a per-label action
still serialize cleanly instead of double-stopping containers.

Also degrade the control's own leg per-node instead of failing the whole
fan-out when its filesystem read throws, and gate fleet-stop and
fleet-prune diagnostics behind developer_mode.

Tests: local-stop auth, tier, validation, and behavior; a remote-leg
routing guard that asserts the fan-out targets local-stop and never the
paid route; local-leg graceful degradation; and the three Fleet Action
card UIs.

* fix(fleet-actions): honor the remote stop receiver's matched flag

The control reached the remote leg only because its own mirror had the
label, then hardcoded matched:true and trusted results without guarding
its shape. A mirror-skewed control (mirror has the label, remote does
not) then showed a remote mismatch as "matched, 0 stacks" instead of
"no matching label", and a malformed 200 body could flow a non-array
into the per-stack renderers.

Honor the remote's own matched flag and coerce results to an array when
the body is malformed. Add regression tests for the matched:false skew
case and the non-array results case.
This commit is contained in:
Anso
2026-06-01 14:18:44 -04:00
committed by GitHub
parent 085267b466
commit 7e0cffa376
9 changed files with 797 additions and 44 deletions
@@ -244,6 +244,103 @@ describe('POST /api/fleet/prune/estimate', () => {
});
});
describe('POST /api/fleet/labels/fleet-stop remote leg', () => {
// Guards the C-1 fix: the remote fan-out must target the admin-only
// /api/fleet-actions/labels/local-stop receiver (reachable on every license),
// never the paid /api/labels/:id/action it used to call, which 403'd on
// Community remotes.
it('fans out to the admin-only local-stop receiver, never the paid per-label action route', async () => {
const remoteId = db.addNode({
name: 'remote-stop',
type: 'remote',
api_url: 'http://remote-stop.example:1852',
api_token: 'tok',
compose_dir: '/app/compose',
is_default: false,
});
try {
const label = db.createLabel(remoteId, `remote-c1-${++labelCounter}`, 'teal');
db.setStackLabels('alpha', remoteId, [label.id]);
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue({
ok: true,
status: 200,
json: async () => ({ matched: true, results: [{ stackName: 'alpha', success: true }] }),
} as unknown as Response);
const res = await request(app)
.post('/api/fleet/labels/fleet-stop')
.set('Authorization', authHeader)
.send({ labelName: label.name });
expect(res.status).toBe(200);
const urls = fetchSpy.mock.calls.map(c => String(c[0]));
expect(urls.some(u => u.endsWith('/api/fleet-actions/labels/local-stop'))).toBe(true);
expect(urls.some(u => u.includes('/api/labels/'))).toBe(false);
const call = fetchSpy.mock.calls.find(c => String(c[0]).endsWith('/api/fleet-actions/labels/local-stop'));
expect(JSON.parse((call![1] as RequestInit).body as string)).toEqual({ labelName: label.name, dryRun: false });
const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
expect(remoteRow.stackResults).toEqual([{ stackName: 'alpha', success: true }]);
} finally {
db.deleteNode(remoteId);
}
});
it('honors the remote matched:false flag over the control mirror (mirror skew)', async () => {
const remoteId = db.addNode({
name: 'remote-skew', type: 'remote', api_url: 'http://remote-skew.example:1852',
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
});
try {
// The control mirror believes the remote carries this label + stack...
const label = db.createLabel(remoteId, `remote-skew-${++labelCounter}`, 'teal');
db.setStackLabels('alpha', remoteId, [label.id]);
// ...but the remote authoritatively reports it has no such label.
vi.spyOn(globalThis, 'fetch').mockResolvedValue({
ok: true, status: 200, json: async () => ({ matched: false, results: [] }),
} as unknown as Response);
const res = await request(app)
.post('/api/fleet/labels/fleet-stop')
.set('Authorization', authHeader)
.send({ labelName: label.name });
expect(res.status).toBe(200);
const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
expect(remoteRow.matched).toBe(false);
expect(remoteRow.stackResults).toEqual([]);
} finally {
db.deleteNode(remoteId);
}
});
it('degrades a malformed 200 body (non-array results) to empty instead of forwarding it', async () => {
const remoteId = db.addNode({
name: 'remote-malformed', type: 'remote', api_url: 'http://remote-malformed.example:1852',
api_token: 'tok', compose_dir: '/app/compose', is_default: false,
});
try {
const label = db.createLabel(remoteId, `remote-malformed-${++labelCounter}`, 'teal');
db.setStackLabels('alpha', remoteId, [label.id]);
vi.spyOn(globalThis, 'fetch').mockResolvedValue({
ok: true, status: 200, json: async () => ({ matched: true, results: 'not-an-array' }),
} as unknown as Response);
const res = await request(app)
.post('/api/fleet/labels/fleet-stop')
.set('Authorization', authHeader)
.send({ labelName: label.name });
expect(res.status).toBe(200);
const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
expect(remoteRow.stackResults).toEqual([]);
} finally {
db.deleteNode(remoteId);
}
});
});
describe('POST /api/fleet/labels/fleet-stop with dryRun: true', () => {
it('marks each stack dryRun: true and does not invoke containerActionForStack', async () => {
const label = await createAssignedLabel('dry-stop', ['alpha', 'beta']);
+146
View File
@@ -3,6 +3,8 @@
* validation, and orchestration shape across the two routes.
*/
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
import fs from 'fs';
import path from 'path';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
@@ -138,3 +140,147 @@ describe('Fleet Actions orchestration shape', () => {
expect(res.body.results[0]).toMatchObject({ success: false, error: 'Invalid stack name' });
});
});
// The per-node local-stop receiver is what a control instance calls on each
// remote during a fleet-wide stop. It must be reachable on every license (only
// admin-gated): the original fleet-stop fan-out hit the paid /api/labels/:id/action
// and 403'd on Community remotes. These tests lock that behavior in.
describe('local-stop receiver auth + tier', () => {
afterEach(() => vi.restoreAllMocks());
it('POST /api/fleet-actions/labels/local-stop returns 401 without auth', async () => {
const res = await request(app).post('/api/fleet-actions/labels/local-stop').send({ labelName: 'prod' });
expect(res.status).toBe(401);
});
it('is reachable on community tier for admins and never returns PAID_REQUIRED', async () => {
mockTier('community');
const res = await request(app)
.post('/api/fleet-actions/labels/local-stop')
.set('Authorization', authHeader)
.send({ labelName: 'this-label-does-not-exist' });
expect(res.status).toBe(200);
expect(res.body.code).not.toBe('PAID_REQUIRED');
expect(res.body).toEqual({ matched: false, results: [] });
});
it('rejects missing labelName', async () => {
const res = await request(app)
.post('/api/fleet-actions/labels/local-stop')
.set('Authorization', authHeader)
.send({});
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/labelName/);
});
it('rejects whitespace-only labelName', async () => {
const res = await request(app)
.post('/api/fleet-actions/labels/local-stop')
.set('Authorization', authHeader)
.send({ labelName: ' ' });
expect(res.status).toBe(400);
});
});
describe('local-stop behavior', () => {
let db: import('../services/DatabaseService').DatabaseService;
let nodeId: number;
beforeAll(async () => {
const { DatabaseService } = await import('../services/DatabaseService');
const { NodeRegistry } = await import('../services/NodeRegistry');
db = DatabaseService.getInstance();
nodeId = NodeRegistry.getInstance().getDefaultNodeId();
});
afterEach(() => vi.restoreAllMocks());
it('matched:true with empty results when the label exists but has no stacks', async () => {
db.createLabel(nodeId, 'no-stacks-label', '#ffffff');
const res = await request(app)
.post('/api/fleet-actions/labels/local-stop')
.set('Authorization', authHeader)
.send({ labelName: 'no-stacks-label' });
expect(res.status).toBe(200);
expect(res.body).toEqual({ matched: true, results: [] });
});
it('reports per-stack lock contention when a bulk action is already running on the node', async () => {
const label = db.createLabel(nodeId, 'busy-label', '#ffffff');
db.setStackLabels('busy-stack', nodeId, [label.id]);
const { activeBulkActions } = await import('../routes/labels');
activeBulkActions.add(`bulk:${nodeId}`);
try {
const res = await request(app)
.post('/api/fleet-actions/labels/local-stop')
.set('Authorization', authHeader)
.send({ labelName: 'busy-label' });
expect(res.status).toBe(200);
expect(res.body.matched).toBe(true);
expect(res.body.results).toEqual([
{ stackName: 'busy-stack', success: false, error: 'A bulk action is already running on this node' },
]);
} finally {
activeBulkActions.delete(`bulk:${nodeId}`);
}
});
it('dry run returns dryRun:true per on-disk stack without touching Docker', async () => {
const composeDir = process.env.COMPOSE_DIR as string;
fs.mkdirSync(path.join(composeDir, 'dry-stack'), { recursive: true });
fs.writeFileSync(path.join(composeDir, 'dry-stack', 'docker-compose.yml'), 'services: {}\n');
const label = db.createLabel(nodeId, 'dry-label', '#ffffff');
db.setStackLabels('dry-stack', nodeId, [label.id]);
const res = await request(app)
.post('/api/fleet-actions/labels/local-stop')
.set('Authorization', authHeader)
.send({ labelName: 'dry-label', dryRun: true });
expect(res.status).toBe(200);
expect(res.body.matched).toBe(true);
expect(res.body.results).toEqual([{ stackName: 'dry-stack', success: true, dryRun: true }]);
});
it('filters out assigned stacks that are not present on disk', async () => {
const label = db.createLabel(nodeId, 'ghost-label', '#ffffff');
db.setStackLabels('ghost-stack', nodeId, [label.id]);
const res = await request(app)
.post('/api/fleet-actions/labels/local-stop')
.set('Authorization', authHeader)
.send({ labelName: 'ghost-label' });
expect(res.status).toBe(200);
expect(res.body).toEqual({ matched: true, results: [] });
});
});
describe('fleet-stop degrades the local leg per-node instead of failing the whole fan-out', () => {
let db: import('../services/DatabaseService').DatabaseService;
let nodeId: number;
beforeAll(async () => {
const { DatabaseService } = await import('../services/DatabaseService');
const { NodeRegistry } = await import('../services/NodeRegistry');
db = DatabaseService.getInstance();
nodeId = NodeRegistry.getInstance().getDefaultNodeId();
});
afterEach(() => vi.restoreAllMocks());
it('returns 200 with per-stack errors when the control filesystem read throws', async () => {
const label = db.createLabel(nodeId, 'degrade-label', '#ffffff');
db.setStackLabels('degrade-stack', nodeId, [label.id]);
const { FileSystemService } = await import('../services/FileSystemService');
vi.spyOn(FileSystemService.prototype, 'getStacks').mockRejectedValue(new Error('compose dir unreadable'));
const res = await request(app)
.post('/api/fleet/labels/fleet-stop')
.set('Authorization', authHeader)
.send({ labelName: 'degrade-label' });
expect(res.status).toBe(200);
const localRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === nodeId);
expect(localRow.matched).toBe(true);
expect(localRow.stackResults).toEqual([
{ stackName: 'degrade-stack', success: false, error: 'compose dir unreadable' },
]);
});
});
+87
View File
@@ -0,0 +1,87 @@
import { DatabaseService } from '../services/DatabaseService';
import { FileSystemService } from '../services/FileSystemService';
import { containerActionForStack } from '../routes/stacks';
import { activeBulkActions } from '../routes/labels';
import { invalidateNodeCaches } from './cacheInvalidation';
export interface StackStopResult {
stackName: string;
success: boolean;
error?: string;
dryRun?: boolean;
}
export interface LabelStopOutcome {
matched: boolean;
stackResults: StackStopResult[];
}
/**
* Wire shape of `POST /api/fleet-actions/labels/local-stop`. The in-process
* helper returns `stackResults`; the HTTP response names the same array
* `results` to match the fleet-stop fan-out's existing remote contract. Keep
* the rename in this one type so the producer and the control-side consumer
* cannot drift.
*/
export interface LabelLocalStopResponse {
matched: boolean;
results: StackStopResult[];
}
/**
* Run a label-name-matched container stop against one node's own local Docker.
*
* Used by the gateway-orchestrated fleet-stop for the control node's own stacks
* and by the per-node `POST /api/fleet-actions/labels/local-stop` receiver that
* a control instance calls on each remote during a fleet-wide stop. Matching by
* name (not by a shared label id) keeps the work self-contained on the executing
* node, so the control never has to assert that its mirrored label ids line up
* with the remote's local ids.
*
* Shares the per-node `bulk:<nodeId>` lock with `POST /api/labels/:id/action` so
* a fleet-stop and a per-label action cannot double-stop the same containers.
*/
export async function runLocalLabelStop(
nodeId: number,
labelName: string,
dryRun: boolean,
): Promise<LabelStopOutcome> {
const db = DatabaseService.getInstance();
const label = db.getLabels(nodeId).find(l => l.name === labelName);
if (!label) return { matched: false, stackResults: [] };
const stackNames = db.getStacksForLabel(label.id, nodeId);
if (stackNames.length === 0) return { matched: true, stackResults: [] };
const lockKey = `bulk:${nodeId}`;
if (activeBulkActions.has(lockKey)) {
return {
matched: true,
stackResults: stackNames.map(stackName => ({
stackName,
success: false,
error: 'A bulk action is already running on this node',
})),
};
}
activeBulkActions.add(lockKey);
try {
const fsStacks = await FileSystemService.getInstance(nodeId).getStacks();
const fsStackSet = new Set(fsStacks);
const validStacks = stackNames.filter(name => fsStackSet.has(name));
const stackResults: StackStopResult[] = [];
for (const stackName of validStacks) {
if (dryRun) {
stackResults.push({ stackName, success: true, dryRun: true });
continue;
}
const outcome = await containerActionForStack(nodeId, stackName, 'stop');
if (outcome.kind === 'ok') stackResults.push({ stackName, success: true });
else if (outcome.kind === 'no-containers') stackResults.push({ stackName, success: false, error: 'No containers found for this stack' });
else stackResults.push({ stackName, success: false, error: outcome.message });
}
if (!dryRun && stackResults.some(r => r.success)) invalidateNodeCaches(nodeId);
return { matched: true, stackResults };
} finally {
activeBulkActions.delete(lockKey);
}
}
+64 -43
View File
@@ -34,8 +34,8 @@ import { formatNoTargetError } from '../utils/remoteTarget';
import { CloudBackupService } from '../services/CloudBackupService';
import { NotificationService } from '../services/NotificationService';
import { invalidateNodeCaches, invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
import { containerActionForStack } from './stacks';
import { activeBulkActions } from './labels';
import { runLocalLabelStop, type LabelLocalStopResponse, type StackStopResult } from '../helpers/fleetLabelStop';
import { buildLocalConfigurationStatus, type ConfigurationStatus } from './dashboard';
import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from '../services/license-headers';
import { LicenseService } from '../services/LicenseService';
@@ -1047,6 +1047,12 @@ fleetRouter.delete('/update-status', authMiddleware, async (req: Request, res: R
// in `routes/fleetActions.ts`. The endpoint below is gateway-orchestrated and
// lives here so it sits behind the `/api/fleet/` proxy-exempt prefix.
// Attribute one error to every stack a node was supposed to act on. Used for
// the fleet-stop failure paths (no proxy target, non-ok remote, transport
// error, local exception) so each stack carries the same node-level cause.
const failAllStacks = (stacks: string[], error: string): StackStopResult[] =>
stacks.map(stackName => ({ stackName, success: false, error }));
// Fleet-wide stop by label name. Matches each node's labels by name and runs
// container stops on each matching stack.
// Tier: requireAdmin (admin-only fleet plumbing; available on every license).
@@ -1067,7 +1073,36 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
try {
const db = DatabaseService.getInstance();
const nodes = db.getNodes();
if (isDebugEnabled()) console.debug('[Fleet:debug] fleet-stop:', { labelName: trimmed, dryRun: isDryRun, nodes: nodes.length });
const results = await Promise.all(nodes.map(async (node) => {
if (node.type === 'local') {
// Match + stop runs in-process against the control's own Docker. The
// helper shares the per-node `bulk:<id>` lock with the per-label action
// route so the two cannot double-stop the same containers. A control-side
// failure (e.g. the compose dir is unreadable) degrades to a per-stack
// error for this node only, the same way the remote leg does, so one bad
// node never discards the rest of the fleet's results.
try {
const outcome = await runLocalLabelStop(node.id, trimmed, isDryRun);
return { nodeId: node.id, nodeName: node.name, matched: outcome.matched, stackResults: outcome.stackResults };
} catch (err) {
const errorMsg = getErrorMessage(err, 'Failed to stop local stacks');
const localLabel = db.getLabels(node.id).find(l => l.name === trimmed);
const localStacks = localLabel ? db.getStacksForLabel(localLabel.id, node.id) : [];
return {
nodeId: node.id, nodeName: node.name, matched: !!localLabel,
stackResults: failAllStacks(localStacks, errorMsg),
};
}
}
// Remote node. The control's mirror tells us which stacks *should* match
// so transport errors can be attributed per stack; the authoritative stop
// runs on the remote via its admin-only local-stop receiver, which reuses
// the same name-matched helper under the remote's own bulk lock. This
// replaces the previous fan-out to `POST /api/labels/:id/action`, a
// paid-tier route that 403'd on Community remotes even though fleet-stop
// itself is available on every license.
const label = db.getLabels(node.id).find(l => l.name === trimmed);
if (!label) {
return { nodeId: node.id, nodeName: node.name, matched: false, stackResults: [] };
@@ -1077,56 +1112,21 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: [] };
}
if (node.type === 'local') {
// Share the per-node bulk lock with `POST /api/labels/:id/action` so
// a fleet-stop and a per-label action cannot double-stop the same
// containers concurrently on the same local node. Dry run acquires
// the same lock so the rehearsal exercises the same contention path.
const lockKey = `bulk:${node.id}`;
if (activeBulkActions.has(lockKey)) {
return {
nodeId: node.id, nodeName: node.name, matched: true,
stackResults: stackNames.map(stackName => ({ stackName, success: false, error: 'A bulk action is already running on this node' })),
};
}
activeBulkActions.add(lockKey);
try {
const fsStacks = await FileSystemService.getInstance(node.id).getStacks();
const fsStackSet = new Set(fsStacks);
const validStacks = stackNames.filter(name => fsStackSet.has(name));
const stackResults: { stackName: string; success: boolean; error?: string; dryRun?: boolean }[] = [];
for (const stackName of validStacks) {
if (isDryRun) {
stackResults.push({ stackName, success: true, dryRun: true });
continue;
}
const outcome = await containerActionForStack(node.id, stackName, 'stop');
if (outcome.kind === 'ok') stackResults.push({ stackName, success: true });
else if (outcome.kind === 'no-containers') stackResults.push({ stackName, success: false, error: 'No containers found for this stack' });
else stackResults.push({ stackName, success: false, error: outcome.message });
}
if (!isDryRun && stackResults.some(r => r.success)) invalidateNodeCaches(node.id);
return { nodeId: node.id, nodeName: node.name, matched: true, stackResults };
} finally {
activeBulkActions.delete(lockKey);
}
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
const error = formatNoTargetError(node);
return {
nodeId: node.id, nodeName: node.name, matched: true,
stackResults: stackNames.map(stackName => ({ stackName, success: false, error })),
stackResults: failAllStacks(stackNames, error),
};
}
try {
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/labels/${label.id}/action`, {
const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/fleet-actions/labels/local-stop`, {
method: 'POST',
headers,
body: JSON.stringify({ action: 'stop', dryRun: isDryRun }),
body: JSON.stringify({ labelName: trimmed, dryRun: isDryRun }),
signal: AbortSignal.timeout(60000),
});
if (!response.ok) {
@@ -1134,19 +1134,34 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
const message = err.error || `Remote returned ${response.status}`;
return {
nodeId: node.id, nodeName: node.name, matched: true,
stackResults: stackNames.map(stackName => ({ stackName, success: false, error: message })),
stackResults: failAllStacks(stackNames, message),
};
}
const remote = (await response.json()) as { results?: { stackName: string; success: boolean; error?: string; dryRun?: boolean }[] };
return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: remote.results ?? [] };
// Trust the remote's own matched flag over the control's mirror: a
// mirror-skewed control could believe the label exists while the remote
// has no such label, which the remote reports as matched:false. Guard
// results as an array so a malformed 200 body degrades to empty rather
// than flowing a non-array into the per-stack renderers.
const remote = (await response.json()) as Partial<LabelLocalStopResponse>;
return {
nodeId: node.id, nodeName: node.name,
matched: remote.matched ?? true,
stackResults: Array.isArray(remote.results) ? remote.results : [],
};
} catch (err) {
const errorMsg = getErrorMessage(err, 'Failed to reach remote node');
return {
nodeId: node.id, nodeName: node.name, matched: true,
stackResults: stackNames.map(stackName => ({ stackName, success: false, error: errorMsg })),
stackResults: failAllStacks(stackNames, errorMsg),
};
}
}));
if (isDebugEnabled()) {
const matched = results.filter(r => r.matched).length;
const stopped = results.reduce((n, r) => n + r.stackResults.filter(s => s.success).length, 0);
const failed = results.reduce((n, r) => n + r.stackResults.filter(s => !s.success).length, 0);
console.debug('[Fleet:debug] fleet-stop complete:', { matched, stopped, failed });
}
res.json({ results });
} catch (error) {
console.error('[Fleet] fleet-stop error:', error);
@@ -1199,6 +1214,7 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res
try {
const db = DatabaseService.getInstance();
const nodes = db.getNodes();
if (isDebugEnabled()) console.debug('[Fleet:debug] fleet-prune:', { targets, scope, dryRun: isDryRun, nodes: nodes.length });
const results: NodeResult[] = await Promise.all(nodes.map(async (node): Promise<NodeResult> => {
if (node.type === 'local') {
@@ -1307,6 +1323,11 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res
};
}));
if (isDebugEnabled()) {
const reachable = results.filter(r => r.reachable).length;
const reclaimed = results.reduce((n, r) => n + r.targets.reduce((m, t) => m + t.reclaimedBytes, 0), 0);
console.debug('[Fleet:debug] fleet-prune complete:', { reachable, unreachable: results.length - reachable, reclaimedBytes: reclaimed });
}
res.json({ results });
} catch (error) {
console.error('[Fleet] fleet-prune error:', error);
+34
View File
@@ -4,6 +4,8 @@ import { authMiddleware } from '../middleware/auth';
import { requireAdmin, requireBody } from '../middleware/tierGates';
import { isValidStackName } from '../utils/validation';
import { getErrorMessage } from '../utils/errors';
import { isDebugEnabled } from '../utils/debug';
import { runLocalLabelStop, type LabelLocalStopResponse } from '../helpers/fleetLabelStop';
// Per-node fleet-action endpoints. Mounted under `/api/fleet-actions/`, which
// is NOT in `PROXY_EXEMPT_PREFIXES`, so when `x-node-id` targets a remote node
@@ -66,3 +68,35 @@ fleetActionsRouter.post(
res.json({ results });
},
);
// Per-node label-matched stop. A control instance calls this on each remote
// node during a fleet-wide stop-by-label so the destructive work runs under the
// remote's own admin auth and per-node bulk lock. Admin-only and available on
// every license, matching the rest of the Fleet Actions surface. The paid
// label-driven action lives at `POST /api/labels/:id/action`; this receiver is
// the fleet-plumbing equivalent the control fans out to, so a fleet-stop on a
// Community fleet stops remote stacks instead of 403'ing on the remote leg.
fleetActionsRouter.post(
'/labels/local-stop',
authMiddleware,
async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
if (!requireBody(req, res)) return;
const { labelName, dryRun } = req.body as { labelName?: unknown; dryRun?: unknown };
if (typeof labelName !== 'string' || labelName.trim().length === 0) {
res.status(400).json({ error: 'labelName is required' });
return;
}
const nodeId = req.nodeId ?? 0;
const trimmedLabel = labelName.trim();
try {
const outcome = await runLocalLabelStop(nodeId, trimmedLabel, dryRun === true);
if (isDebugEnabled()) console.debug('[FleetActions:debug] local-stop:', { nodeId, dryRun: dryRun === true, matched: outcome.matched, stacks: outcome.stackResults.length });
const body: LabelLocalStopResponse = { matched: outcome.matched, results: outcome.stackResults };
res.json(body);
} catch (err) {
console.error('[FleetActions] local-stop error:', { nodeId, labelName: trimmedLabel }, err);
res.status(500).json({ error: getErrorMessage(err, 'Failed to run local label stop') });
}
},
);
+1 -1
View File
@@ -72,7 +72,7 @@ A few quirks worth knowing:
- The endpoint always returns 200 with a `results` array. Partial failures live inside that array; the HTTP status is not the place to look.
- Each remote node call carries a 60-second timeout. A slow remote with many stacks can produce a clean per-stack list or a timeout row, depending on whether the remote streamed before the timeout fired.
- Local nodes share a per-node bulk-action lock with the per-label action endpoint, so a fleet stop and a per-label stop initiated against the same node serialize cleanly instead of double-stopping the same containers.
- Every node, local or remote, runs the stop under a per-node bulk-action lock shared with the per-label action endpoint, so a fleet stop and a per-label stop aimed at the same node serialize cleanly instead of double-stopping the same containers. On a remote, the lock is held on that remote while it runs its share of the stop.
## Bulk label assign
@@ -0,0 +1,103 @@
/**
* Coverage for BulkLabelAssignCard.
*
* Loads the node's stacks + labels, gates Apply on a stack-and-label selection,
* confirms before applying, sends the assignment per-node, and surfaces a toast
* on every outcome (no silent failure).
*/
import { it, expect, vi, beforeEach } from 'vitest';
import { render, screen, waitFor, within } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
vi.mock('@/lib/api', () => ({ fetchForNode: vi.fn() }));
const toastError = vi.fn();
const toastSuccess = vi.fn();
vi.mock('@/components/ui/toast-store', () => ({
toast: {
error: (...a: unknown[]) => toastError(...a),
success: (...a: unknown[]) => toastSuccess(...a),
info: vi.fn(),
warning: vi.fn(),
loading: vi.fn(() => 'toast-id'),
dismiss: vi.fn(),
},
}));
import { fetchForNode } from '@/lib/api';
import { BulkLabelAssignCard } from './BulkLabelAssignCard';
import type { FleetNode } from '@/components/FleetView/types';
const mockedFetchForNode = fetchForNode as unknown as ReturnType<typeof vi.fn>;
function jsonResponse(status: number, body: unknown): Response {
return { ok: status >= 200 && status < 300, status, json: async () => body } as unknown as Response;
}
const nodes = [{ id: 1, name: 'central', type: 'local', status: 'online' }] as unknown as FleetNode[];
beforeEach(() => {
vi.clearAllMocks();
mockedFetchForNode.mockImplementation((path: string) => {
if (path === '/fleet/node/1/stacks') return Promise.resolve(jsonResponse(200, ['web']));
if (path === '/labels') return Promise.resolve(jsonResponse(200, [{ id: 10, name: 'prod', color: '#3b82f6' }]));
return Promise.resolve(jsonResponse(200, { results: [] }));
});
});
it('keeps Apply disabled until both a stack and a label are selected', async () => {
const user = userEvent.setup();
render(<BulkLabelAssignCard nodes={nodes} />);
await screen.findByText('web');
expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled();
await user.click(screen.getByRole('checkbox'));
// Stack selected but no label yet.
expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled();
await user.click(screen.getByText('prod'));
expect(screen.getByRole('button', { name: 'Apply' })).toBeEnabled();
});
it('applies the assignment after confirmation and renders per-stack results', async () => {
const user = userEvent.setup();
mockedFetchForNode.mockImplementation((path: string) => {
if (path === '/fleet/node/1/stacks') return Promise.resolve(jsonResponse(200, ['web']));
if (path === '/labels') return Promise.resolve(jsonResponse(200, [{ id: 10, name: 'prod', color: '#3b82f6' }]));
if (path === '/fleet-actions/labels/bulk-assign') return Promise.resolve(jsonResponse(200, { results: [{ stackName: 'web', success: true }] }));
return Promise.resolve(jsonResponse(200, {}));
});
render(<BulkLabelAssignCard nodes={nodes} />);
await screen.findByText('web');
await user.click(screen.getByRole('checkbox'));
await user.click(screen.getByText('prod'));
await user.click(screen.getByRole('button', { name: 'Apply' }));
const dialog = await screen.findByRole('alertdialog');
await user.click(within(dialog).getByRole('button', { name: 'Apply' }));
await waitFor(() => {
const call = mockedFetchForNode.mock.calls.find(c => c[0] === '/fleet-actions/labels/bulk-assign');
expect(call).toBeTruthy();
expect(JSON.parse(call![2].body)).toEqual({ assignments: [{ stackName: 'web', labelIds: [10] }] });
});
await waitFor(() => expect(toastSuccess).toHaveBeenCalled());
});
it('surfaces an error toast when the assignment returns non-ok', async () => {
const user = userEvent.setup();
mockedFetchForNode.mockImplementation((path: string) => {
if (path === '/fleet/node/1/stacks') return Promise.resolve(jsonResponse(200, ['web']));
if (path === '/labels') return Promise.resolve(jsonResponse(200, [{ id: 10, name: 'prod', color: '#3b82f6' }]));
if (path === '/fleet-actions/labels/bulk-assign') return Promise.resolve(jsonResponse(500, { error: 'assign failed' }));
return Promise.resolve(jsonResponse(200, {}));
});
render(<BulkLabelAssignCard nodes={nodes} />);
await screen.findByText('web');
await user.click(screen.getByRole('checkbox'));
await user.click(screen.getByText('prod'));
await user.click(screen.getByRole('button', { name: 'Apply' }));
const dialog = await screen.findByRole('alertdialog');
await user.click(within(dialog).getByRole('button', { name: 'Apply' }));
await waitFor(() => expect(toastError).toHaveBeenCalledWith('assign failed'));
});
@@ -0,0 +1,114 @@
/**
* Coverage for FleetPruneCard.
*
* The key safety property: the destructive "Prune fleet" confirm is blocked
* until the operator has seen a live reclaim estimate. Also locks dry-run
* payload shape, the all-scope confirm copy, and the failure toast.
*/
import { it, expect, vi, beforeEach } from 'vitest';
import { render, screen, waitFor, within } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() }));
const toastError = vi.fn();
const toastSuccess = vi.fn();
vi.mock('@/components/ui/toast-store', () => ({
toast: {
error: (...a: unknown[]) => toastError(...a),
success: (...a: unknown[]) => toastSuccess(...a),
info: vi.fn(),
warning: vi.fn(),
loading: vi.fn(() => 'toast-id'),
dismiss: vi.fn(),
},
}));
import { apiFetch } from '@/lib/api';
import { FleetPruneCard } from './FleetPruneCard';
import type { FleetNode } from '@/components/FleetView/types';
const mockedFetch = apiFetch as unknown as ReturnType<typeof vi.fn>;
function jsonResponse(status: number, body: unknown): Response {
return { ok: status >= 200 && status < 300, status, json: async () => body } as unknown as Response;
}
const nodes = [{ id: 1, name: 'central', status: 'online' }] as unknown as FleetNode[];
beforeEach(() => {
vi.clearAllMocks();
mockedFetch.mockResolvedValue(jsonResponse(404, {}));
});
it('blocks the prune confirm until a reclaim estimate is ready', async () => {
// Estimate endpoint never resolves to ready (404 -> unavailable).
render(<FleetPruneCard nodes={nodes} />);
// images is selected by default, so an estimate is requested but unavailable.
await waitFor(() => expect(screen.getByText('~ estimate unavailable')).toBeInTheDocument());
expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeDisabled();
});
it('enables the prune confirm once the estimate resolves', async () => {
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/prune/estimate') {
return Promise.resolve(jsonResponse(200, { totalBytes: 1024, perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }] }));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<FleetPruneCard nodes={nodes} />);
await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled());
});
it('all-scope confirm spells out the irreversible all-unused prune', async () => {
const user = userEvent.setup();
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/prune/estimate') {
return Promise.resolve(jsonResponse(200, { totalBytes: 2048, perNode: [] }));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<FleetPruneCard nodes={nodes} />);
await user.click(screen.getByRole('button', { name: 'All unused' }));
await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled());
await user.click(screen.getByRole('button', { name: 'Prune fleet' }));
const dialog = await screen.findByRole('alertdialog');
expect(within(dialog).getByText('Prune ALL unused resources across the fleet?')).toBeInTheDocument();
expect(within(dialog).getByText(/This cannot be undone\./)).toBeInTheDocument();
});
it('dry run sends dryRun:true and reports reclaimable bytes', async () => {
const user = userEvent.setup();
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/prune/estimate') {
return Promise.resolve(jsonResponse(200, { totalBytes: 0, perNode: [] }));
}
if (url === '/fleet/labels/fleet-prune') {
return Promise.resolve(jsonResponse(200, {
results: [{ nodeId: 1, nodeName: 'central', reachable: true, targets: [{ target: 'images', success: true, reclaimedBytes: 4096, dryRun: true }] }],
}));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<FleetPruneCard nodes={nodes} />);
await user.click(screen.getByRole('button', { name: 'Dry run' }));
await waitFor(() => {
const call = mockedFetch.mock.calls.find(c => c[0] === '/fleet/labels/fleet-prune');
expect(call).toBeTruthy();
expect(JSON.parse(call![1].body)).toEqual({ targets: ['images'], scope: 'managed', dryRun: true });
});
await waitFor(() => expect(toastSuccess).toHaveBeenCalled());
});
it('surfaces an error toast when the prune returns non-ok', async () => {
const user = userEvent.setup();
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/labels/fleet-prune') {
return Promise.resolve(jsonResponse(500, { error: 'prune blew up' }));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<FleetPruneCard nodes={nodes} />);
await user.click(screen.getByRole('button', { name: 'Dry run' }));
await waitFor(() => expect(toastError).toHaveBeenCalledWith('prune blew up'));
});
@@ -0,0 +1,151 @@
/**
* Coverage for LabelFleetStopCard.
*
* Locks the destructive-action contract: buttons gated on a label name, the
* confirm modal gates the real stop, dry run bypasses the modal, per-node
* results render, and every failure path surfaces a toast (no silent failure).
*/
import { it, expect, vi, beforeEach } from 'vitest';
import { render, screen, waitFor, within } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
vi.mock('@/lib/api', () => ({
apiFetch: vi.fn(),
fetchForNode: vi.fn(),
}));
const toastError = vi.fn();
const toastSuccess = vi.fn();
const toastInfo = vi.fn();
const toastWarning = vi.fn();
vi.mock('@/components/ui/toast-store', () => ({
toast: {
error: (...a: unknown[]) => toastError(...a),
success: (...a: unknown[]) => toastSuccess(...a),
info: (...a: unknown[]) => toastInfo(...a),
warning: (...a: unknown[]) => toastWarning(...a),
loading: vi.fn(() => 'toast-id'),
dismiss: vi.fn(),
},
}));
import { apiFetch, fetchForNode } from '@/lib/api';
import { LabelFleetStopCard } from './LabelFleetStopCard';
import type { FleetNode } from '@/components/FleetView/types';
const mockedFetch = apiFetch as unknown as ReturnType<typeof vi.fn>;
const mockedFetchForNode = fetchForNode as unknown as ReturnType<typeof vi.fn>;
function jsonResponse(status: number, body: unknown): Response {
return { ok: status >= 200 && status < 300, status, json: async () => body } as unknown as Response;
}
const nodes = [
{ id: 1, name: 'central', status: 'online' },
{ id: 2, name: 'edge-1', status: 'online' },
] as unknown as FleetNode[];
beforeEach(() => {
vi.clearAllMocks();
// Suggestion load on mount: no labels.
mockedFetchForNode.mockResolvedValue(jsonResponse(200, []));
// Default: preview unavailable so the debounce effect never throws.
mockedFetch.mockResolvedValue(jsonResponse(404, {}));
});
it('disables both actions until a label name is entered', () => {
render(<LabelFleetStopCard nodes={nodes} />);
expect(screen.getByRole('button', { name: 'Stop fleet' })).toBeDisabled();
expect(screen.getByRole('button', { name: 'Dry run' })).toBeDisabled();
});
it('enables the actions once a label is typed', async () => {
const user = userEvent.setup();
render(<LabelFleetStopCard nodes={nodes} />);
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
expect(screen.getByRole('button', { name: 'Stop fleet' })).toBeEnabled();
expect(screen.getByRole('button', { name: 'Dry run' })).toBeEnabled();
});
it('dry run calls fleet-stop with dryRun:true and never opens the confirm modal', async () => {
const user = userEvent.setup();
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/labels/fleet-stop') {
return Promise.resolve(jsonResponse(200, {
results: [{ nodeId: 1, nodeName: 'central', matched: true, stackResults: [{ stackName: 'web', success: true, dryRun: true }] }],
}));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<LabelFleetStopCard nodes={nodes} />);
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
await user.click(screen.getByRole('button', { name: 'Dry run' }));
await waitFor(() => {
const stopCall = mockedFetch.mock.calls.find(c => c[0] === '/fleet/labels/fleet-stop');
expect(stopCall).toBeTruthy();
expect(JSON.parse(stopCall![1].body)).toEqual({ labelName: 'prod', dryRun: true });
});
expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument();
await waitFor(() => expect(toastSuccess).toHaveBeenCalled());
});
it('stop fleet opens the confirm modal, and confirming runs a real stop that renders per-node results', async () => {
const user = userEvent.setup();
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/labels/fleet-stop') {
return Promise.resolve(jsonResponse(200, {
results: [
{ nodeId: 1, nodeName: 'central', matched: true, stackResults: [{ stackName: 'web', success: true }] },
{ nodeId: 2, nodeName: 'edge-1', matched: false, stackResults: [] },
],
}));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<LabelFleetStopCard nodes={nodes} />);
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
await user.click(screen.getByRole('button', { name: 'Stop fleet' }));
const dialog = await screen.findByRole('alertdialog');
expect(within(dialog).getByText('Stop all stacks labeled "prod"?')).toBeInTheDocument();
// The real stop must not have fired yet.
expect(mockedFetch.mock.calls.find(c => c[0] === '/fleet/labels/fleet-stop')).toBeFalsy();
await user.click(within(dialog).getByRole('button', { name: 'Stop fleet' }));
await waitFor(() => {
const stopCall = mockedFetch.mock.calls.find(c => c[0] === '/fleet/labels/fleet-stop');
expect(JSON.parse(stopCall![1].body)).toEqual({ labelName: 'prod', dryRun: false });
});
expect(await screen.findByText('web')).toBeInTheDocument();
await waitFor(() => expect(toastSuccess).toHaveBeenCalled());
});
it('surfaces an error toast when fleet-stop returns a non-ok response', async () => {
const user = userEvent.setup();
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/labels/fleet-stop') {
return Promise.resolve(jsonResponse(500, { error: 'boom' }));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<LabelFleetStopCard nodes={nodes} />);
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
await user.click(screen.getByRole('button', { name: 'Dry run' }));
await waitFor(() => expect(toastError).toHaveBeenCalledWith('boom'));
});
it('populates the blast readout from the debounced match-preview', async () => {
const user = userEvent.setup();
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/labels/match-preview') {
return Promise.resolve(jsonResponse(200, { matchedNodes: 2, matchedStacks: 3, perNode: [] }));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<LabelFleetStopCard nodes={nodes} />);
await user.type(screen.getByPlaceholderText('e.g. production'), 'prod');
await waitFor(() => expect(screen.getByText('3 stacks · 2 nodes')).toBeInTheDocument(), { timeout: 2000 });
});