diff --git a/backend/src/__tests__/fleet-action-card-endpoints.test.ts b/backend/src/__tests__/fleet-action-card-endpoints.test.ts index 0017808a..d3d798d7 100644 --- a/backend/src/__tests__/fleet-action-card-endpoints.test.ts +++ b/backend/src/__tests__/fleet-action-card-endpoints.test.ts @@ -244,6 +244,103 @@ describe('POST /api/fleet/prune/estimate', () => { }); }); +describe('POST /api/fleet/labels/fleet-stop remote leg', () => { + // Guards the C-1 fix: the remote fan-out must target the admin-only + // /api/fleet-actions/labels/local-stop receiver (reachable on every license), + // never the paid /api/labels/:id/action it used to call, which 403'd on + // Community remotes. + it('fans out to the admin-only local-stop receiver, never the paid per-label action route', async () => { + const remoteId = db.addNode({ + name: 'remote-stop', + type: 'remote', + api_url: 'http://remote-stop.example:1852', + api_token: 'tok', + compose_dir: '/app/compose', + is_default: false, + }); + try { + const label = db.createLabel(remoteId, `remote-c1-${++labelCounter}`, 'teal'); + db.setStackLabels('alpha', remoteId, [label.id]); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ matched: true, results: [{ stackName: 'alpha', success: true }] }), + } as unknown as Response); + + const res = await request(app) + .post('/api/fleet/labels/fleet-stop') + .set('Authorization', authHeader) + .send({ labelName: label.name }); + + expect(res.status).toBe(200); + const urls = fetchSpy.mock.calls.map(c => String(c[0])); + expect(urls.some(u => u.endsWith('/api/fleet-actions/labels/local-stop'))).toBe(true); + expect(urls.some(u => u.includes('/api/labels/'))).toBe(false); + + const call = fetchSpy.mock.calls.find(c => String(c[0]).endsWith('/api/fleet-actions/labels/local-stop')); + expect(JSON.parse((call![1] as RequestInit).body as string)).toEqual({ labelName: label.name, dryRun: false }); + + const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId); + expect(remoteRow.stackResults).toEqual([{ stackName: 'alpha', success: true }]); + } finally { + db.deleteNode(remoteId); + } + }); + + it('honors the remote matched:false flag over the control mirror (mirror skew)', async () => { + const remoteId = db.addNode({ + name: 'remote-skew', type: 'remote', api_url: 'http://remote-skew.example:1852', + api_token: 'tok', compose_dir: '/app/compose', is_default: false, + }); + try { + // The control mirror believes the remote carries this label + stack... + const label = db.createLabel(remoteId, `remote-skew-${++labelCounter}`, 'teal'); + db.setStackLabels('alpha', remoteId, [label.id]); + // ...but the remote authoritatively reports it has no such label. + vi.spyOn(globalThis, 'fetch').mockResolvedValue({ + ok: true, status: 200, json: async () => ({ matched: false, results: [] }), + } as unknown as Response); + + const res = await request(app) + .post('/api/fleet/labels/fleet-stop') + .set('Authorization', authHeader) + .send({ labelName: label.name }); + + expect(res.status).toBe(200); + const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId); + expect(remoteRow.matched).toBe(false); + expect(remoteRow.stackResults).toEqual([]); + } finally { + db.deleteNode(remoteId); + } + }); + + it('degrades a malformed 200 body (non-array results) to empty instead of forwarding it', async () => { + const remoteId = db.addNode({ + name: 'remote-malformed', type: 'remote', api_url: 'http://remote-malformed.example:1852', + api_token: 'tok', compose_dir: '/app/compose', is_default: false, + }); + try { + const label = db.createLabel(remoteId, `remote-malformed-${++labelCounter}`, 'teal'); + db.setStackLabels('alpha', remoteId, [label.id]); + vi.spyOn(globalThis, 'fetch').mockResolvedValue({ + ok: true, status: 200, json: async () => ({ matched: true, results: 'not-an-array' }), + } as unknown as Response); + + const res = await request(app) + .post('/api/fleet/labels/fleet-stop') + .set('Authorization', authHeader) + .send({ labelName: label.name }); + + expect(res.status).toBe(200); + const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId); + expect(remoteRow.stackResults).toEqual([]); + } finally { + db.deleteNode(remoteId); + } + }); +}); + describe('POST /api/fleet/labels/fleet-stop with dryRun: true', () => { it('marks each stack dryRun: true and does not invoke containerActionForStack', async () => { const label = await createAssignedLabel('dry-stop', ['alpha', 'beta']); diff --git a/backend/src/__tests__/fleet-actions.test.ts b/backend/src/__tests__/fleet-actions.test.ts index 97423eab..cb311e4d 100644 --- a/backend/src/__tests__/fleet-actions.test.ts +++ b/backend/src/__tests__/fleet-actions.test.ts @@ -3,6 +3,8 @@ * validation, and orchestration shape across the two routes. */ import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; +import fs from 'fs'; +import path from 'path'; import request from 'supertest'; import jwt from 'jsonwebtoken'; import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; @@ -138,3 +140,147 @@ describe('Fleet Actions orchestration shape', () => { expect(res.body.results[0]).toMatchObject({ success: false, error: 'Invalid stack name' }); }); }); + +// The per-node local-stop receiver is what a control instance calls on each +// remote during a fleet-wide stop. It must be reachable on every license (only +// admin-gated): the original fleet-stop fan-out hit the paid /api/labels/:id/action +// and 403'd on Community remotes. These tests lock that behavior in. +describe('local-stop receiver auth + tier', () => { + afterEach(() => vi.restoreAllMocks()); + + it('POST /api/fleet-actions/labels/local-stop returns 401 without auth', async () => { + const res = await request(app).post('/api/fleet-actions/labels/local-stop').send({ labelName: 'prod' }); + expect(res.status).toBe(401); + }); + + it('is reachable on community tier for admins and never returns PAID_REQUIRED', async () => { + mockTier('community'); + const res = await request(app) + .post('/api/fleet-actions/labels/local-stop') + .set('Authorization', authHeader) + .send({ labelName: 'this-label-does-not-exist' }); + expect(res.status).toBe(200); + expect(res.body.code).not.toBe('PAID_REQUIRED'); + expect(res.body).toEqual({ matched: false, results: [] }); + }); + + it('rejects missing labelName', async () => { + const res = await request(app) + .post('/api/fleet-actions/labels/local-stop') + .set('Authorization', authHeader) + .send({}); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/labelName/); + }); + + it('rejects whitespace-only labelName', async () => { + const res = await request(app) + .post('/api/fleet-actions/labels/local-stop') + .set('Authorization', authHeader) + .send({ labelName: ' ' }); + expect(res.status).toBe(400); + }); +}); + +describe('local-stop behavior', () => { + let db: import('../services/DatabaseService').DatabaseService; + let nodeId: number; + + beforeAll(async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + db = DatabaseService.getInstance(); + nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + }); + + afterEach(() => vi.restoreAllMocks()); + + it('matched:true with empty results when the label exists but has no stacks', async () => { + db.createLabel(nodeId, 'no-stacks-label', '#ffffff'); + const res = await request(app) + .post('/api/fleet-actions/labels/local-stop') + .set('Authorization', authHeader) + .send({ labelName: 'no-stacks-label' }); + expect(res.status).toBe(200); + expect(res.body).toEqual({ matched: true, results: [] }); + }); + + it('reports per-stack lock contention when a bulk action is already running on the node', async () => { + const label = db.createLabel(nodeId, 'busy-label', '#ffffff'); + db.setStackLabels('busy-stack', nodeId, [label.id]); + const { activeBulkActions } = await import('../routes/labels'); + activeBulkActions.add(`bulk:${nodeId}`); + try { + const res = await request(app) + .post('/api/fleet-actions/labels/local-stop') + .set('Authorization', authHeader) + .send({ labelName: 'busy-label' }); + expect(res.status).toBe(200); + expect(res.body.matched).toBe(true); + expect(res.body.results).toEqual([ + { stackName: 'busy-stack', success: false, error: 'A bulk action is already running on this node' }, + ]); + } finally { + activeBulkActions.delete(`bulk:${nodeId}`); + } + }); + + it('dry run returns dryRun:true per on-disk stack without touching Docker', async () => { + const composeDir = process.env.COMPOSE_DIR as string; + fs.mkdirSync(path.join(composeDir, 'dry-stack'), { recursive: true }); + fs.writeFileSync(path.join(composeDir, 'dry-stack', 'docker-compose.yml'), 'services: {}\n'); + const label = db.createLabel(nodeId, 'dry-label', '#ffffff'); + db.setStackLabels('dry-stack', nodeId, [label.id]); + const res = await request(app) + .post('/api/fleet-actions/labels/local-stop') + .set('Authorization', authHeader) + .send({ labelName: 'dry-label', dryRun: true }); + expect(res.status).toBe(200); + expect(res.body.matched).toBe(true); + expect(res.body.results).toEqual([{ stackName: 'dry-stack', success: true, dryRun: true }]); + }); + + it('filters out assigned stacks that are not present on disk', async () => { + const label = db.createLabel(nodeId, 'ghost-label', '#ffffff'); + db.setStackLabels('ghost-stack', nodeId, [label.id]); + const res = await request(app) + .post('/api/fleet-actions/labels/local-stop') + .set('Authorization', authHeader) + .send({ labelName: 'ghost-label' }); + expect(res.status).toBe(200); + expect(res.body).toEqual({ matched: true, results: [] }); + }); +}); + +describe('fleet-stop degrades the local leg per-node instead of failing the whole fan-out', () => { + let db: import('../services/DatabaseService').DatabaseService; + let nodeId: number; + + beforeAll(async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + db = DatabaseService.getInstance(); + nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + }); + + afterEach(() => vi.restoreAllMocks()); + + it('returns 200 with per-stack errors when the control filesystem read throws', async () => { + const label = db.createLabel(nodeId, 'degrade-label', '#ffffff'); + db.setStackLabels('degrade-stack', nodeId, [label.id]); + const { FileSystemService } = await import('../services/FileSystemService'); + vi.spyOn(FileSystemService.prototype, 'getStacks').mockRejectedValue(new Error('compose dir unreadable')); + + const res = await request(app) + .post('/api/fleet/labels/fleet-stop') + .set('Authorization', authHeader) + .send({ labelName: 'degrade-label' }); + + expect(res.status).toBe(200); + const localRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === nodeId); + expect(localRow.matched).toBe(true); + expect(localRow.stackResults).toEqual([ + { stackName: 'degrade-stack', success: false, error: 'compose dir unreadable' }, + ]); + }); +}); diff --git a/backend/src/helpers/fleetLabelStop.ts b/backend/src/helpers/fleetLabelStop.ts new file mode 100644 index 00000000..3933703e --- /dev/null +++ b/backend/src/helpers/fleetLabelStop.ts @@ -0,0 +1,87 @@ +import { DatabaseService } from '../services/DatabaseService'; +import { FileSystemService } from '../services/FileSystemService'; +import { containerActionForStack } from '../routes/stacks'; +import { activeBulkActions } from '../routes/labels'; +import { invalidateNodeCaches } from './cacheInvalidation'; + +export interface StackStopResult { + stackName: string; + success: boolean; + error?: string; + dryRun?: boolean; +} + +export interface LabelStopOutcome { + matched: boolean; + stackResults: StackStopResult[]; +} + +/** + * Wire shape of `POST /api/fleet-actions/labels/local-stop`. The in-process + * helper returns `stackResults`; the HTTP response names the same array + * `results` to match the fleet-stop fan-out's existing remote contract. Keep + * the rename in this one type so the producer and the control-side consumer + * cannot drift. + */ +export interface LabelLocalStopResponse { + matched: boolean; + results: StackStopResult[]; +} + +/** + * Run a label-name-matched container stop against one node's own local Docker. + * + * Used by the gateway-orchestrated fleet-stop for the control node's own stacks + * and by the per-node `POST /api/fleet-actions/labels/local-stop` receiver that + * a control instance calls on each remote during a fleet-wide stop. Matching by + * name (not by a shared label id) keeps the work self-contained on the executing + * node, so the control never has to assert that its mirrored label ids line up + * with the remote's local ids. + * + * Shares the per-node `bulk:` lock with `POST /api/labels/:id/action` so + * a fleet-stop and a per-label action cannot double-stop the same containers. + */ +export async function runLocalLabelStop( + nodeId: number, + labelName: string, + dryRun: boolean, +): Promise { + const db = DatabaseService.getInstance(); + const label = db.getLabels(nodeId).find(l => l.name === labelName); + if (!label) return { matched: false, stackResults: [] }; + const stackNames = db.getStacksForLabel(label.id, nodeId); + if (stackNames.length === 0) return { matched: true, stackResults: [] }; + + const lockKey = `bulk:${nodeId}`; + if (activeBulkActions.has(lockKey)) { + return { + matched: true, + stackResults: stackNames.map(stackName => ({ + stackName, + success: false, + error: 'A bulk action is already running on this node', + })), + }; + } + activeBulkActions.add(lockKey); + try { + const fsStacks = await FileSystemService.getInstance(nodeId).getStacks(); + const fsStackSet = new Set(fsStacks); + const validStacks = stackNames.filter(name => fsStackSet.has(name)); + const stackResults: StackStopResult[] = []; + for (const stackName of validStacks) { + if (dryRun) { + stackResults.push({ stackName, success: true, dryRun: true }); + continue; + } + const outcome = await containerActionForStack(nodeId, stackName, 'stop'); + if (outcome.kind === 'ok') stackResults.push({ stackName, success: true }); + else if (outcome.kind === 'no-containers') stackResults.push({ stackName, success: false, error: 'No containers found for this stack' }); + else stackResults.push({ stackName, success: false, error: outcome.message }); + } + if (!dryRun && stackResults.some(r => r.success)) invalidateNodeCaches(nodeId); + return { matched: true, stackResults }; + } finally { + activeBulkActions.delete(lockKey); + } +} diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts index f7b17fcf..c267de5f 100644 --- a/backend/src/routes/fleet.ts +++ b/backend/src/routes/fleet.ts @@ -34,8 +34,8 @@ import { formatNoTargetError } from '../utils/remoteTarget'; import { CloudBackupService } from '../services/CloudBackupService'; import { NotificationService } from '../services/NotificationService'; import { invalidateNodeCaches, invalidateRemoteMetaCache } from '../helpers/cacheInvalidation'; -import { containerActionForStack } from './stacks'; import { activeBulkActions } from './labels'; +import { runLocalLabelStop, type LabelLocalStopResponse, type StackStopResult } from '../helpers/fleetLabelStop'; import { buildLocalConfigurationStatus, type ConfigurationStatus } from './dashboard'; import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from '../services/license-headers'; import { LicenseService } from '../services/LicenseService'; @@ -1047,6 +1047,12 @@ fleetRouter.delete('/update-status', authMiddleware, async (req: Request, res: R // in `routes/fleetActions.ts`. The endpoint below is gateway-orchestrated and // lives here so it sits behind the `/api/fleet/` proxy-exempt prefix. +// Attribute one error to every stack a node was supposed to act on. Used for +// the fleet-stop failure paths (no proxy target, non-ok remote, transport +// error, local exception) so each stack carries the same node-level cause. +const failAllStacks = (stacks: string[], error: string): StackStopResult[] => + stacks.map(stackName => ({ stackName, success: false, error })); + // Fleet-wide stop by label name. Matches each node's labels by name and runs // container stops on each matching stack. // Tier: requireAdmin (admin-only fleet plumbing; available on every license). @@ -1067,7 +1073,36 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res: try { const db = DatabaseService.getInstance(); const nodes = db.getNodes(); + if (isDebugEnabled()) console.debug('[Fleet:debug] fleet-stop:', { labelName: trimmed, dryRun: isDryRun, nodes: nodes.length }); const results = await Promise.all(nodes.map(async (node) => { + if (node.type === 'local') { + // Match + stop runs in-process against the control's own Docker. The + // helper shares the per-node `bulk:` lock with the per-label action + // route so the two cannot double-stop the same containers. A control-side + // failure (e.g. the compose dir is unreadable) degrades to a per-stack + // error for this node only, the same way the remote leg does, so one bad + // node never discards the rest of the fleet's results. + try { + const outcome = await runLocalLabelStop(node.id, trimmed, isDryRun); + return { nodeId: node.id, nodeName: node.name, matched: outcome.matched, stackResults: outcome.stackResults }; + } catch (err) { + const errorMsg = getErrorMessage(err, 'Failed to stop local stacks'); + const localLabel = db.getLabels(node.id).find(l => l.name === trimmed); + const localStacks = localLabel ? db.getStacksForLabel(localLabel.id, node.id) : []; + return { + nodeId: node.id, nodeName: node.name, matched: !!localLabel, + stackResults: failAllStacks(localStacks, errorMsg), + }; + } + } + + // Remote node. The control's mirror tells us which stacks *should* match + // so transport errors can be attributed per stack; the authoritative stop + // runs on the remote via its admin-only local-stop receiver, which reuses + // the same name-matched helper under the remote's own bulk lock. This + // replaces the previous fan-out to `POST /api/labels/:id/action`, a + // paid-tier route that 403'd on Community remotes even though fleet-stop + // itself is available on every license. const label = db.getLabels(node.id).find(l => l.name === trimmed); if (!label) { return { nodeId: node.id, nodeName: node.name, matched: false, stackResults: [] }; @@ -1077,56 +1112,21 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res: return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: [] }; } - if (node.type === 'local') { - // Share the per-node bulk lock with `POST /api/labels/:id/action` so - // a fleet-stop and a per-label action cannot double-stop the same - // containers concurrently on the same local node. Dry run acquires - // the same lock so the rehearsal exercises the same contention path. - const lockKey = `bulk:${node.id}`; - if (activeBulkActions.has(lockKey)) { - return { - nodeId: node.id, nodeName: node.name, matched: true, - stackResults: stackNames.map(stackName => ({ stackName, success: false, error: 'A bulk action is already running on this node' })), - }; - } - activeBulkActions.add(lockKey); - try { - const fsStacks = await FileSystemService.getInstance(node.id).getStacks(); - const fsStackSet = new Set(fsStacks); - const validStacks = stackNames.filter(name => fsStackSet.has(name)); - const stackResults: { stackName: string; success: boolean; error?: string; dryRun?: boolean }[] = []; - for (const stackName of validStacks) { - if (isDryRun) { - stackResults.push({ stackName, success: true, dryRun: true }); - continue; - } - const outcome = await containerActionForStack(node.id, stackName, 'stop'); - if (outcome.kind === 'ok') stackResults.push({ stackName, success: true }); - else if (outcome.kind === 'no-containers') stackResults.push({ stackName, success: false, error: 'No containers found for this stack' }); - else stackResults.push({ stackName, success: false, error: outcome.message }); - } - if (!isDryRun && stackResults.some(r => r.success)) invalidateNodeCaches(node.id); - return { nodeId: node.id, nodeName: node.name, matched: true, stackResults }; - } finally { - activeBulkActions.delete(lockKey); - } - } - const target = NodeRegistry.getInstance().getProxyTarget(node.id); if (!target) { const error = formatNoTargetError(node); return { nodeId: node.id, nodeName: node.name, matched: true, - stackResults: stackNames.map(stackName => ({ stackName, success: false, error })), + stackResults: failAllStacks(stackNames, error), }; } try { const headers: Record = { 'Content-Type': 'application/json' }; if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`; - const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/labels/${label.id}/action`, { + const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/fleet-actions/labels/local-stop`, { method: 'POST', headers, - body: JSON.stringify({ action: 'stop', dryRun: isDryRun }), + body: JSON.stringify({ labelName: trimmed, dryRun: isDryRun }), signal: AbortSignal.timeout(60000), }); if (!response.ok) { @@ -1134,19 +1134,34 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res: const message = err.error || `Remote returned ${response.status}`; return { nodeId: node.id, nodeName: node.name, matched: true, - stackResults: stackNames.map(stackName => ({ stackName, success: false, error: message })), + stackResults: failAllStacks(stackNames, message), }; } - const remote = (await response.json()) as { results?: { stackName: string; success: boolean; error?: string; dryRun?: boolean }[] }; - return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: remote.results ?? [] }; + // Trust the remote's own matched flag over the control's mirror: a + // mirror-skewed control could believe the label exists while the remote + // has no such label, which the remote reports as matched:false. Guard + // results as an array so a malformed 200 body degrades to empty rather + // than flowing a non-array into the per-stack renderers. + const remote = (await response.json()) as Partial; + return { + nodeId: node.id, nodeName: node.name, + matched: remote.matched ?? true, + stackResults: Array.isArray(remote.results) ? remote.results : [], + }; } catch (err) { const errorMsg = getErrorMessage(err, 'Failed to reach remote node'); return { nodeId: node.id, nodeName: node.name, matched: true, - stackResults: stackNames.map(stackName => ({ stackName, success: false, error: errorMsg })), + stackResults: failAllStacks(stackNames, errorMsg), }; } })); + if (isDebugEnabled()) { + const matched = results.filter(r => r.matched).length; + const stopped = results.reduce((n, r) => n + r.stackResults.filter(s => s.success).length, 0); + const failed = results.reduce((n, r) => n + r.stackResults.filter(s => !s.success).length, 0); + console.debug('[Fleet:debug] fleet-stop complete:', { matched, stopped, failed }); + } res.json({ results }); } catch (error) { console.error('[Fleet] fleet-stop error:', error); @@ -1199,6 +1214,7 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res try { const db = DatabaseService.getInstance(); const nodes = db.getNodes(); + if (isDebugEnabled()) console.debug('[Fleet:debug] fleet-prune:', { targets, scope, dryRun: isDryRun, nodes: nodes.length }); const results: NodeResult[] = await Promise.all(nodes.map(async (node): Promise => { if (node.type === 'local') { @@ -1307,6 +1323,11 @@ fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res }; })); + if (isDebugEnabled()) { + const reachable = results.filter(r => r.reachable).length; + const reclaimed = results.reduce((n, r) => n + r.targets.reduce((m, t) => m + t.reclaimedBytes, 0), 0); + console.debug('[Fleet:debug] fleet-prune complete:', { reachable, unreachable: results.length - reachable, reclaimedBytes: reclaimed }); + } res.json({ results }); } catch (error) { console.error('[Fleet] fleet-prune error:', error); diff --git a/backend/src/routes/fleetActions.ts b/backend/src/routes/fleetActions.ts index ca559b2b..c4b0353c 100644 --- a/backend/src/routes/fleetActions.ts +++ b/backend/src/routes/fleetActions.ts @@ -4,6 +4,8 @@ import { authMiddleware } from '../middleware/auth'; import { requireAdmin, requireBody } from '../middleware/tierGates'; import { isValidStackName } from '../utils/validation'; import { getErrorMessage } from '../utils/errors'; +import { isDebugEnabled } from '../utils/debug'; +import { runLocalLabelStop, type LabelLocalStopResponse } from '../helpers/fleetLabelStop'; // Per-node fleet-action endpoints. Mounted under `/api/fleet-actions/`, which // is NOT in `PROXY_EXEMPT_PREFIXES`, so when `x-node-id` targets a remote node @@ -66,3 +68,35 @@ fleetActionsRouter.post( res.json({ results }); }, ); + +// Per-node label-matched stop. A control instance calls this on each remote +// node during a fleet-wide stop-by-label so the destructive work runs under the +// remote's own admin auth and per-node bulk lock. Admin-only and available on +// every license, matching the rest of the Fleet Actions surface. The paid +// label-driven action lives at `POST /api/labels/:id/action`; this receiver is +// the fleet-plumbing equivalent the control fans out to, so a fleet-stop on a +// Community fleet stops remote stacks instead of 403'ing on the remote leg. +fleetActionsRouter.post( + '/labels/local-stop', + authMiddleware, + async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + if (!requireBody(req, res)) return; + const { labelName, dryRun } = req.body as { labelName?: unknown; dryRun?: unknown }; + if (typeof labelName !== 'string' || labelName.trim().length === 0) { + res.status(400).json({ error: 'labelName is required' }); + return; + } + const nodeId = req.nodeId ?? 0; + const trimmedLabel = labelName.trim(); + try { + const outcome = await runLocalLabelStop(nodeId, trimmedLabel, dryRun === true); + if (isDebugEnabled()) console.debug('[FleetActions:debug] local-stop:', { nodeId, dryRun: dryRun === true, matched: outcome.matched, stacks: outcome.stackResults.length }); + const body: LabelLocalStopResponse = { matched: outcome.matched, results: outcome.stackResults }; + res.json(body); + } catch (err) { + console.error('[FleetActions] local-stop error:', { nodeId, labelName: trimmedLabel }, err); + res.status(500).json({ error: getErrorMessage(err, 'Failed to run local label stop') }); + } + }, +); diff --git a/docs/features/fleet-actions.mdx b/docs/features/fleet-actions.mdx index e8fde1fc..d3ee6414 100644 --- a/docs/features/fleet-actions.mdx +++ b/docs/features/fleet-actions.mdx @@ -72,7 +72,7 @@ A few quirks worth knowing: - The endpoint always returns 200 with a `results` array. Partial failures live inside that array; the HTTP status is not the place to look. - Each remote node call carries a 60-second timeout. A slow remote with many stacks can produce a clean per-stack list or a timeout row, depending on whether the remote streamed before the timeout fired. -- Local nodes share a per-node bulk-action lock with the per-label action endpoint, so a fleet stop and a per-label stop initiated against the same node serialize cleanly instead of double-stopping the same containers. +- Every node, local or remote, runs the stop under a per-node bulk-action lock shared with the per-label action endpoint, so a fleet stop and a per-label stop aimed at the same node serialize cleanly instead of double-stopping the same containers. On a remote, the lock is held on that remote while it runs its share of the stop. ## Bulk label assign diff --git a/frontend/src/components/fleet/FleetActions/cards/BulkLabelAssignCard.test.tsx b/frontend/src/components/fleet/FleetActions/cards/BulkLabelAssignCard.test.tsx new file mode 100644 index 00000000..d7b8af81 --- /dev/null +++ b/frontend/src/components/fleet/FleetActions/cards/BulkLabelAssignCard.test.tsx @@ -0,0 +1,103 @@ +/** + * Coverage for BulkLabelAssignCard. + * + * Loads the node's stacks + labels, gates Apply on a stack-and-label selection, + * confirms before applying, sends the assignment per-node, and surfaces a toast + * on every outcome (no silent failure). + */ +import { it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor, within } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +vi.mock('@/lib/api', () => ({ fetchForNode: vi.fn() })); + +const toastError = vi.fn(); +const toastSuccess = vi.fn(); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { + error: (...a: unknown[]) => toastError(...a), + success: (...a: unknown[]) => toastSuccess(...a), + info: vi.fn(), + warning: vi.fn(), + loading: vi.fn(() => 'toast-id'), + dismiss: vi.fn(), + }, +})); + +import { fetchForNode } from '@/lib/api'; +import { BulkLabelAssignCard } from './BulkLabelAssignCard'; +import type { FleetNode } from '@/components/FleetView/types'; + +const mockedFetchForNode = fetchForNode as unknown as ReturnType; + +function jsonResponse(status: number, body: unknown): Response { + return { ok: status >= 200 && status < 300, status, json: async () => body } as unknown as Response; +} + +const nodes = [{ id: 1, name: 'central', type: 'local', status: 'online' }] as unknown as FleetNode[]; + +beforeEach(() => { + vi.clearAllMocks(); + mockedFetchForNode.mockImplementation((path: string) => { + if (path === '/fleet/node/1/stacks') return Promise.resolve(jsonResponse(200, ['web'])); + if (path === '/labels') return Promise.resolve(jsonResponse(200, [{ id: 10, name: 'prod', color: '#3b82f6' }])); + return Promise.resolve(jsonResponse(200, { results: [] })); + }); +}); + +it('keeps Apply disabled until both a stack and a label are selected', async () => { + const user = userEvent.setup(); + render(); + await screen.findByText('web'); + expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled(); + + await user.click(screen.getByRole('checkbox')); + // Stack selected but no label yet. + expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled(); + + await user.click(screen.getByText('prod')); + expect(screen.getByRole('button', { name: 'Apply' })).toBeEnabled(); +}); + +it('applies the assignment after confirmation and renders per-stack results', async () => { + const user = userEvent.setup(); + mockedFetchForNode.mockImplementation((path: string) => { + if (path === '/fleet/node/1/stacks') return Promise.resolve(jsonResponse(200, ['web'])); + if (path === '/labels') return Promise.resolve(jsonResponse(200, [{ id: 10, name: 'prod', color: '#3b82f6' }])); + if (path === '/fleet-actions/labels/bulk-assign') return Promise.resolve(jsonResponse(200, { results: [{ stackName: 'web', success: true }] })); + return Promise.resolve(jsonResponse(200, {})); + }); + render(); + await screen.findByText('web'); + await user.click(screen.getByRole('checkbox')); + await user.click(screen.getByText('prod')); + await user.click(screen.getByRole('button', { name: 'Apply' })); + + const dialog = await screen.findByRole('alertdialog'); + await user.click(within(dialog).getByRole('button', { name: 'Apply' })); + + await waitFor(() => { + const call = mockedFetchForNode.mock.calls.find(c => c[0] === '/fleet-actions/labels/bulk-assign'); + expect(call).toBeTruthy(); + expect(JSON.parse(call![2].body)).toEqual({ assignments: [{ stackName: 'web', labelIds: [10] }] }); + }); + await waitFor(() => expect(toastSuccess).toHaveBeenCalled()); +}); + +it('surfaces an error toast when the assignment returns non-ok', async () => { + const user = userEvent.setup(); + mockedFetchForNode.mockImplementation((path: string) => { + if (path === '/fleet/node/1/stacks') return Promise.resolve(jsonResponse(200, ['web'])); + if (path === '/labels') return Promise.resolve(jsonResponse(200, [{ id: 10, name: 'prod', color: '#3b82f6' }])); + if (path === '/fleet-actions/labels/bulk-assign') return Promise.resolve(jsonResponse(500, { error: 'assign failed' })); + return Promise.resolve(jsonResponse(200, {})); + }); + render(); + await screen.findByText('web'); + await user.click(screen.getByRole('checkbox')); + await user.click(screen.getByText('prod')); + await user.click(screen.getByRole('button', { name: 'Apply' })); + const dialog = await screen.findByRole('alertdialog'); + await user.click(within(dialog).getByRole('button', { name: 'Apply' })); + await waitFor(() => expect(toastError).toHaveBeenCalledWith('assign failed')); +}); diff --git a/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx new file mode 100644 index 00000000..df538584 --- /dev/null +++ b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx @@ -0,0 +1,114 @@ +/** + * Coverage for FleetPruneCard. + * + * The key safety property: the destructive "Prune fleet" confirm is blocked + * until the operator has seen a live reclaim estimate. Also locks dry-run + * payload shape, the all-scope confirm copy, and the failure toast. + */ +import { it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor, within } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); + +const toastError = vi.fn(); +const toastSuccess = vi.fn(); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { + error: (...a: unknown[]) => toastError(...a), + success: (...a: unknown[]) => toastSuccess(...a), + info: vi.fn(), + warning: vi.fn(), + loading: vi.fn(() => 'toast-id'), + dismiss: vi.fn(), + }, +})); + +import { apiFetch } from '@/lib/api'; +import { FleetPruneCard } from './FleetPruneCard'; +import type { FleetNode } from '@/components/FleetView/types'; + +const mockedFetch = apiFetch as unknown as ReturnType; + +function jsonResponse(status: number, body: unknown): Response { + return { ok: status >= 200 && status < 300, status, json: async () => body } as unknown as Response; +} + +const nodes = [{ id: 1, name: 'central', status: 'online' }] as unknown as FleetNode[]; + +beforeEach(() => { + vi.clearAllMocks(); + mockedFetch.mockResolvedValue(jsonResponse(404, {})); +}); + +it('blocks the prune confirm until a reclaim estimate is ready', async () => { + // Estimate endpoint never resolves to ready (404 -> unavailable). + render(); + // images is selected by default, so an estimate is requested but unavailable. + await waitFor(() => expect(screen.getByText('~ estimate unavailable')).toBeInTheDocument()); + expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeDisabled(); +}); + +it('enables the prune confirm once the estimate resolves', async () => { + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + return Promise.resolve(jsonResponse(200, { totalBytes: 1024, perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }] })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); +}); + +it('all-scope confirm spells out the irreversible all-unused prune', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + return Promise.resolve(jsonResponse(200, { totalBytes: 2048, perNode: [] })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + render(); + await user.click(screen.getByRole('button', { name: 'All unused' })); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); + await user.click(screen.getByRole('button', { name: 'Prune fleet' })); + const dialog = await screen.findByRole('alertdialog'); + expect(within(dialog).getByText('Prune ALL unused resources across the fleet?')).toBeInTheDocument(); + expect(within(dialog).getByText(/This cannot be undone\./)).toBeInTheDocument(); +}); + +it('dry run sends dryRun:true and reports reclaimable bytes', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + return Promise.resolve(jsonResponse(200, { totalBytes: 0, perNode: [] })); + } + if (url === '/fleet/labels/fleet-prune') { + return Promise.resolve(jsonResponse(200, { + results: [{ nodeId: 1, nodeName: 'central', reachable: true, targets: [{ target: 'images', success: true, reclaimedBytes: 4096, dryRun: true }] }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + render(); + await user.click(screen.getByRole('button', { name: 'Dry run' })); + await waitFor(() => { + const call = mockedFetch.mock.calls.find(c => c[0] === '/fleet/labels/fleet-prune'); + expect(call).toBeTruthy(); + expect(JSON.parse(call![1].body)).toEqual({ targets: ['images'], scope: 'managed', dryRun: true }); + }); + await waitFor(() => expect(toastSuccess).toHaveBeenCalled()); +}); + +it('surfaces an error toast when the prune returns non-ok', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/labels/fleet-prune') { + return Promise.resolve(jsonResponse(500, { error: 'prune blew up' })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + render(); + await user.click(screen.getByRole('button', { name: 'Dry run' })); + await waitFor(() => expect(toastError).toHaveBeenCalledWith('prune blew up')); +}); diff --git a/frontend/src/components/fleet/FleetActions/cards/LabelFleetStopCard.test.tsx b/frontend/src/components/fleet/FleetActions/cards/LabelFleetStopCard.test.tsx new file mode 100644 index 00000000..af9124b9 --- /dev/null +++ b/frontend/src/components/fleet/FleetActions/cards/LabelFleetStopCard.test.tsx @@ -0,0 +1,151 @@ +/** + * Coverage for LabelFleetStopCard. + * + * Locks the destructive-action contract: buttons gated on a label name, the + * confirm modal gates the real stop, dry run bypasses the modal, per-node + * results render, and every failure path surfaces a toast (no silent failure). + */ +import { it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor, within } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +vi.mock('@/lib/api', () => ({ + apiFetch: vi.fn(), + fetchForNode: vi.fn(), +})); + +const toastError = vi.fn(); +const toastSuccess = vi.fn(); +const toastInfo = vi.fn(); +const toastWarning = vi.fn(); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { + error: (...a: unknown[]) => toastError(...a), + success: (...a: unknown[]) => toastSuccess(...a), + info: (...a: unknown[]) => toastInfo(...a), + warning: (...a: unknown[]) => toastWarning(...a), + loading: vi.fn(() => 'toast-id'), + dismiss: vi.fn(), + }, +})); + +import { apiFetch, fetchForNode } from '@/lib/api'; +import { LabelFleetStopCard } from './LabelFleetStopCard'; +import type { FleetNode } from '@/components/FleetView/types'; + +const mockedFetch = apiFetch as unknown as ReturnType; +const mockedFetchForNode = fetchForNode as unknown as ReturnType; + +function jsonResponse(status: number, body: unknown): Response { + return { ok: status >= 200 && status < 300, status, json: async () => body } as unknown as Response; +} + +const nodes = [ + { id: 1, name: 'central', status: 'online' }, + { id: 2, name: 'edge-1', status: 'online' }, +] as unknown as FleetNode[]; + +beforeEach(() => { + vi.clearAllMocks(); + // Suggestion load on mount: no labels. + mockedFetchForNode.mockResolvedValue(jsonResponse(200, [])); + // Default: preview unavailable so the debounce effect never throws. + mockedFetch.mockResolvedValue(jsonResponse(404, {})); +}); + +it('disables both actions until a label name is entered', () => { + render(); + expect(screen.getByRole('button', { name: 'Stop fleet' })).toBeDisabled(); + expect(screen.getByRole('button', { name: 'Dry run' })).toBeDisabled(); +}); + +it('enables the actions once a label is typed', async () => { + const user = userEvent.setup(); + render(); + await user.type(screen.getByPlaceholderText('e.g. production'), 'prod'); + expect(screen.getByRole('button', { name: 'Stop fleet' })).toBeEnabled(); + expect(screen.getByRole('button', { name: 'Dry run' })).toBeEnabled(); +}); + +it('dry run calls fleet-stop with dryRun:true and never opens the confirm modal', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/labels/fleet-stop') { + return Promise.resolve(jsonResponse(200, { + results: [{ nodeId: 1, nodeName: 'central', matched: true, stackResults: [{ stackName: 'web', success: true, dryRun: true }] }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + render(); + await user.type(screen.getByPlaceholderText('e.g. production'), 'prod'); + await user.click(screen.getByRole('button', { name: 'Dry run' })); + + await waitFor(() => { + const stopCall = mockedFetch.mock.calls.find(c => c[0] === '/fleet/labels/fleet-stop'); + expect(stopCall).toBeTruthy(); + expect(JSON.parse(stopCall![1].body)).toEqual({ labelName: 'prod', dryRun: true }); + }); + expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument(); + await waitFor(() => expect(toastSuccess).toHaveBeenCalled()); +}); + +it('stop fleet opens the confirm modal, and confirming runs a real stop that renders per-node results', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/labels/fleet-stop') { + return Promise.resolve(jsonResponse(200, { + results: [ + { nodeId: 1, nodeName: 'central', matched: true, stackResults: [{ stackName: 'web', success: true }] }, + { nodeId: 2, nodeName: 'edge-1', matched: false, stackResults: [] }, + ], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + render(); + await user.type(screen.getByPlaceholderText('e.g. production'), 'prod'); + await user.click(screen.getByRole('button', { name: 'Stop fleet' })); + + const dialog = await screen.findByRole('alertdialog'); + expect(within(dialog).getByText('Stop all stacks labeled "prod"?')).toBeInTheDocument(); + + // The real stop must not have fired yet. + expect(mockedFetch.mock.calls.find(c => c[0] === '/fleet/labels/fleet-stop')).toBeFalsy(); + + await user.click(within(dialog).getByRole('button', { name: 'Stop fleet' })); + + await waitFor(() => { + const stopCall = mockedFetch.mock.calls.find(c => c[0] === '/fleet/labels/fleet-stop'); + expect(JSON.parse(stopCall![1].body)).toEqual({ labelName: 'prod', dryRun: false }); + }); + expect(await screen.findByText('web')).toBeInTheDocument(); + await waitFor(() => expect(toastSuccess).toHaveBeenCalled()); +}); + +it('surfaces an error toast when fleet-stop returns a non-ok response', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/labels/fleet-stop') { + return Promise.resolve(jsonResponse(500, { error: 'boom' })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + render(); + await user.type(screen.getByPlaceholderText('e.g. production'), 'prod'); + await user.click(screen.getByRole('button', { name: 'Dry run' })); + await waitFor(() => expect(toastError).toHaveBeenCalledWith('boom')); +}); + +it('populates the blast readout from the debounced match-preview', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/labels/match-preview') { + return Promise.resolve(jsonResponse(200, { matchedNodes: 2, matchedStacks: 3, perNode: [] })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + render(); + await user.type(screen.getByPlaceholderText('e.g. production'), 'prod'); + await waitFor(() => expect(screen.getByText('3 stacks ยท 2 nodes')).toBeInTheDocument(), { timeout: 2000 }); +});