feat(fleet): add Fleet Actions tab for cross-node bulk operations (#963)

* feat(fleet): add Fleet Actions tab for cross-node bulk operations

Introduces a new "Actions" sub-tab in Fleet view with two Skipper+ cards
that fill gaps in the existing surface:

- Stop fleet by label: matches a label name across every node and stops
  every stack assigned to it, reporting per-node and per-stack results.
- Bulk label assign: applies the same label set to many stacks on one
  node in a single round trip.

Other bulk operations stay in their existing homes (sidebar bulk mode,
Schedules, NodeUpdatesSheet) to avoid duplicate surfaces.

Backend:
- POST /api/fleet/labels/fleet-stop (gateway-orchestrated, multi-node)
- POST /api/fleet-actions/labels/bulk-assign (per-node, capped at 1000)
- Tightens /api/fleet proxy-exempt prefix to /api/fleet/ so
  /api/fleet-actions/* is routed through the proxy for per-node calls.
- Exports activeBulkActions from labels.ts so fleet-stop and label-action
  share the per-node lock and cannot double-stop the same containers.
- Extracts containerActionForStack helper from stacks.ts for reuse.

* chore(fleet): rename Actions tab to Fleet Actions and reorder Fleet sub-tabs

- Tab label "Actions" -> "Fleet Actions" so the surface is unambiguous
  alongside Schedules and the sidebar bulk bar.
- Reorder Fleet sub-tabs as Overview / Snapshots / Status | Deployments /
  Traffic / Fleet Actions, with the separator after Status.
- Rename "Traffic · Routing" -> "Traffic" and update Sencho Mesh docs to
  match the shorter label.
- Update Fleet Actions docs to the new tab name and placement.

backend/src/__tests__/fleet-actions.test.ts

@@ -0,0 +1,145 @@
+/**
+ * Tests for the Fleet Actions tab endpoints. Covers auth, tier gating, input
+ * validation, and orchestration shape across the two routes.
+ */
+import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
+import request from 'supertest';
+import jwt from 'jsonwebtoken';
+import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
+
+let tmpDir: string;
+let app: import('express').Express;
+let authHeader: string;
+let LicenseService: typeof import('../services/LicenseService').LicenseService;
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ app } = await import('../index'));
+  ({ LicenseService } = await import('../services/LicenseService'));
+  const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
+  authHeader = `Bearer ${token}`;
+});
+
+afterAll(() => cleanupTestDb(tmpDir));
+
+function mockTier(tier: 'paid' | 'community') {
+  vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue(tier);
+}
+
+describe('Fleet Actions endpoints require authentication', () => {
+  it('POST /api/fleet-actions/labels/bulk-assign returns 401 without auth', async () => {
+    const res = await request(app).post('/api/fleet-actions/labels/bulk-assign').send({ assignments: [] });
+    expect(res.status).toBe(401);
+  });
+
+  it('POST /api/fleet/labels/fleet-stop returns 401 without auth', async () => {
+    const res = await request(app).post('/api/fleet/labels/fleet-stop').send({ labelName: 'prod' });
+    expect(res.status).toBe(401);
+  });
+});
+
+describe('Fleet Actions tier gating', () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it('POST /api/fleet/labels/fleet-stop returns 403 on community tier (Skipper+)', async () => {
+    mockTier('community');
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: 'prod' });
+    expect(res.status).toBe(403);
+    expect(res.body.code).toBe('PAID_REQUIRED');
+  });
+
+  it('POST /api/fleet-actions/labels/bulk-assign returns 403 on community tier (Skipper+)', async () => {
+    mockTier('community');
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/bulk-assign')
+      .set('Authorization', authHeader)
+      .send({ assignments: [] });
+    expect(res.status).toBe(403);
+    expect(res.body.code).toBe('PAID_REQUIRED');
+  });
+});
+
+describe('Fleet Actions input validation', () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it('POST /api/fleet/labels/fleet-stop rejects missing labelName', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-stop')
+      .set('Authorization', authHeader)
+      .send({});
+    expect(res.status).toBe(400);
+    expect(res.body.error).toMatch(/labelName/);
+  });
+
+  it('POST /api/fleet/labels/fleet-stop rejects whitespace-only labelName', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: '   ' });
+    expect(res.status).toBe(400);
+  });
+
+  it('POST /api/fleet-actions/labels/bulk-assign rejects non-array assignments', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/bulk-assign')
+      .set('Authorization', authHeader)
+      .send({ assignments: 'oops' });
+    expect(res.status).toBe(400);
+    expect(res.body.error).toMatch(/assignments must be an array/);
+  });
+
+  it('POST /api/fleet-actions/labels/bulk-assign rejects oversized payload', async () => {
+    mockTier('paid');
+    const big = Array.from({ length: 1001 }, (_, i) => ({ stackName: `s${i}`, labelIds: [] }));
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/bulk-assign')
+      .set('Authorization', authHeader)
+      .send({ assignments: big });
+    expect(res.status).toBe(400);
+    expect(res.body.error).toMatch(/may not exceed/);
+  });
+});
+
+describe('Fleet Actions orchestration shape', () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it('POST /api/fleet/labels/fleet-stop with unknown label returns matched:false per node', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: 'this-label-does-not-exist' });
+    expect(res.status).toBe(200);
+    expect(Array.isArray(res.body.results)).toBe(true);
+    for (const row of res.body.results) {
+      expect(row.matched).toBe(false);
+      expect(row.stackResults).toEqual([]);
+    }
+  });
+
+  it('POST /api/fleet-actions/labels/bulk-assign accepts empty assignments and returns empty results', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/bulk-assign')
+      .set('Authorization', authHeader)
+      .send({ assignments: [] });
+    expect(res.status).toBe(200);
+    expect(res.body.results).toEqual([]);
+  });
+
+  it('POST /api/fleet-actions/labels/bulk-assign rejects an entry with bad stack name in-line', async () => {
+    mockTier('paid');
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/bulk-assign')
+      .set('Authorization', authHeader)
+      .send({ assignments: [{ stackName: 'has spaces!', labelIds: [1] }] });
+    expect(res.status).toBe(200);
+    expect(res.body.results[0]).toMatchObject({ success: false, error: 'Invalid stack name' });
+  });
+});

backend/src/helpers/proxyExemptPaths.ts

@@ -10,7 +10,7 @@ export const PROXY_EXEMPT_PREFIXES: readonly string[] = [
   '/api/auth/',
   '/api/nodes',
   '/api/license',
-  '/api/fleet',
+  '/api/fleet/',
   '/api/webhooks',
   '/api/meta',
 ];

backend/src/index.ts

@@ -20,6 +20,7 @@ import { webhooksRouter } from './routes/webhooks';
 import { usersRouter } from './routes/users';
 import { gitSourcesRouter, stackGitSourceRouter } from './routes/gitSources';
 import { fleetRouter } from './routes/fleet';
+import { fleetActionsRouter } from './routes/fleetActions';
 import { cloudBackupRouter } from './routes/cloudBackup';
 import { permissionsRouter } from './routes/permissions';
 import { convertRouter } from './routes/convert';
@@ -98,6 +99,7 @@ app.use('/api/stacks', stackLabelsRouter);
 app.use('/api/api-tokens', apiTokensRouter);
 app.use('/api/audit-log', auditLogRouter);
 app.use('/api/fleet', fleetRouter);
+app.use('/api/fleet-actions', fleetActionsRouter);
 app.use('/api/cloud-backup', cloudBackupRouter);
 app.use('/api/webhooks', webhooksRouter);
 app.use('/api/users', usersRouter);

backend/src/routes/fleet.ts

@@ -25,6 +25,9 @@ import { POLICY_SEVERITIES } from '../utils/severity';
 import { sanitizeForLog } from '../utils/safeLog';
 import { CloudBackupService } from '../services/CloudBackupService';
 import { NotificationService } from '../services/NotificationService';
+import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
+import { containerActionForStack } from './stacks';
+import { activeBulkActions } from './labels';
 import { buildLocalConfigurationStatus, type ConfigurationStatus } from './dashboard';
 import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from '../services/license-headers';
 import { LicenseService } from '../services/LicenseService';
@@ -844,6 +847,110 @@ fleetRouter.delete('/update-status', authMiddleware, async (req: Request, res: R
   res.status(204).send();
 });
 
+// ─── Fleet Actions: gateway-orchestrated endpoints (multi-node) ───
+//
+// Per-node fleet-action endpoints (run on the target node via the proxy) live
+// in `routes/fleetActions.ts`. The endpoint below is gateway-orchestrated and
+// lives here so it sits behind the `/api/fleet/` proxy-exempt prefix.
+
+// Fleet-wide stop by label name. Matches each node's labels by name and runs
+// container stops on each matching stack.
+// Tier: requirePaid + requireAdmin.
+fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res: Response): Promise<void> => {
+  if (!requirePaid(req, res)) return;
+  if (!requireAdmin(req, res)) return;
+  const body = req.body as { labelName?: unknown } | undefined;
+  if (!body || typeof body !== 'object') {
+    res.status(400).json({ error: 'Request body is required' });
+    return;
+  }
+  const { labelName } = body;
+  if (typeof labelName !== 'string' || labelName.trim().length === 0) {
+    res.status(400).json({ error: 'labelName is required' });
+    return;
+  }
+  const trimmed = labelName.trim();
+  try {
+    const db = DatabaseService.getInstance();
+    const nodes = db.getNodes();
+    const results = await Promise.all(nodes.map(async (node) => {
+      const label = db.getLabels(node.id).find(l => l.name === trimmed);
+      if (!label) {
+        return { nodeId: node.id, nodeName: node.name, matched: false, stackResults: [] };
+      }
+      const stackNames = db.getStacksForLabel(label.id, node.id);
+      if (stackNames.length === 0) {
+        return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: [] };
+      }
+
+      if (node.type === 'local') {
+        // Share the per-node bulk lock with `POST /api/labels/:id/action` so
+        // a fleet-stop and a per-label action cannot double-stop the same
+        // containers concurrently on the same local node.
+        const lockKey = `bulk:${node.id}`;
+        if (activeBulkActions.has(lockKey)) {
+          return {
+            nodeId: node.id, nodeName: node.name, matched: true,
+            stackResults: stackNames.map(stackName => ({ stackName, success: false, error: 'A bulk action is already running on this node' })),
+          };
+        }
+        activeBulkActions.add(lockKey);
+        try {
+          const fsStacks = await FileSystemService.getInstance(node.id).getStacks();
+          const fsStackSet = new Set(fsStacks);
+          const validStacks = stackNames.filter(name => fsStackSet.has(name));
+          const stackResults: { stackName: string; success: boolean; error?: string }[] = [];
+          for (const stackName of validStacks) {
+            const outcome = await containerActionForStack(node.id, stackName, 'stop');
+            if (outcome.kind === 'ok') stackResults.push({ stackName, success: true });
+            else if (outcome.kind === 'no-containers') stackResults.push({ stackName, success: false, error: 'No containers found for this stack' });
+            else stackResults.push({ stackName, success: false, error: outcome.message });
+          }
+          if (stackResults.some(r => r.success)) invalidateNodeCaches(node.id);
+          return { nodeId: node.id, nodeName: node.name, matched: true, stackResults };
+        } finally {
+          activeBulkActions.delete(lockKey);
+        }
+      }
+
+      if (!node.api_url || !node.api_token) {
+        return {
+          nodeId: node.id, nodeName: node.name, matched: true,
+          stackResults: stackNames.map(stackName => ({ stackName, success: false, error: 'Remote node not configured' })),
+        };
+      }
+      try {
+        const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/labels/${label.id}/action`, {
+          method: 'POST',
+          headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' },
+          body: JSON.stringify({ action: 'stop' }),
+          signal: AbortSignal.timeout(60000),
+        });
+        if (!response.ok) {
+          const err = (await response.json().catch(() => ({}))) as { error?: string };
+          const message = err.error || `Remote returned ${response.status}`;
+          return {
+            nodeId: node.id, nodeName: node.name, matched: true,
+            stackResults: stackNames.map(stackName => ({ stackName, success: false, error: message })),
+          };
+        }
+        const remote = (await response.json()) as { results?: { stackName: string; success: boolean; error?: string }[] };
+        return { nodeId: node.id, nodeName: node.name, matched: true, stackResults: remote.results ?? [] };
+      } catch (err) {
+        const errorMsg = getErrorMessage(err, 'Failed to reach remote node');
+        return {
+          nodeId: node.id, nodeName: node.name, matched: true,
+          stackResults: stackNames.map(stackName => ({ stackName, success: false, error: errorMsg })),
+        };
+      }
+    }));
+    res.json({ results });
+  } catch (error) {
+    console.error('[Fleet] fleet-stop error:', error);
+    res.status(500).json({ error: getErrorMessage(error, 'Failed to run fleet stop') });
+  }
+});
+
 // ─── Fleet Snapshots (manual: Community; scheduled: Skipper+) ───
 
 fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Response): Promise<void> => {

backend/src/routes/fleetActions.ts

@@ -0,0 +1,66 @@
+import { Router, type Request, type Response } from 'express';
+import { DatabaseService } from '../services/DatabaseService';
+import { authMiddleware } from '../middleware/auth';
+import { requirePaid, requireAdmin, requireBody } from '../middleware/tierGates';
+import { isValidStackName } from '../utils/validation';
+import { getErrorMessage } from '../utils/errors';
+
+// Per-node fleet-action endpoints. Mounted under `/api/fleet-actions/`, which
+// is NOT in `PROXY_EXEMPT_PREFIXES`, so when `x-node-id` targets a remote node
+// the gateway proxies the call and the remote Sencho instance runs its own
+// local handler. Multi-node orchestration endpoints live in `routes/fleet.ts`
+// because their path must sit behind the `/api/fleet/` proxy-exempt prefix.
+export const fleetActionsRouter = Router();
+
+// Hard cap to bound a single bulk-assign request. A node typically has tens of
+// stacks, not thousands; the cap protects against accidental or malicious
+// payloads that would force thousands of DB writes in one handler.
+const MAX_ASSIGNMENTS = 1000;
+
+// Bulk label assignment for many stacks on a single node. The single-stack
+// endpoint at `PUT /api/stacks/:stackName/labels` covers one stack at a time;
+// this wrapper applies the same operation to many stacks atomically per HTTP
+// request. Tier: requirePaid + requireAdmin (matches the per-stack endpoint).
+fleetActionsRouter.post(
+  '/labels/bulk-assign',
+  authMiddleware,
+  async (req: Request, res: Response): Promise<void> => {
+    if (!requirePaid(req, res)) return;
+    if (!requireAdmin(req, res)) return;
+    if (!requireBody(req, res)) return;
+    const { assignments } = req.body as { assignments?: unknown };
+    if (!Array.isArray(assignments)) {
+      res.status(400).json({ error: 'assignments must be an array' });
+      return;
+    }
+    if (assignments.length > MAX_ASSIGNMENTS) {
+      res.status(400).json({ error: `assignments may not exceed ${MAX_ASSIGNMENTS} entries` });
+      return;
+    }
+    const nodeId = req.nodeId ?? 0;
+    const db = DatabaseService.getInstance();
+    const results: { stackName: string; success: boolean; error?: string }[] = [];
+    for (const entry of assignments as unknown[]) {
+      if (!entry || typeof entry !== 'object') {
+        results.push({ stackName: '', success: false, error: 'Invalid assignment entry' });
+        continue;
+      }
+      const { stackName, labelIds } = entry as { stackName?: unknown; labelIds?: unknown };
+      if (typeof stackName !== 'string' || !isValidStackName(stackName)) {
+        results.push({ stackName: typeof stackName === 'string' ? stackName : '', success: false, error: 'Invalid stack name' });
+        continue;
+      }
+      if (!Array.isArray(labelIds) || !labelIds.every(id => typeof id === 'number')) {
+        results.push({ stackName, success: false, error: 'labelIds must be an array of numbers' });
+        continue;
+      }
+      try {
+        db.setStackLabels(stackName, nodeId, labelIds);
+        results.push({ stackName, success: true });
+      } catch (err) {
+        results.push({ stackName, success: false, error: getErrorMessage(err, 'Failed to set stack labels') });
+      }
+    }
+    res.json({ results });
+  },
+);

backend/src/routes/labels.ts

@@ -15,7 +15,11 @@ import { getErrorMessage, isSqliteUniqueViolation } from '../utils/errors';
 import { parseIntParam } from '../utils/parseIntParam';
 import { sanitizeForLog } from '../utils/safeLog';
 
-const activeBulkActions = new Set<string>();
+// Module-scope lock shared by `POST /api/labels/:id/action` and the fleet-wide
+// bulk endpoints in `routes/fleet.ts`. Keyed by `${nodeId}` so concurrent bulk
+// actions targeting the same node serialize and a fleet-stop cannot race a
+// per-label action on the same containers.
+export const activeBulkActions = new Set<string>();
 
 export const labelsRouter = Router();
 

backend/src/routes/stacks.ts

@@ -624,7 +624,7 @@ stacksRouter.post('/:stackName/down', async (req: Request, res: Response) => {
   }
 });
 
-type StackContainerAction = 'restart' | 'stop' | 'start';
+export type StackContainerAction = 'restart' | 'stop' | 'start';
 
 const CONTAINER_ACTION_META: Record<StackContainerAction, { category: NotificationCategory; pastTense: string }> = {
   restart: { category: 'stack_restarted', pastTense: 'restarted' },
@@ -632,6 +632,31 @@ const CONTAINER_ACTION_META: Record<StackContainerAction, { category: Notificati
   start:   { category: 'stack_started',   pastTense: 'started'   },
 };
 
+export type ContainerActionOutcome =
+  | { kind: 'ok'; count: number }
+  | { kind: 'no-containers' }
+  | { kind: 'error'; message: string };
+
+export async function containerActionForStack(
+  nodeId: number,
+  stackName: string,
+  action: StackContainerAction,
+): Promise<ContainerActionOutcome> {
+  try {
+    const dockerController = DockerController.getInstance(nodeId);
+    const containers = await dockerController.getContainersByStack(stackName);
+    if (!containers || containers.length === 0) return { kind: 'no-containers' };
+    const op =
+      action === 'restart' ? (id: string) => dockerController.restartContainer(id)
+        : action === 'stop' ? (id: string) => dockerController.stopContainer(id)
+          : (id: string) => dockerController.startContainer(id);
+    await Promise.all(containers.map(c => op(c.Id)));
+    return { kind: 'ok', count: containers.length };
+  } catch (error: unknown) {
+    return { kind: 'error', message: getErrorMessage(error, `Failed to ${action} containers`) };
+  }
+}
+
 async function bulkContainerOp(
   req: Request,
   res: Response,
@@ -640,34 +665,24 @@ async function bulkContainerOp(
   const stackName = req.params.stackName as string;
   if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
   const titleCase = action.charAt(0).toUpperCase() + action.slice(1);
-  try {
-    const dockerController = DockerController.getInstance(req.nodeId);
-    const containers = await dockerController.getContainersByStack(stackName);
+  const outcome = await containerActionForStack(req.nodeId, stackName, action);
 
-    if (!containers || containers.length === 0) {
-      res.status(404).json({ error: 'No containers found for this stack.' });
-      return;
-    }
-
-    const op =
-      action === 'restart' ? (id: string) => dockerController.restartContainer(id)
-        : action === 'stop' ? (id: string) => dockerController.stopContainer(id)
-          : (id: string) => dockerController.startContainer(id);
-
-    await Promise.all(containers.map(c => op(c.Id)));
-    invalidateNodeCaches(req.nodeId);
-    console.log(`[Stacks] ${titleCase} completed: ${sanitizeForLog(stackName)} (${containers.length} containers)`);
-    res.json({ success: true, message: `${titleCase} completed via Engine API.` });
-    const { category, pastTense } = CONTAINER_ACTION_META[action];
-    notifyActionSuccess(category, `${stackName} ${pastTense}`, stackName, req.user?.username ?? 'system');
-  } catch (error: unknown) {
-    console.error('[Stacks] %s failed: %s', sanitizeForLog(titleCase), sanitizeForLog(stackName), error);
-    const message = getErrorMessage(error, `Failed to ${action} containers`);
-    if (action !== 'start') {
-      notifyActionFailure(action, stackName, error);
-    }
-    res.status(500).json({ error: message });
+  if (outcome.kind === 'no-containers') {
+    res.status(404).json({ error: 'No containers found for this stack.' });
+    return;
   }
+  if (outcome.kind === 'error') {
+    console.error('[Stacks] %s failed: %s %s', sanitizeForLog(titleCase), sanitizeForLog(stackName), sanitizeForLog(outcome.message));
+    if (action !== 'start') notifyActionFailure(action, stackName, new Error(outcome.message));
+    res.status(500).json({ error: outcome.message });
+    return;
+  }
+
+  invalidateNodeCaches(req.nodeId);
+  console.log(`[Stacks] ${titleCase} completed: ${sanitizeForLog(stackName)} (${outcome.count} containers)`);
+  res.json({ success: true, message: `${titleCase} completed via Engine API.` });
+  const { category, pastTense } = CONTAINER_ACTION_META[action];
+  notifyActionSuccess(category, `${stackName} ${pastTense}`, stackName, req.user?.username ?? 'system');
 }
 
 stacksRouter.post('/:stackName/restart', (req, res) => bulkContainerOp(req, res, 'restart'));