fix(fleet-actions): stop-by-label works on Community remote nodes (#1270)

* fix(fleet-actions): stop-by-label works on Community remote nodes Fleet-stop's remote leg fanned out to POST /api/labels/:id/action, which is gated to Skipper/Admiral, so on a Community fleet the control node stopped its own stacks but every remote node returned 403. Fleet-stop itself is admin-only and available on every license, so the remote leg contradicted the feature's own gate. Extract the label-match plus bulk-stop logic into a shared runLocalLabelStop helper and add an admin-only, every-license POST /api/fleet-actions/labels/local-stop receiver. The control now fans out to that receiver, so remote stacks stop on every tier. Each node runs under its own per-node bulk lock, so a fleet-stop and a per-label action still serialize cleanly instead of double-stopping containers. Also degrade the control's own leg per-node instead of failing the whole fan-out when its filesystem read throws, and gate fleet-stop and fleet-prune diagnostics behind developer_mode. Tests: local-stop auth, tier, validation, and behavior; a remote-leg routing guard that asserts the fan-out targets local-stop and never the paid route; local-leg graceful degradation; and the three Fleet Action card UIs. * fix(fleet-actions): honor the remote stop receiver's matched flag The control reached the remote leg only because its own mirror had the label, then hardcoded matched:true and trusted results without guarding its shape. A mirror-skewed control (mirror has the label, remote does not) then showed a remote mismatch as "matched, 0 stacks" instead of "no matching label", and a malformed 200 body could flow a non-array into the per-stack renderers. Honor the remote's own matched flag and coerce results to an array when the body is malformed. Add regression tests for the matched:false skew case and the non-array results case.
2026-08-25 17:57:06 +00:00 · 2026-06-01 14:18:44 -04:00
parent 085267b466
commit 7e0cffa376
9 changed files with 797 additions and 44 deletions
@@ -244,6 +244,103 @@ describe('POST /api/fleet/prune/estimate', () => {
  });
 });

+describe('POST /api/fleet/labels/fleet-stop remote leg', () => {
+  // Guards the C-1 fix: the remote fan-out must target the admin-only
+  // /api/fleet-actions/labels/local-stop receiver (reachable on every license),
+  // never the paid /api/labels/:id/action it used to call, which 403'd on
+  // Community remotes.
+  it('fans out to the admin-only local-stop receiver, never the paid per-label action route', async () => {
+    const remoteId = db.addNode({
+      name: 'remote-stop',
+      type: 'remote',
+      api_url: 'http://remote-stop.example:1852',
+      api_token: 'tok',
+      compose_dir: '/app/compose',
+      is_default: false,
+    });
+    try {
+      const label = db.createLabel(remoteId, `remote-c1-${++labelCounter}`, 'teal');
+      db.setStackLabels('alpha', remoteId, [label.id]);
+      const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue({
+        ok: true,
+        status: 200,
+        json: async () => ({ matched: true, results: [{ stackName: 'alpha', success: true }] }),
+      } as unknown as Response);
+
+      const res = await request(app)
+        .post('/api/fleet/labels/fleet-stop')
+        .set('Authorization', authHeader)
+        .send({ labelName: label.name });
+
+      expect(res.status).toBe(200);
+      const urls = fetchSpy.mock.calls.map(c => String(c[0]));
+      expect(urls.some(u => u.endsWith('/api/fleet-actions/labels/local-stop'))).toBe(true);
+      expect(urls.some(u => u.includes('/api/labels/'))).toBe(false);
+
+      const call = fetchSpy.mock.calls.find(c => String(c[0]).endsWith('/api/fleet-actions/labels/local-stop'));
+      expect(JSON.parse((call![1] as RequestInit).body as string)).toEqual({ labelName: label.name, dryRun: false });
+
+      const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
+      expect(remoteRow.stackResults).toEqual([{ stackName: 'alpha', success: true }]);
+    } finally {
+      db.deleteNode(remoteId);
+    }
+  });
+
+  it('honors the remote matched:false flag over the control mirror (mirror skew)', async () => {
+    const remoteId = db.addNode({
+      name: 'remote-skew', type: 'remote', api_url: 'http://remote-skew.example:1852',
+      api_token: 'tok', compose_dir: '/app/compose', is_default: false,
+    });
+    try {
+      // The control mirror believes the remote carries this label + stack...
+      const label = db.createLabel(remoteId, `remote-skew-${++labelCounter}`, 'teal');
+      db.setStackLabels('alpha', remoteId, [label.id]);
+      // ...but the remote authoritatively reports it has no such label.
+      vi.spyOn(globalThis, 'fetch').mockResolvedValue({
+        ok: true, status: 200, json: async () => ({ matched: false, results: [] }),
+      } as unknown as Response);
+
+      const res = await request(app)
+        .post('/api/fleet/labels/fleet-stop')
+        .set('Authorization', authHeader)
+        .send({ labelName: label.name });
+
+      expect(res.status).toBe(200);
+      const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
+      expect(remoteRow.matched).toBe(false);
+      expect(remoteRow.stackResults).toEqual([]);
+    } finally {
+      db.deleteNode(remoteId);
+    }
+  });
+
+  it('degrades a malformed 200 body (non-array results) to empty instead of forwarding it', async () => {
+    const remoteId = db.addNode({
+      name: 'remote-malformed', type: 'remote', api_url: 'http://remote-malformed.example:1852',
+      api_token: 'tok', compose_dir: '/app/compose', is_default: false,
+    });
+    try {
+      const label = db.createLabel(remoteId, `remote-malformed-${++labelCounter}`, 'teal');
+      db.setStackLabels('alpha', remoteId, [label.id]);
+      vi.spyOn(globalThis, 'fetch').mockResolvedValue({
+        ok: true, status: 200, json: async () => ({ matched: true, results: 'not-an-array' }),
+      } as unknown as Response);
+
+      const res = await request(app)
+        .post('/api/fleet/labels/fleet-stop')
+        .set('Authorization', authHeader)
+        .send({ labelName: label.name });
+
+      expect(res.status).toBe(200);
+      const remoteRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
+      expect(remoteRow.stackResults).toEqual([]);
+    } finally {
+      db.deleteNode(remoteId);
+    }
+  });
+});
+
 describe('POST /api/fleet/labels/fleet-stop with dryRun: true', () => {
  it('marks each stack dryRun: true and does not invoke containerActionForStack', async () => {
    const label = await createAssignedLabel('dry-stop', ['alpha', 'beta']);
@@ -3,6 +3,8 @@
 * validation, and orchestration shape across the two routes.
 */
 import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
+import fs from 'fs';
+import path from 'path';
 import request from 'supertest';
 import jwt from 'jsonwebtoken';
 import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
@@ -138,3 +140,147 @@ describe('Fleet Actions orchestration shape', () => {
    expect(res.body.results[0]).toMatchObject({ success: false, error: 'Invalid stack name' });
  });
 });
+
+// The per-node local-stop receiver is what a control instance calls on each
+// remote during a fleet-wide stop. It must be reachable on every license (only
+// admin-gated): the original fleet-stop fan-out hit the paid /api/labels/:id/action
+// and 403'd on Community remotes. These tests lock that behavior in.
+describe('local-stop receiver auth + tier', () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it('POST /api/fleet-actions/labels/local-stop returns 401 without auth', async () => {
+    const res = await request(app).post('/api/fleet-actions/labels/local-stop').send({ labelName: 'prod' });
+    expect(res.status).toBe(401);
+  });
+
+  it('is reachable on community tier for admins and never returns PAID_REQUIRED', async () => {
+    mockTier('community');
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/local-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: 'this-label-does-not-exist' });
+    expect(res.status).toBe(200);
+    expect(res.body.code).not.toBe('PAID_REQUIRED');
+    expect(res.body).toEqual({ matched: false, results: [] });
+  });
+
+  it('rejects missing labelName', async () => {
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/local-stop')
+      .set('Authorization', authHeader)
+      .send({});
+    expect(res.status).toBe(400);
+    expect(res.body.error).toMatch(/labelName/);
+  });
+
+  it('rejects whitespace-only labelName', async () => {
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/local-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: '   ' });
+    expect(res.status).toBe(400);
+  });
+});
+
+describe('local-stop behavior', () => {
+  let db: import('../services/DatabaseService').DatabaseService;
+  let nodeId: number;
+
+  beforeAll(async () => {
+    const { DatabaseService } = await import('../services/DatabaseService');
+    const { NodeRegistry } = await import('../services/NodeRegistry');
+    db = DatabaseService.getInstance();
+    nodeId = NodeRegistry.getInstance().getDefaultNodeId();
+  });
+
+  afterEach(() => vi.restoreAllMocks());
+
+  it('matched:true with empty results when the label exists but has no stacks', async () => {
+    db.createLabel(nodeId, 'no-stacks-label', '#ffffff');
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/local-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: 'no-stacks-label' });
+    expect(res.status).toBe(200);
+    expect(res.body).toEqual({ matched: true, results: [] });
+  });
+
+  it('reports per-stack lock contention when a bulk action is already running on the node', async () => {
+    const label = db.createLabel(nodeId, 'busy-label', '#ffffff');
+    db.setStackLabels('busy-stack', nodeId, [label.id]);
+    const { activeBulkActions } = await import('../routes/labels');
+    activeBulkActions.add(`bulk:${nodeId}`);
+    try {
+      const res = await request(app)
+        .post('/api/fleet-actions/labels/local-stop')
+        .set('Authorization', authHeader)
+        .send({ labelName: 'busy-label' });
+      expect(res.status).toBe(200);
+      expect(res.body.matched).toBe(true);
+      expect(res.body.results).toEqual([
+        { stackName: 'busy-stack', success: false, error: 'A bulk action is already running on this node' },
+      ]);
+    } finally {
+      activeBulkActions.delete(`bulk:${nodeId}`);
+    }
+  });
+
+  it('dry run returns dryRun:true per on-disk stack without touching Docker', async () => {
+    const composeDir = process.env.COMPOSE_DIR as string;
+    fs.mkdirSync(path.join(composeDir, 'dry-stack'), { recursive: true });
+    fs.writeFileSync(path.join(composeDir, 'dry-stack', 'docker-compose.yml'), 'services: {}\n');
+    const label = db.createLabel(nodeId, 'dry-label', '#ffffff');
+    db.setStackLabels('dry-stack', nodeId, [label.id]);
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/local-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: 'dry-label', dryRun: true });
+    expect(res.status).toBe(200);
+    expect(res.body.matched).toBe(true);
+    expect(res.body.results).toEqual([{ stackName: 'dry-stack', success: true, dryRun: true }]);
+  });
+
+  it('filters out assigned stacks that are not present on disk', async () => {
+    const label = db.createLabel(nodeId, 'ghost-label', '#ffffff');
+    db.setStackLabels('ghost-stack', nodeId, [label.id]);
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/local-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: 'ghost-label' });
+    expect(res.status).toBe(200);
+    expect(res.body).toEqual({ matched: true, results: [] });
+  });
+});
+
+describe('fleet-stop degrades the local leg per-node instead of failing the whole fan-out', () => {
+  let db: import('../services/DatabaseService').DatabaseService;
+  let nodeId: number;
+
+  beforeAll(async () => {
+    const { DatabaseService } = await import('../services/DatabaseService');
+    const { NodeRegistry } = await import('../services/NodeRegistry');
+    db = DatabaseService.getInstance();
+    nodeId = NodeRegistry.getInstance().getDefaultNodeId();
+  });
+
+  afterEach(() => vi.restoreAllMocks());
+
+  it('returns 200 with per-stack errors when the control filesystem read throws', async () => {
+    const label = db.createLabel(nodeId, 'degrade-label', '#ffffff');
+    db.setStackLabels('degrade-stack', nodeId, [label.id]);
+    const { FileSystemService } = await import('../services/FileSystemService');
+    vi.spyOn(FileSystemService.prototype, 'getStacks').mockRejectedValue(new Error('compose dir unreadable'));
+
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: 'degrade-label' });
+
+    expect(res.status).toBe(200);
+    const localRow = res.body.results.find((r: { nodeId: number }) => r.nodeId === nodeId);
+    expect(localRow.matched).toBe(true);
+    expect(localRow.stackResults).toEqual([
+      { stackName: 'degrade-stack', success: false, error: 'compose dir unreadable' },
+    ]);
+  });
+});