fix: probe remote RBAC capability live and enforce exact stop-result membership (#1510)

The cross-node capability gate cached its verdict, so a remote replaced by older
code at the same URL stayed trusted until the cache expired, reopening the
non-admin HTTP escalation and the over-broad stop. The probe now hits the
remote's live /api/meta on every gated action (concurrent calls deduped, never
cached across requests, fail-closed), so a downgraded remote is detected
immediately.

Two stop-result gaps are also closed:

- A remote stop result must now cover exactly the confirmed stacks (one per
  stack, no extras, no omissions), not merely exclude extras, so a dropped
  confirmed stack is no longer accepted as clean. runLocalLabelStop reports one
  result per confirmed stack even when the label has vanished, so a current
  remote always satisfies the check.
- The local stop exception path now reports the full confirmed set, so a
  confirmed stack that lost its label is not dropped when the local stop throws.

backend/src/__tests__/fleet-action-card-endpoints.test.ts

@@ -14,6 +14,7 @@ import jwt from 'jsonwebtoken';
 import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
 
 let mockFsStacks: string[] = [];
+let mockFsStacksError: Error | null = null;
 const pruneManagedOnly = vi.fn();
 const pruneSystem = vi.fn();
 const estimateManagedReclaim = vi.fn();
@@ -31,7 +32,10 @@ vi.mock('../helpers/remoteCapabilities', () => ({
 vi.mock('../services/FileSystemService', () => ({
   FileSystemService: {
     getInstance: vi.fn(() => ({
-      getStacks: vi.fn(async () => mockFsStacks),
+      getStacks: vi.fn(async () => {
+        if (mockFsStacksError) throw mockFsStacksError;
+        return mockFsStacks;
+      }),
     })),
   },
 }));
@@ -81,6 +85,7 @@ beforeEach(() => {
   vi.restoreAllMocks();
   vi.clearAllMocks();
   mockFsStacks = ['alpha', 'beta'];
+  mockFsStacksError = null;
   pruneManagedOnly.mockResolvedValue({ success: true, reclaimedBytes: 0 });
   pruneSystem.mockResolvedValue({ success: true, reclaimedBytes: 0 });
   estimateManagedReclaim.mockResolvedValue({ reclaimableBytes: 0 });
@@ -878,7 +883,7 @@ describe('POST /api/fleet/labels/fleet-stop confirmed-target allowlist', () => {
     }
   });
 
-  it('fails a node whose stop results name stacks outside the confirmed set', async () => {
+  it('fails a node whose stop results include a stack outside the confirmed set', async () => {
     remoteSupportsCrossNodeRbac.mockResolvedValue(true);
     const remoteId = db.addNode({
       name: 'lying-remote', type: 'remote', api_url: 'http://lying.example:1852',
@@ -902,11 +907,57 @@ describe('POST /api/fleet/labels/fleet-stop confirmed-target allowlist', () => {
       expect(res.status).toBe(200);
       const node = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
       expect(node.reachable).toBe(false);
-      expect(node.error).toMatch(/outside the confirmed set/i);
+      expect(node.error).toMatch(/exactly the confirmed stacks/i);
     } finally {
       db.deleteNode(remoteId);
     }
   });
+
+  it('fails a node whose stop results omit a confirmed stack (partial result)', async () => {
+    remoteSupportsCrossNodeRbac.mockResolvedValue(true);
+    const remoteId = db.addNode({
+      name: 'dropping-remote', type: 'remote', api_url: 'http://dropping.example:1852',
+      api_token: 'tok', compose_dir: '/app/compose', is_default: false,
+    });
+    try {
+      vi.spyOn(globalThis, 'fetch').mockResolvedValue({
+        ok: true, status: 200,
+        // Two stacks confirmed, but the remote reports only one.
+        json: async () => ({ matched: true, results: [{ stackName: 'alpha', success: true }] }),
+      } as unknown as Response);
+      const res = await request(app)
+        .post('/api/fleet/labels/fleet-stop')
+        .set('Authorization', authHeader)
+        .send({ labelName: 'any-label', targets: [{ nodeId: remoteId, stackNames: ['alpha', 'beta'] }] });
+      expect(res.status).toBe(200);
+      const node = res.body.results.find((r: { nodeId: number }) => r.nodeId === remoteId);
+      expect(node.reachable).toBe(false);
+      expect(node.error).toMatch(/exactly the confirmed stacks/i);
+    } finally {
+      db.deleteNode(remoteId);
+    }
+  });
+
+  it('local exception path reports every confirmed stack, including one that lost its label', async () => {
+    // runLocalLabelStop throws (filesystem read fails) after the label is
+    // resolved; the catch must report the full confirmed set, not the current
+    // assignment, so a confirmed stack that lost its label still surfaces.
+    const localNodeId = db.getNodes().find(n => n.type === 'local')!.id;
+    const label = await createAssignedLabel('local-throw', ['alpha']); // only alpha is still assigned
+    expect(label.node_id).toBe(localNodeId);
+    mockFsStacksError = new Error('compose dir unreadable');
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: label.name, targets: [{ nodeId: localNodeId, stackNames: ['alpha', 'lost-label-stack'] }] });
+    expect(res.status).toBe(200);
+    const node = res.body.results.find((r: { nodeId: number }) => r.nodeId === localNodeId);
+    const byName = Object.fromEntries(node.stackResults.map((s: { stackName: string }) => [s.stackName, s]));
+    // Both confirmed stacks are failed; the one that lost its label is not dropped.
+    expect(byName['alpha']).toMatchObject({ success: false });
+    expect(byName['lost-label-stack']).toMatchObject({ success: false });
+    expect(node.stackResults).toHaveLength(2);
+  });
 });
 
 describe('POST /api/fleet/labels/fleet-stop with dryRun: true', () => {

backend/src/__tests__/fleet-actions.test.ts

@@ -420,6 +420,22 @@ describe('local-stop behavior', () => {
     expect(res.status).toBe(200);
     expect(res.body.results).toEqual([{ stackName: 'ghostdisk-stack', success: false, error: 'Stack not found on this node' }]);
   });
+
+  it('reports one failure per confirmed stack when the label no longer exists', async () => {
+    // The label vanished between preview and execution. With a confirmed
+    // allowlist the receiver must return one result per confirmed stack (so the
+    // control's exact-membership check sees a complete set), not an empty body.
+    const res = await request(app)
+      .post('/api/fleet-actions/labels/local-stop')
+      .set('Authorization', authHeader)
+      .send({ labelName: 'no-such-label', dryRun: true, stackNames: ['gone-a', 'gone-b'] });
+    expect(res.status).toBe(200);
+    expect(res.body.matched).toBe(false);
+    expect(res.body.results).toEqual([
+      { stackName: 'gone-a', success: false, error: 'No longer carries this label' },
+      { stackName: 'gone-b', success: false, error: 'No longer carries this label' },
+    ]);
+  });
 });
 
 // Orchestrator-level binding: the control sends the exact node + stack list the

backend/src/__tests__/proxy-cross-node-rbac-gate.test.ts

@@ -23,8 +23,9 @@ let noCapNodeId: number;
 
 const PROXIED_PATH = '/api/stacks';
 
-function metaServer(capabilities: string[]): http.Server {
+function metaServer(capabilities: string[], seen?: string[]): http.Server {
   return http.createServer((req, res) => {
+    if (seen && req.url) seen.push(req.url);
     res.writeHead(200, { 'Content-Type': 'application/json' });
     if (req.url?.startsWith('/api/meta')) {
       res.end(JSON.stringify({ version: '0.93.0', capabilities }));
@@ -35,6 +36,8 @@ function metaServer(capabilities: string[]): http.Server {
   });
 }
 
+const noCapPaths: string[] = [];
+
 async function listen(server: http.Server): Promise<number> {
   await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
   return (server.address() as import('net').AddressInfo).port;
@@ -53,7 +56,7 @@ beforeAll(async () => {
   adminBearer = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
 
   capServer = metaServer(['cross-node-rbac']);
-  noCapServer = metaServer(['fleet']); // older remote: no cross-node-rbac
+  noCapServer = metaServer(['fleet'], noCapPaths); // older remote: no cross-node-rbac
   const capPort = await listen(capServer);
   const noCapPort = await listen(noCapServer);
 
@@ -99,4 +102,22 @@ describe('remote proxy cross-node-rbac gate', () => {
       .set('x-node-id', String(noCapNodeId));
     expect(res.status).not.toBe(403);
   });
+
+  it('refuses a real stop to a remote lacking cross-node-rbac via the live probe, never contacting its local-stop receiver', async () => {
+    // Exercises the fleet-stop gate end to end through the REAL capability
+    // helper (not a mock): the live /api/meta probe of the no-cap remote returns
+    // no capability, so the stop is refused and the destructive receiver on that
+    // remote is never contacted.
+    noCapPaths.length = 0;
+    const res = await request(app)
+      .post('/api/fleet/labels/fleet-stop')
+      .set('Authorization', `Bearer ${adminBearer}`)
+      .send({ labelName: 'any-label', targets: [{ nodeId: noCapNodeId, stackNames: ['x'] }] });
+    expect(res.status).toBe(200);
+    const node = res.body.results.find((r: { nodeId: number }) => r.nodeId === noCapNodeId);
+    expect(node.reachable).toBe(false);
+    expect(node.error).toMatch(/upgrade/i);
+    expect(noCapPaths.some(p => p.startsWith('/api/meta'))).toBe(true);
+    expect(noCapPaths.some(p => p.includes('/api/fleet-actions/labels/local-stop'))).toBe(false);
+  });
 });

backend/src/__tests__/remote-capabilities.test.ts

@@ -1,8 +1,9 @@
 /**
  * Unit coverage for the cross-node-rbac capability probe used to gate
- * mixed-version cross-node operations. It reads the shared remote-meta cache
- * (fetching once on a cold miss) and must fail closed: a node whose capability
- * cannot be determined is treated as unsupported.
+ * mixed-version cross-node operations. It probes the remote's live /api/meta on
+ * every call (no cross-request caching), must fail closed when the capability
+ * cannot be determined, and must re-verify each time so a downgraded remote is
+ * detected immediately rather than trusted from a stale verdict.
  */
 import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
 import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
@@ -10,7 +11,6 @@ import type { RemoteMeta } from '../services/CapabilityRegistry';
 
 let remoteSupportsCrossNodeRbac: typeof import('../helpers/remoteCapabilities').remoteSupportsCrossNodeRbac;
 let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
-let CacheService: typeof import('../services/CacheService').CacheService;
 let tmpDir: string;
 
 const NODE_ID = 4242;
@@ -19,40 +19,63 @@ beforeAll(async () => {
   tmpDir = await setupTestDb();
   ({ remoteSupportsCrossNodeRbac } = await import('../helpers/remoteCapabilities'));
   ({ NodeRegistry } = await import('../services/NodeRegistry'));
-  ({ CacheService } = await import('../services/CacheService'));
 });
 
 afterAll(() => cleanupTestDb(tmpDir));
 
-afterEach(() => {
-  vi.restoreAllMocks();
-  CacheService.getInstance().invalidate(`remote-meta:${NODE_ID}`);
-});
-
-function mockMeta(meta: RemoteMeta): void {
-  vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(meta);
-}
+afterEach(() => vi.restoreAllMocks());
 
 const ONLINE = { startedAt: null, updateError: null, online: true } as const;
+const capable: RemoteMeta = { version: '0.93.0', capabilities: ['fleet', 'cross-node-rbac'], ...ONLINE };
+const incapable: RemoteMeta = { version: '0.92.0', capabilities: ['fleet', 'labels'], ...ONLINE };
 
 describe('remoteSupportsCrossNodeRbac', () => {
   it('returns true when the remote advertises cross-node-rbac', async () => {
-    mockMeta({ version: '0.93.0', capabilities: ['fleet', 'cross-node-rbac'], ...ONLINE });
+    vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(capable);
     expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(true);
   });
 
   it('returns false when the remote does not advertise it', async () => {
-    mockMeta({ version: '0.92.0', capabilities: ['fleet', 'labels'], ...ONLINE });
+    vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(incapable);
     expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(false);
   });
 
-  it('fails closed when the remote meta has no resolvable version', async () => {
-    mockMeta({ version: null, capabilities: [], startedAt: null, updateError: null, online: false });
+  it('fails closed when the remote is offline (empty capabilities)', async () => {
+    vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode')
+      .mockResolvedValue({ version: null, capabilities: [], startedAt: null, updateError: null, online: false });
     expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(false);
   });
 
+  it('trusts a reachable remote that advertises the capability even with a non-semver version', async () => {
+    // A 0.0.0-dev image reports version null (non-semver) but is reachable and
+    // genuinely advertises the capability; it must not be wrongly denied.
+    vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode')
+      .mockResolvedValue({ version: null, capabilities: ['fleet', 'cross-node-rbac'], startedAt: null, updateError: null, online: true });
+    expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(true);
+  });
+
   it('fails closed when the meta fetch throws (unreachable)', async () => {
     vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockRejectedValue(new Error('unreachable'));
     expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(false);
   });
+
+  it('re-probes on every call so a downgraded remote is detected immediately (no stale verdict)', async () => {
+    const spy = vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode')
+      .mockResolvedValueOnce(capable)   // first probe: current remote
+      .mockResolvedValue(incapable);    // after the remote is swapped for older code
+    expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(true);
+    expect(await remoteSupportsCrossNodeRbac(NODE_ID)).toBe(false);
+    expect(spy).toHaveBeenCalledTimes(2);
+  });
+
+  it('dedupes concurrent probes for the same node into a single fetch', async () => {
+    const spy = vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(capable);
+    const [a, b] = await Promise.all([
+      remoteSupportsCrossNodeRbac(NODE_ID),
+      remoteSupportsCrossNodeRbac(NODE_ID),
+    ]);
+    expect(a).toBe(true);
+    expect(b).toBe(true);
+    expect(spy).toHaveBeenCalledTimes(1);
+  });
 });