fix: probe remote RBAC capability live and enforce exact stop-result membership (#1510)

The cross-node capability gate cached its verdict, so a remote replaced by older
code at the same URL stayed trusted until the cache expired, reopening the
non-admin HTTP escalation and the over-broad stop. The probe now hits the
remote's live /api/meta on every gated action (concurrent calls deduped, never
cached across requests, fail-closed), so a downgraded remote is detected
immediately.

Two stop-result gaps are also closed:

- A remote stop result must now cover exactly the confirmed stacks (one per
  stack, no extras, no omissions), not merely exclude extras, so a dropped
  confirmed stack is no longer accepted as clean. runLocalLabelStop reports one
  result per confirmed stack even when the label has vanished, so a current
  remote always satisfies the check.
- The local stop exception path now reports the full confirmed set, so a
  confirmed stack that lost its label is not dropped when the local stop throws.

backend/src/helpers/fleetLabelStop.ts

@@ -79,7 +79,19 @@ export async function runLocalLabelStop(
 ): Promise<LabelStopOutcome> {
   const db = DatabaseService.getInstance();
   const label = db.getLabels(nodeId).find(l => l.name === labelName);
-  if (!label) return { matched: false, stackResults: [] };
+  if (!label) {
+    // With a confirmed allowlist, report each confirmed stack as a failure
+    // rather than an empty result: the label vanished here between preview and
+    // execution, so none can be stopped, and the control's exact-membership
+    // check must still see one result per confirmed stack (not a silent no-op).
+    if (allowedStacks) {
+      return {
+        matched: false,
+        stackResults: [...allowedStacks].map(stackName => ({ stackName, success: false, error: 'No longer carries this label' })),
+      };
+    }
+    return { matched: false, stackResults: [] };
+  }
   const stackNames = db.getStacksForLabel(label.id, nodeId);
   // With no confirmed allowlist this is the unbound path: an unassigned label is
   // a clean no-op. When stacks were confirmed we fall through so any that left

backend/src/helpers/remoteCapabilities.ts

@@ -1,44 +1,54 @@
-import { CacheService } from '../services/CacheService';
 import { NodeRegistry } from '../services/NodeRegistry';
-import { CROSS_NODE_RBAC_CAPABILITY, type RemoteMeta } from '../services/CapabilityRegistry';
-import { REMOTE_META_NAMESPACE } from './cacheInvalidation';
+import { CROSS_NODE_RBAC_CAPABILITY } from '../services/CapabilityRegistry';
 import { getErrorMessage } from '../utils/errors';
 
-// Mirrors the node-meta endpoint's TTL and shares its `remote-meta:<id>` cache
-// key, so a recent /api/nodes/:id/meta read warms this check and vice versa.
-const REMOTE_META_CACHE_TTL = 3 * 60 * 1000;
+// In-flight probes deduped per node so a burst of concurrent gated requests
+// shares one /api/meta round-trip. The entry is dropped as soon as it settles,
+// so the NEXT request re-probes. The verdict is deliberately NOT cached across
+// requests: a remote can be replaced by older code at the same URL (a rollback
+// or image pin), and a stale "supported" verdict would reopen the cross-node
+// escalation, so each gated action re-verifies against the live remote.
+const inFlight = new Map<number, Promise<boolean>>();
 
 /**
  * Whether a remote node advertises that it enforces cross-node RBAC: the
  * forwarded actor role on HTTP requests and the exact-stack allowlist on
- * stop-by-label. Reads the shared remote-meta cache, fetching once on a cold
- * miss.
+ * stop-by-label. Probes the remote's live /api/meta on every call (concurrent
+ * calls for the same node share one probe).
  *
- * Fails closed: when the capability cannot be established (an un-upgraded
- * remote, or a cold cache whose meta fetch fails) it returns false, so the
- * caller denies rather than risk escalating a non-admin request or over-stopping
- * on an un-upgraded node. A node previously cached as supported may be served
- * that value while a later refresh is failing (getOrFetch serves stale on
- * error); that is safe because capabilities are append-only and a request to an
- * unreachable node fails at the transport regardless.
+ * Fails closed: a remote that does not advertise the capability, that cannot be
+ * read (offline/unreachable, which yields empty capabilities), or that errors
+ * is treated as unsupported, so the caller denies rather than risk escalating a
+ * non-admin request or over-stopping. Because the probe is live, a remote
+ * downgraded to older code is detected on the next gated action rather than
+ * trusted until a cache expires.
  */
 export async function remoteSupportsCrossNodeRbac(nodeId: number): Promise<boolean> {
+  const existing = inFlight.get(nodeId);
+  if (existing) return existing;
+
+  const probe = (async (): Promise<boolean> => {
+    try {
+      const meta = await NodeRegistry.getInstance().fetchMetaForNode(nodeId);
+      // Check the advertised capability directly. An offline/unreadable remote
+      // yields OFFLINE_META with empty capabilities, so this already fails
+      // closed; keying off the capability (not the version) also correctly
+      // trusts a reachable remote whose version string is non-semver, e.g. a
+      // 0.0.0-dev image, but that genuinely advertises the capability.
+      return meta.capabilities.includes(CROSS_NODE_RBAC_CAPABILITY);
+    } catch (err) {
+      console.warn(
+        `[CrossNodeRBAC] Could not verify capability for node ${nodeId}; treating as unsupported:`,
+        getErrorMessage(err, 'unknown'),
+      );
+      return false;
+    }
+  })();
+
+  inFlight.set(nodeId, probe);
   try {
-    const meta = await CacheService.getInstance().getOrFetch<RemoteMeta>(
-      `${REMOTE_META_NAMESPACE}:${nodeId}`,
-      REMOTE_META_CACHE_TTL,
-      async () => {
-        const fetched = await NodeRegistry.getInstance().fetchMetaForNode(nodeId);
-        if (fetched.version === null) throw new Error('Remote meta fetch returned null version');
-        return fetched;
-      },
-    );
-    return meta.capabilities.includes(CROSS_NODE_RBAC_CAPABILITY);
-  } catch (err) {
-    console.warn(
-      `[CrossNodeRBAC] Could not determine capability for node ${nodeId}; treating as unsupported:`,
-      getErrorMessage(err, 'unknown'),
-    );
-    return false;
+    return await probe;
+  } finally {
+    inFlight.delete(nodeId);
   }
 }