feat(settings): surface security, notifications, and app store on remote nodes (#716)

Flip Security (Trivy), Notifications (agents + history), and App Store from
global-and-hidden-on-remote to node-scoped so operators can manage them when a
remote node is selected in the node picker. The primary instance proxies the
calls to each remote, which resolves the correct per-instance binary state,
agent config, and template registry.

Backend: key `agents` and `notification_history` by `node_id` with idempotent
column-add migrations and a `(node_id, type)` unique index on agents, matching
the Labels pattern. Thread `req.nodeId` through the /api/agents and
/api/notifications routes. Internal NotificationService and ImageUpdateService
writes resolve the middleware default via `NodeRegistry.getDefaultNodeId()` so
monitor-emitted rows share a bucket with user-facing ones (avoids split-brain
where the UI sees test notifications but not internal alerts).

Frontend: split Security on remote to render only the scanner card and hide
scan policies and CVE suppressions (those remain control-plane-only). Drop the
misleading "Always Local" badge on Developer since retention windows govern
backend jobs, not UI state. Flip the App Store registry to node-scoped.

Docs: add a "What Settings apply per node" table to multi-node, clarify
remote alert setup in alerts-notifications, and note Trivy's per-host install
in vulnerability-scanning.

backend/src/__tests__/database-metrics.test.ts

@@ -170,12 +170,12 @@ describe('DatabaseService - cleanupOldNotifications', () => {
     const oldTimestamp = Date.now() - 60 * 24 * 60 * 60 * 1000; // 60 days ago
     const recentTimestamp = Date.now() - 1 * 24 * 60 * 60 * 1000; // 1 day ago
 
-    db.addNotificationHistory({ level: 'info', message: 'old notification', timestamp: oldTimestamp });
-    db.addNotificationHistory({ level: 'info', message: 'recent notification', timestamp: recentTimestamp });
+    db.addNotificationHistory(0, { level: 'info', message: 'old notification', timestamp: oldTimestamp });
+    db.addNotificationHistory(0, { level: 'info', message: 'recent notification', timestamp: recentTimestamp });
 
     db.cleanupOldNotifications(30);
 
-    const history = db.getNotificationHistory(200);
+    const history = db.getNotificationHistory(0, 200);
     const old = history.find((n: any) => n.message === 'old notification');
     const recent = history.find((n: any) => n.message === 'recent notification');
     expect(old).toBeUndefined();
@@ -223,7 +223,7 @@ describe('DatabaseService - notification history cap', () => {
   it('auto-prunes to 100 entries when adding notifications', () => {
     // Insert 105 notifications
     for (let i = 0; i < 105; i++) {
-      db.addNotificationHistory({
+      db.addNotificationHistory(0, {
         level: 'info',
         message: `cap-test-${i}`,
         timestamp: Date.now() + i,
@@ -231,23 +231,23 @@ describe('DatabaseService - notification history cap', () => {
     }
 
     // The table should have at most 100 rows
-    const all = db.getNotificationHistory(200);
+    const all = db.getNotificationHistory(0, 200);
     expect(all.length).toBeLessThanOrEqual(100);
   });
 
   it('keeps the most recent entries after pruning', () => {
     // Clear all first
-    db.deleteAllNotifications();
+    db.deleteAllNotifications(0);
 
     for (let i = 0; i < 105; i++) {
-      db.addNotificationHistory({
+      db.addNotificationHistory(0, {
         level: 'info',
         message: `order-test-${i}`,
         timestamp: Date.now() + i * 10,
       });
     }
 
-    const all = db.getNotificationHistory(200);
+    const all = db.getNotificationHistory(0, 200);
     // The newest entries should survive (ordered DESC by timestamp)
     expect(all[0].message).toContain('order-test-');
     // The oldest entries (0-4) should have been pruned

backend/src/__tests__/image-update-service.test.ts

@@ -81,6 +81,7 @@ vi.mock('../services/NodeRegistry', () => ({
   NodeRegistry: {
     getInstance: () => ({
       getComposeDir: () => '/tmp/compose',
+      getDefaultNodeId: () => 1,
     }),
   },
 }));
@@ -436,7 +437,7 @@ services:
 
     await (service as any).checkNode(1, 'local', fakeDb());
 
-    expect(mockAddNotificationHistory).toHaveBeenCalledWith(expect.objectContaining({
+    expect(mockAddNotificationHistory).toHaveBeenCalledWith(1, expect.objectContaining({
       level: 'error',
       message: expect.stringContaining('webhook timeout'),
     }));

backend/src/__tests__/notification-routing.test.ts

@@ -35,6 +35,17 @@ vi.mock('../services/DatabaseService', () => ({
   },
 }));
 
+// NodeRegistry is consulted to resolve this instance's default node id so
+// internal dispatch writes land on the same key the middleware sets for user
+// requests. Mock it to a fixed id so assertions are deterministic.
+vi.mock('../services/NodeRegistry', () => ({
+  NodeRegistry: {
+    getInstance: () => ({
+      getDefaultNodeId: () => 1,
+    }),
+  },
+}));
+
 // Spy on global fetch for webhook dispatch verification
 const mockFetch = vi.fn().mockResolvedValue({ ok: true });
 vi.stubGlobal('fetch', mockFetch);
@@ -210,7 +221,7 @@ describe('NotificationService - routing logic', () => {
 
     await svc.dispatchAlert('info', 'Should be logged');
 
-    expect(mockAddNotificationHistory).toHaveBeenCalledWith({
+    expect(mockAddNotificationHistory).toHaveBeenCalledWith(1, {
       level: 'info',
       message: 'Should be logged',
       timestamp: expect.any(Number),
@@ -225,7 +236,7 @@ describe('NotificationService - routing logic', () => {
 
     await svc.dispatchAlert('warning', 'Restarted', 'my-app', 'my-app-web-1');
 
-    expect(mockAddNotificationHistory).toHaveBeenCalledWith({
+    expect(mockAddNotificationHistory).toHaveBeenCalledWith(1, {
       level: 'warning',
       message: 'Restarted',
       timestamp: expect.any(Number),

backend/src/index.ts

@@ -5572,7 +5572,8 @@ function validateHttpsUrl(value: unknown): string | null {
 
 app.get('/api/agents', authMiddleware, async (req: Request, res: Response) => {
   try {
-    const agents = DatabaseService.getInstance().getAgents();
+    const nodeId = req.nodeId ?? 0;
+    const agents = DatabaseService.getInstance().getAgents(nodeId);
     res.json(agents);
   } catch (error) {
     console.error('Failed to fetch agents:', error);
@@ -5594,7 +5595,8 @@ app.post('/api/agents', authMiddleware, async (req: Request, res: Response) => {
       res.status(400).json({ error: 'enabled must be a boolean' });
       return;
     }
-    DatabaseService.getInstance().upsertAgent({ type, url, enabled });
+    const nodeId = req.nodeId ?? 0;
+    DatabaseService.getInstance().upsertAgent(nodeId, { type, url, enabled });
     console.log(`[Agents] Agent ${type} updated`);
     if (isDebugEnabled()) console.log(`[Agents:diag] Agent ${type} upsert: enabled=${enabled}`);
     res.json({ success: true });
@@ -5846,7 +5848,8 @@ app.get('/api/auto-heal/policies/:id/history', authMiddleware, (req: Request, re
 
 app.get('/api/notifications', authMiddleware, async (req: Request, res: Response) => {
   try {
-    const history = DatabaseService.getInstance().getNotificationHistory();
+    const nodeId = req.nodeId ?? 0;
+    const history = DatabaseService.getInstance().getNotificationHistory(nodeId);
     res.json(history);
   } catch (error) {
     res.status(500).json({ error: 'Failed to fetch notifications' });
@@ -5855,7 +5858,8 @@ app.get('/api/notifications', authMiddleware, async (req: Request, res: Response
 
 app.post('/api/notifications/read', authMiddleware, async (req: Request, res: Response) => {
   try {
-    DatabaseService.getInstance().markAllNotificationsRead();
+    const nodeId = req.nodeId ?? 0;
+    DatabaseService.getInstance().markAllNotificationsRead(nodeId);
     res.json({ success: true });
   } catch (error) {
     res.status(500).json({ error: 'Failed to mark notifications read' });
@@ -5866,7 +5870,8 @@ app.delete('/api/notifications/:id', authMiddleware, async (req: Request, res: R
   try {
     const id = parseInt(req.params.id as string, 10);
     if (isNaN(id)) { res.status(400).json({ error: 'Invalid notification ID' }); return; }
-    DatabaseService.getInstance().deleteNotification(id);
+    const nodeId = req.nodeId ?? 0;
+    DatabaseService.getInstance().deleteNotification(nodeId, id);
     res.json({ success: true });
   } catch (error) {
     res.status(500).json({ error: 'Failed to delete notification' });
@@ -5875,7 +5880,8 @@ app.delete('/api/notifications/:id', authMiddleware, async (req: Request, res: R
 
 app.delete('/api/notifications', authMiddleware, async (req: Request, res: Response) => {
   try {
-    DatabaseService.getInstance().deleteAllNotifications();
+    const nodeId = req.nodeId ?? 0;
+    DatabaseService.getInstance().deleteAllNotifications(nodeId);
     res.json({ success: true });
   } catch (error) {
     res.status(500).json({ error: 'Failed to clear notifications' });

backend/src/services/DatabaseService.ts

@@ -449,6 +449,7 @@ export class DatabaseService {
         this.migrateNotificationHistoryContext();
         this.migrateScanPolicyFleetColumns();
         this.migrateSecretMisconfigColumns();
+        this.migrateAgentsAndNotificationsNodeId();
     }
 
     public static getInstance(): DatabaseService {
@@ -466,6 +467,7 @@ export class DatabaseService {
         this.db.exec(`
       CREATE TABLE IF NOT EXISTS agents (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
+        node_id INTEGER NOT NULL DEFAULT 0,
         type TEXT NOT NULL,
         url TEXT NOT NULL,
         enabled INTEGER DEFAULT 0
@@ -489,6 +491,7 @@ export class DatabaseService {
 
       CREATE TABLE IF NOT EXISTS notification_history (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
+        node_id INTEGER NOT NULL DEFAULT 0,
         level TEXT NOT NULL,
         message TEXT NOT NULL,
         timestamp INTEGER NOT NULL,
@@ -1115,32 +1118,59 @@ export class DatabaseService {
         tryAddColumn('vulnerability_scans', 'scanners_used', "TEXT NOT NULL DEFAULT 'vuln'");
     }
 
+    private migrateAgentsAndNotificationsNodeId(): void {
+        const tryAddColumn = (table: string, col: string, def: string) => {
+            try {
+                this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run();
+            } catch {
+                /* column already present */
+            }
+        };
+        tryAddColumn('agents', 'node_id', 'INTEGER NOT NULL DEFAULT 0');
+        tryAddColumn('notification_history', 'node_id', 'INTEGER NOT NULL DEFAULT 0');
+        const tryIndex = (sql: string, label: string) => {
+            try {
+                this.db.prepare(sql).run();
+            } catch (e) {
+                console.warn(`[DatabaseService] Could not create ${label}:`, (e as Error).message);
+            }
+        };
+        tryIndex(
+            'CREATE UNIQUE INDEX IF NOT EXISTS idx_agents_node_type ON agents(node_id, type)',
+            'agents(node_id, type) unique index',
+        );
+        tryIndex(
+            'CREATE INDEX IF NOT EXISTS idx_notif_history_node_timestamp ON notification_history(node_id, timestamp DESC)',
+            'notification_history(node_id, timestamp) index',
+        );
+    }
+
     // --- Agents ---
 
-    public getAgents(): Agent[] {
-        const stmt = this.db.prepare('SELECT * FROM agents');
-        return stmt.all().map((row: any) => ({
+    public getAgents(nodeId: number): Agent[] {
+        const stmt = this.db.prepare('SELECT * FROM agents WHERE node_id = ?');
+        return stmt.all(nodeId).map((row: any) => ({
             ...row,
             enabled: row.enabled === 1
         }));
     }
 
-    public getEnabledAgents(): Agent[] {
-        const stmt = this.db.prepare('SELECT * FROM agents WHERE enabled = 1');
-        return stmt.all().map((row: any) => ({
+    public getEnabledAgents(nodeId: number): Agent[] {
+        const stmt = this.db.prepare('SELECT * FROM agents WHERE node_id = ? AND enabled = 1');
+        return stmt.all(nodeId).map((row: any) => ({
             ...row,
             enabled: row.enabled === 1
         }));
     }
 
-    public upsertAgent(agent: Agent): void {
-        const existing = this.db.prepare('SELECT id FROM agents WHERE type = ?').get(agent.type) as any;
+    public upsertAgent(nodeId: number, agent: Agent): void {
+        const existing = this.db.prepare('SELECT id FROM agents WHERE node_id = ? AND type = ?').get(nodeId, agent.type) as any;
         if (existing) {
-            const stmt = this.db.prepare('UPDATE agents SET url = ?, enabled = ? WHERE type = ?');
-            stmt.run(agent.url, agent.enabled ? 1 : 0, agent.type);
+            const stmt = this.db.prepare('UPDATE agents SET url = ?, enabled = ? WHERE node_id = ? AND type = ?');
+            stmt.run(agent.url, agent.enabled ? 1 : 0, nodeId, agent.type);
         } else {
-            const stmt = this.db.prepare('INSERT INTO agents (type, url, enabled) VALUES (?, ?, ?)');
-            stmt.run(agent.type, agent.url, agent.enabled ? 1 : 0);
+            const stmt = this.db.prepare('INSERT INTO agents (node_id, type, url, enabled) VALUES (?, ?, ?, ?)');
+            stmt.run(nodeId, agent.type, agent.url, agent.enabled ? 1 : 0);
         }
     }
 
@@ -1370,9 +1400,9 @@ export class DatabaseService {
 
     // --- Notification History ---
 
-    public getNotificationHistory(limit = 50): NotificationHistory[] {
-        const stmt = this.db.prepare('SELECT * FROM notification_history ORDER BY timestamp DESC LIMIT ?');
-        return stmt.all(limit).map((row: any) => ({
+    public getNotificationHistory(nodeId: number, limit = 50): NotificationHistory[] {
+        const stmt = this.db.prepare('SELECT * FROM notification_history WHERE node_id = ? ORDER BY timestamp DESC LIMIT ?');
+        return stmt.all(nodeId, limit).map((row: any) => ({
             ...row,
             is_read: row.is_read === 1,
             stack_name: row.stack_name ?? undefined,
@@ -1380,11 +1410,12 @@ export class DatabaseService {
         }));
     }
 
-    public addNotificationHistory(notification: Omit<NotificationHistory, 'id' | 'is_read'>): NotificationHistory {
+    public addNotificationHistory(nodeId: number, notification: Omit<NotificationHistory, 'id' | 'is_read'>): NotificationHistory {
         const stmt = this.db.prepare(
-            'INSERT INTO notification_history (level, message, timestamp, is_read, stack_name, container_name) VALUES (?, ?, ?, 0, ?, ?)'
+            'INSERT INTO notification_history (node_id, level, message, timestamp, is_read, stack_name, container_name) VALUES (?, ?, ?, ?, 0, ?, ?)'
         );
         const result = stmt.run(
+            nodeId,
             notification.level,
             notification.message,
             notification.timestamp,
@@ -1392,12 +1423,12 @@ export class DatabaseService {
             notification.container_name ?? null
         );
 
-        this.db.exec(`
+        this.db.prepare(`
       DELETE FROM notification_history
-      WHERE id NOT IN (
-        SELECT id FROM notification_history ORDER BY timestamp DESC LIMIT 100
+      WHERE node_id = ? AND id NOT IN (
+        SELECT id FROM notification_history WHERE node_id = ? ORDER BY timestamp DESC LIMIT 100
       )
-    `);
+    `).run(nodeId, nodeId);
 
         return {
             id: result.lastInsertRowid as number,
@@ -1410,19 +1441,19 @@ export class DatabaseService {
         };
     }
 
-    public markAllNotificationsRead(): void {
-        const stmt = this.db.prepare('UPDATE notification_history SET is_read = 1');
-        stmt.run();
+    public markAllNotificationsRead(nodeId: number): void {
+        const stmt = this.db.prepare('UPDATE notification_history SET is_read = 1 WHERE node_id = ?');
+        stmt.run(nodeId);
     }
 
-    public deleteNotification(id: number): void {
-        const stmt = this.db.prepare('DELETE FROM notification_history WHERE id = ?');
-        stmt.run(id);
+    public deleteNotification(nodeId: number, id: number): void {
+        const stmt = this.db.prepare('DELETE FROM notification_history WHERE node_id = ? AND id = ?');
+        stmt.run(nodeId, id);
     }
 
-    public deleteAllNotifications(): void {
-        const stmt = this.db.prepare('DELETE FROM notification_history');
-        stmt.run();
+    public deleteAllNotifications(nodeId: number): void {
+        const stmt = this.db.prepare('DELETE FROM notification_history WHERE node_id = ?');
+        stmt.run(nodeId);
     }
 
     public updateNotificationDispatchError(id: number, error: string): void {

backend/src/services/ImageUpdateService.ts

@@ -281,8 +281,10 @@ export class ImageUpdateService {
                 } catch (e) {
                     console.error(`[ImageUpdateService] Failed to dispatch update notification for "${stackName}":`, e);
                     // Direct DB write to avoid recursing through dispatchAlert if it is what failed.
+                    // Key on the local default: the iterated `nodeId` may be a remote's id in the
+                    // control plane's DB, and the UI never queries that row (it proxies instead).
                     try {
-                        db.addNotificationHistory({
+                        db.addNotificationHistory(NodeRegistry.getInstance().getDefaultNodeId(), {
                             level: 'error',
                             message: `[Node: ${nodeName}] Failed to notify about image updates for stack "${stackName}": ${getErrorMessage(e, String(e))}`,
                             timestamp: Date.now(),

backend/src/services/NotificationService.ts

@@ -1,4 +1,5 @@
 import { DatabaseService, NotificationHistory } from './DatabaseService';
+import { NodeRegistry } from './NodeRegistry';
 import { isDebugEnabled } from '../utils/debug';
 import { getErrorMessage } from '../utils/errors';
 
@@ -45,8 +46,11 @@ export class NotificationService {
         stackName?: string,
         containerName?: string,
     ) {
-        // 1. Log to history and get the full inserted record (with id)
-        const notification = this.dbService.addNotificationHistory({
+        // Internal writes use the middleware default so they share a row key
+        // with user-initiated requests; otherwise the UI and monitors split
+        // between different node_id buckets.
+        const localNodeId = NodeRegistry.getInstance().getDefaultNodeId();
+        const notification = this.dbService.addNotificationHistory(localNodeId, {
             level,
             message,
             timestamp: Date.now(),
@@ -84,8 +88,8 @@ export class NotificationService {
             }
         }
 
-        // 4. Fall back to global agents
-        const agents = this.dbService.getEnabledAgents();
+        // 4. Fall back to this instance's agents (keyed by this instance's default node id).
+        const agents = this.dbService.getEnabledAgents(localNodeId);
         if (agents.length === 0) {
             if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch');
             return;