fix(auto-update): paid-gate execute route and harden image-check watchdog (#1257)

* fix(auto-update): paid-gate execute route and harden image-check watchdog

Auto-update execution is a paid capability, but POST /api/auto-update/execute
was reachable by any admin regardless of license. Add the paid guard so it
matches the rest of the surface (scheduled-task management and fleet refresh).
The scheduler dispatch to remote nodes still works because the controlling
instance forwards its tier with the request.

Restrict GET /api/image-updates/fleet to admins. The single-node status
endpoint that drives the sidebar update dot stays open to all roles.

Replace the image-check watchdog timer that released the run lock after five
minutes. On a healthy but slow scan it let a manual refresh start a second
concurrent check, duplicating notifications and racing the status writes. The
scan now owns its lock for its full duration, and every Docker socket and
filesystem read is bounded so a wedged daemon or mount cannot stall a scan
forever.

* test(auto-update): assert the debug skip-log branch in the image-check guard

The concurrency-guard test covered the warn branch for a trigger arriving past
the long-run threshold but never exercised the developer-mode debug skip log.
Add a case that enables developer mode and asserts the debug line fires for a
mid-scan trigger under the threshold.

backend/src/__tests__/image-update-service.test.ts

@@ -143,6 +143,25 @@ describe('ImageUpdateService - image ref parsing (via checkImage)', () => {
     expect(result.error).toContain('Failed to inspect local image');
   });
 
+  it('bounds a hung local inspect instead of hanging the scan', async () => {
+    // A wedged Docker socket must not stall the check forever: withTimeout
+    // rejects the inspect, the existing catch turns it into an error result.
+    const docker = {
+      getDocker: () => ({
+        getImage: () => ({ inspect: vi.fn().mockImplementation(() => new Promise(() => { /* never resolves */ })) }),
+      }),
+    } as any;
+    const orig = (ImageUpdateService as any).SOCKET_TIMEOUT_MS;
+    (ImageUpdateService as any).SOCKET_TIMEOUT_MS = 20;
+    try {
+      const result = await service.checkImage(docker, 'nginx:latest');
+      expect(result.hasUpdate).toBe(false);
+      expect(result.error).toContain('Failed to inspect local image');
+    } finally {
+      (ImageUpdateService as any).SOCKET_TIMEOUT_MS = orig;
+    }
+  });
+
   it('returns { hasUpdate: false } when no RepoDigests match', async () => {
     // Empty RepoDigests means locally built image
     const docker = makeMockDocker([]);
@@ -516,21 +535,20 @@ services:
   });
 });
 
-// ── check() timeout ─────────────────────────────────────────────────────
+// ── check() concurrency guard ───────────────────────────────────────────
 
-describe('ImageUpdateService - check() timeout', () => {
+describe('ImageUpdateService - check() concurrency guard', () => {
 
   beforeEach(() => {
     vi.clearAllMocks();
     (ImageUpdateService as any).instance = undefined;
   });
 
-  it('releases isRunning lock after CHECK_TIMEOUT_MS', async () => {
-    // Override the module-level DatabaseService mock to return a node
+  async function stubDbWithLocalNode(developerMode: '0' | '1' = '0') {
     const dbModule = await import('../services/DatabaseService');
-    const origGetInstance = dbModule.DatabaseService.getInstance;
+    const orig = dbModule.DatabaseService.getInstance;
     dbModule.DatabaseService.getInstance = (() => ({
-      getGlobalSettings: () => ({ developer_mode: '0' }),
+      getGlobalSettings: () => ({ developer_mode: developerMode }),
       getNodes: () => [{ type: 'local', id: 1, name: 'local', mode: 'proxy', compose_dir: '/tmp/compose', is_default: true, status: 'online', created_at: 1 }],
       upsertStackUpdateStatus: mockUpsertStackUpdateStatus,
       getStackUpdateStatus: mockGetStackUpdateStatus,
@@ -539,33 +557,120 @@ describe('ImageUpdateService - check() timeout', () => {
       setSystemState: mockSetSystemState,
       addNotificationHistory: mockAddNotificationHistory,
     })) as unknown as typeof dbModule.DatabaseService.getInstance;
+    return () => { dbModule.DatabaseService.getInstance = orig; };
+  }
 
+  it('does not start a second check body while one is in flight', async () => {
+    const restoreDb = await stubDbWithLocalNode();
     const service = ImageUpdateService.getInstance();
-    // Make checkNode hang indefinitely so the timeout fires
-    (service as any).checkNode = vi.fn().mockImplementation(() =>
+    // checkNode never resolves: simulate a scan that overruns / a wedged socket.
+    const checkNodeMock = vi.fn().mockImplementation(() =>
       new Promise(() => { /* never resolves */ })
     );
+    (service as any).checkNode = checkNodeMock;
 
-    // Override timeout to 100ms for fast test
-    const orig = (ImageUpdateService as any).CHECK_TIMEOUT_MS;
-    (ImageUpdateService as any).CHECK_TIMEOUT_MS = 100;
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+    const skipWarn = /running for \d+ minute/;
 
-    // Don't await; this will hang intentionally
-    const checkPromise = (service as any).check();
-
-    // Let microtasks flush so check() enters its body
+    const first = (service as any).check();
     await new Promise(r => setTimeout(r, 10));
     expect(service.isChecking()).toBe(true);
+    expect(checkNodeMock).toHaveBeenCalledTimes(1);
 
-    // Wait for timeout to fire
-    await new Promise(r => setTimeout(r, 200));
+    // A concurrent trigger (e.g. a manual refresh) under the long-run threshold
+    // must be a silent no-op: no second body, no warning.
+    await (service as any).check();
+    expect(checkNodeMock).toHaveBeenCalledTimes(1);
+    expect(service.isChecking()).toBe(true);
+    expect(warnSpy.mock.calls.some(c => skipWarn.test(String(c[0])))).toBe(false);
 
-    expect(service.isChecking()).toBe(false);
+    // Past the long-run threshold the trigger warns (operator signal) but still
+    // must not spawn a concurrent body.
+    const orig = (ImageUpdateService as any).CHECK_TIMEOUT_MS;
+    (ImageUpdateService as any).CHECK_TIMEOUT_MS = 1;
+    await new Promise(r => setTimeout(r, 5));
+    await (service as any).check();
+    expect(checkNodeMock).toHaveBeenCalledTimes(1);
+    expect(service.isChecking()).toBe(true);
+    expect(warnSpy.mock.calls.some(c => skipWarn.test(String(c[0])))).toBe(true);
 
-    // Cleanup
     (ImageUpdateService as any).CHECK_TIMEOUT_MS = orig;
-    dbModule.DatabaseService.getInstance = origGetInstance;
-    checkPromise.catch(() => {});
+    warnSpy.mockRestore();
+    restoreDb();
+    first.catch(() => {});
+  });
+
+  it('treats a manual refresh during an in-flight check as a no-op', async () => {
+    const restoreDb = await stubDbWithLocalNode();
+    const service = ImageUpdateService.getInstance();
+    const checkNodeMock = vi.fn().mockImplementation(() =>
+      new Promise(() => { /* never resolves */ })
+    );
+    (service as any).checkNode = checkNodeMock;
+
+    const first = (service as any).check();
+    await new Promise(r => setTimeout(r, 10));
+    expect(checkNodeMock).toHaveBeenCalledTimes(1);
+
+    // This is the exact regression the guard replaces: a manual refresh firing
+    // while a scan is in flight. It reports it fired (the cooldown is clear) but
+    // the in-check guard prevents a second concurrent scan body.
+    const triggered = service.triggerManualRefresh();
+    await new Promise(r => setTimeout(r, 10));
+    expect(triggered).toBe(true);
+    expect(checkNodeMock).toHaveBeenCalledTimes(1);
+    expect(service.isChecking()).toBe(true);
+
+    restoreDb();
+    first.catch(() => {});
+  });
+
+  it('logs a debug skip line for a mid-scan trigger when developer mode is on', async () => {
+    const restoreDb = await stubDbWithLocalNode('1');
+    // isDebugEnabled short-circuits to false under NODE_ENV=test unless DATA_DIR
+    // is set; set it so the developer_mode flag is actually consulted.
+    const prevDataDir = process.env.DATA_DIR;
+    process.env.DATA_DIR = prevDataDir ?? '/tmp/image-update-debug-test';
+    const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+    try {
+      const service = ImageUpdateService.getInstance();
+      const checkNodeMock = vi.fn().mockImplementation(() =>
+        new Promise(() => { /* never resolves */ })
+      );
+      (service as any).checkNode = checkNodeMock;
+
+      const first = (service as any).check();
+      await new Promise(r => setTimeout(r, 10));
+
+      // Under the long-run threshold with developer mode on, the skipped trigger
+      // takes the debug branch rather than the WARN branch.
+      await (service as any).check();
+      expect(checkNodeMock).toHaveBeenCalledTimes(1);
+      expect(logSpy.mock.calls.some(c => /Check already in progress; skipping/.test(String(c[0])))).toBe(true);
+
+      first.catch(() => {});
+    } finally {
+      logSpy.mockRestore();
+      if (prevDataDir === undefined) delete process.env.DATA_DIR; else process.env.DATA_DIR = prevDataDir;
+      restoreDb();
+    }
+  });
+
+  it('releases the lock when a check finishes and allows the next run', async () => {
+    const restoreDb = await stubDbWithLocalNode();
+    const service = ImageUpdateService.getInstance();
+    const checkNodeMock = vi.fn().mockResolvedValue(undefined);
+    (service as any).checkNode = checkNodeMock;
+
+    await (service as any).check();
+    expect(service.isChecking()).toBe(false);
+    expect(checkNodeMock).toHaveBeenCalledTimes(1);
+
+    // A fresh trigger after completion runs a new body.
+    await (service as any).check();
+    expect(checkNodeMock).toHaveBeenCalledTimes(2);
+
+    restoreDb();
   });
 });
 

backend/src/__tests__/image-updates-routes.test.ts

@@ -6,7 +6,9 @@
 import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
 import request from 'supertest';
 import bcrypt from 'bcrypt';
-import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
+import jwt from 'jsonwebtoken';
+import { setupTestDb, cleanupTestDb, loginAsTestAdmin, TEST_JWT_SECRET } from './helpers/setupTestDb';
+import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from '../services/license-headers';
 
 let tmpDir: string;
 let app: import('express').Express;
@@ -87,6 +89,13 @@ describe('GET /api/image-updates/fleet', () => {
     expect(res.status).toBe(401);
   });
 
+  it('rejects non-admin users with 403', async () => {
+    // The cross-node aggregation is part of the admin-only readiness surface;
+    // the single-node GET / endpoint stays open for the sidebar update dot.
+    const res = await request(app).get('/api/image-updates/fleet').set('Cookie', viewerCookie);
+    expect(res.status).toBe(403);
+  });
+
   it('returns the fleet-wide aggregation map', async () => {
     const res = await request(app).get('/api/image-updates/fleet').set('Cookie', adminCookie);
     expect(res.status).toBe(200);
@@ -156,6 +165,69 @@ describe('POST /api/auto-update/execute', () => {
     expect(res.status).toBe(403);
   });
 
+  it('rejects a Community-tier admin with 403 PAID_REQUIRED', async () => {
+    // Auto-update execution is a paid capability; an admin on a Community
+    // license must not be able to drive it directly through the API.
+    const { LicenseService } = await import('../services/LicenseService');
+    const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
+    try {
+      const res = await request(app)
+        .post('/api/auto-update/execute')
+        .set('Cookie', adminCookie)
+        .send({ target: '*' });
+      expect(res.status).toBe(403);
+      expect(res.body.code).toBe('PAID_REQUIRED');
+    } finally {
+      tierSpy.mockRestore();
+      vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+    }
+  });
+
+  it('honors a paid proxy tier header from a node_proxy caller on a Community runtime', async () => {
+    // The scheduler dispatches to a remote's /execute with a node_proxy Bearer
+    // token and the controlling instance's tier header. A Community-licensed
+    // remote runtime must still run the update because the trusted header, not
+    // the local license, decides entitlement.
+    const { LicenseService } = await import('../services/LicenseService');
+    const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
+    const proxyToken = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '5m' });
+    try {
+      const res = await request(app)
+        .post('/api/auto-update/execute')
+        .set('Authorization', `Bearer ${proxyToken}`)
+        .set(PROXY_TIER_HEADER, 'paid')
+        .set(PROXY_VARIANT_HEADER, 'admiral')
+        .send({ target: '*' });
+      // Gate passes: no stacks on the fresh instance, so the handler returns
+      // the "no stacks found" summary rather than a 403.
+      expect(res.status).toBe(200);
+      expect(typeof res.body.result).toBe('string');
+    } finally {
+      tierSpy.mockRestore();
+      vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+    }
+  });
+
+  it('rejects a node_proxy caller whose tier header is community with 403', async () => {
+    // The trusted header, not the local license, decides entitlement: a paid
+    // local runtime must still 403 when the controlling instance is Community.
+    const { LicenseService } = await import('../services/LicenseService');
+    const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+    const proxyToken = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '5m' });
+    try {
+      const res = await request(app)
+        .post('/api/auto-update/execute')
+        .set('Authorization', `Bearer ${proxyToken}`)
+        .set(PROXY_TIER_HEADER, 'community')
+        .send({ target: '*' });
+      expect(res.status).toBe(403);
+      expect(res.body.code).toBe('PAID_REQUIRED');
+    } finally {
+      tierSpy.mockRestore();
+      vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
+    }
+  });
+
   it('rejects missing target with 400', async () => {
     const res = await request(app)
       .post('/api/auto-update/execute')