feat(scheduler): include node identifier in remote proxy error messages (#1609)

* feat(scheduler): include node identifier in remote proxy error messages Remote lifecycle proxy errors from postToRemoteStack, executeUpdateRemote, postToRemoteContainer, and getRemoteContainers now include a stable node label (name and numeric id) so a hub operator reading run history can correlate failures to a node without cross-referencing the task table. The prefix matches the existing containerNotFoundMessage pattern in the same class. Transport-level failures (DNS, ECONNREFUSED, timeout) also receive the node label. A startsWith guard prevents double- prefixing within the same method. The restart fan-out catch relies on the inner postToRemoteStack error for node context so the label appears exactly once. * fix(scheduler): pilot-aware remote proxy no-target error context Use formatNoTargetError for null proxy targets so pilot tunnel disconnects are not misdiagnosed as missing credentials. Standardize the failure prefix to Remote node name/id, share require/rethrow helpers, and cover pilot no-target plus remote container action failures in tests.
2026-07-26 11:49:16 +00:00 · 2026-07-10 13:05:03 -04:00
parent ba2e7bded9
commit 8fd526ba05
2 changed files with 247 additions and 89 deletions
@@ -131,6 +131,17 @@ vi.mock('../services/DockerController', () => ({
  },
 }));

+vi.mock('../services/SelfIdentityService', () => ({
+  default: {
+    getInstance: () => ({
+      initialize: vi.fn().mockResolvedValue(undefined),
+      isOwnContainer: vi.fn().mockReturnValue(false),
+      isOwnImage: vi.fn().mockReturnValue(false),
+      resetForTesting: vi.fn(),
+    }),
+  },
+}));
+
 vi.mock('../services/ComposeService', () => ({
  ComposeService: {
    getInstance: () => ({
@@ -1741,7 +1752,7 @@ describe('SchedulerService - executeUpdateRemote', () => {

    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Internal error') })
+      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): Internal error') })
    );
  });
 });
@@ -1946,7 +1957,7 @@ describe('SchedulerService - lifecycle remote proxy', () => {
    await SchedulerService.getInstance().triggerTask(300);
    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Docker daemon is unreachable') }),
+      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): Docker daemon is unreachable') }),
    );
  });

@@ -1962,13 +1973,13 @@ describe('SchedulerService - lifecycle remote proxy', () => {
    await SchedulerService.getInstance().triggerTask(300);
    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('HTTP 502') }),
+      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): HTTP 502') }),
    );
  });

-  it('records failure when the remote node has no proxy credentials', async () => {
-    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
-    // getProxyTarget defaults to null (no credentials configured).
+  it('records failure when the remote proxy target is unavailable', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online', mode: 'proxy' });
+    // getProxyTarget defaults to null (proxy credentials missing / unreachable).
    const fetchMock = vi.fn();
    vi.stubGlobal('fetch', fetchMock);
    mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
@@ -1976,7 +1987,34 @@ describe('SchedulerService - lifecycle remote proxy', () => {
    expect(fetchMock).not.toHaveBeenCalled();
    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('not configured or missing API credentials') }),
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): Remote node not configured'),
+      }),
+    );
+  });
+
+  it('records pilot-aware no-target messaging when the tunnel is disconnected', async () => {
+    mockGetNode.mockReturnValue({
+      id: 2,
+      name: 'edge-pilot',
+      type: 'remote',
+      status: 'online',
+      mode: 'pilot_agent',
+    });
+    const fetchMock = vi.fn();
+    vi.stubGlobal('fetch', fetchMock);
+    mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
+    await SchedulerService.getInstance().triggerTask(300);
+    expect(fetchMock).not.toHaveBeenCalled();
+    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining(
+          'Remote node "edge-pilot" (id=2): Pilot tunnel to "edge-pilot" is disconnected',
+        ),
+      }),
    );
  });

@@ -1993,11 +2031,91 @@ describe('SchedulerService - lifecycle remote proxy', () => {
    await SchedulerService.getInstance().triggerTask(300);
    // Third service is never reached after the second fails.
    expect(fetchMock).toHaveBeenCalledTimes(2);
+    const call = (mockUpdateScheduledTaskRun as ReturnType<typeof vi.fn>).mock.calls.at(-1)!;
+    const errorMsg: string = call[1].error;
+    // Node context appears exactly once (via postToRemoteStack's inner prefix, not duplicated
+    // by executeRestartRemote's catch wrapper).
+    const nodeLabel = 'Remote node "remote" (id=2)';
+    expect(errorMsg).toContain(nodeLabel);
+    expect(errorMsg.indexOf(nodeLabel)).toBe(errorMsg.lastIndexOf(nodeLabel));
+    expect(errorMsg).toContain('already restarted: api');
+    expect(errorMsg).toContain('worker');
+    expect(errorMsg).toContain('boom');
+  });
+});
+
+// ── Remote container proxy error context ─────────────────────────────
+
+describe('SchedulerService - remote container proxy error context', () => {
+  it('prefixes transport failures with node context', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
+    mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
+    vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('connect ECONNREFUSED')));
+    mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
+    await SchedulerService.getInstance().triggerTask(300);
    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('already restarted: api') }),
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): connect ECONNREFUSED'),
+      }),
    );
  });
+
+  it('prefixes remote container list failures with node context', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
+    mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
+      ok: false,
+      status: 503,
+      json: async () => ({ error: 'Docker daemon is unreachable' }),
+    }));
+    mockGetScheduledTask.mockReturnValue({
+      ...makeLifecycleTask('restart', { node_id: 2 }),
+      target_type: 'container',
+      target_services: null,
+    });
+    await SchedulerService.getInstance().triggerTask(300);
+    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): Docker daemon is unreachable'),
+      }),
+    );
+  });
+
+  it('prefixes remote container action failures with node context', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
+    mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
+    const fetchMock = vi.fn()
+      .mockResolvedValueOnce({
+        ok: true,
+        json: async () => ([{ Id: 'ctrdeadbeef01', Names: ['/watchtower'], State: 'running', Image: 'containrrr/watchtower' }]),
+      })
+      .mockResolvedValueOnce({
+        ok: false,
+        status: 500,
+        json: async () => ({ error: 'container stop failed' }),
+      });
+    vi.stubGlobal('fetch', fetchMock);
+    mockGetScheduledTask.mockReturnValue({
+      ...makeLifecycleTask('auto_stop', { node_id: 2, target_id: 'watchtower' }),
+      target_type: 'container',
+      target_services: null,
+    });
+    await SchedulerService.getInstance().triggerTask(300);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(fetchMock.mock.calls[1][0]).toContain('/api/containers/ctrdeadbeef01/stop');
+    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): container stop failed'),
+      }),
+    );
+  });
+
 });

 // ── Unpaid-tier guard in executeTask ────────────────────────────────────
@@ -12,6 +12,7 @@ import { ImageUpdateService } from './ImageUpdateService';
 import type { ImageCheckResult } from './ImageUpdateService';
 import { isDebugEnabled } from '../utils/debug';
 import { getErrorMessage } from '../utils/errors';
+import { formatNoTargetError } from '../utils/remoteTarget';
 import { sanitizeForLog } from '../utils/safeLog';
 import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, type SnapshotNodeData } from '../utils/snapshot-capture';
 import { NodeRegistry } from './NodeRegistry';
@@ -771,17 +772,14 @@ export class SchedulerService {
     * The remote node runs the image checks and compose update locally.
     */
    private async executeUpdateRemote(nodeId: number, target: string): Promise<string> {
-        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
-        if (!proxyTarget) {
-            throw new Error('Remote node is not configured or missing API credentials');
-        }
-
+        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
        if (isDebugEnabled()) {
            console.log(`[SchedulerService] executeUpdateRemote: node=${nodeId} target=${target}`);
        }
        const startTime = Date.now();
+        try {
            const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
                method: 'POST',
                headers: {
@@ -794,8 +792,7 @@ export class SchedulerService {
            });

            if (!response.ok) {
-            const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
-            throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
+                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
            }

            const body = await response.json() as { result?: string };
@@ -803,6 +800,9 @@ export class SchedulerService {
                console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`);
            }
            return body.result || 'Remote auto-update completed (no details returned).';
+        } catch (err) {
+            this.rethrowRemoteProxyError(nodeId, err);
+        }
    }

    /**
@@ -818,6 +818,46 @@ export class SchedulerService {
        return `Container "${containerName}" not found on node "${nodeName}". It may have been renamed or removed.`;
    }

+    /** Hub target tag used for startsWith rethrow guards (no trailing detail). */
+    private remoteProxyNodeTag(nodeId: number): string {
+        const nodeName = NodeRegistry.getInstance().getNode(nodeId)?.name ?? String(nodeId);
+        return `Remote node "${nodeName}" (id=${nodeId})`;
+    }
+
+    /** Operator-facing remote proxy failure with stable node correlation. */
+    private remoteProxyFailureMessage(nodeId: number, detail: string): string {
+        return `${this.remoteProxyNodeTag(nodeId)}: ${detail}`;
+    }
+
+    private noProxyTargetDetail(nodeId: number): string {
+        const node = NodeRegistry.getInstance().getNode(nodeId);
+        if (!node) return 'Remote node is not configured or missing API credentials';
+        return formatNoTargetError(node);
+    }
+
+    private async remoteResponseDetail(response: Response): Promise<string> {
+        try {
+            const body = await response.json() as { error?: string };
+            return body.error || `Remote node returned ${response.status}`;
+        } catch {
+            return `HTTP ${response.status}`;
+        }
+    }
+
+    private requireRemoteProxyTarget(nodeId: number): { apiUrl: string; apiToken: string } {
+        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
+        if (!proxyTarget) {
+            throw new Error(this.remoteProxyFailureMessage(nodeId, this.noProxyTargetDetail(nodeId)));
+        }
+        return proxyTarget;
+    }
+
+    private rethrowRemoteProxyError(nodeId: number, err: unknown): never {
+        const tag = this.remoteProxyNodeTag(nodeId);
+        if (err instanceof Error && err.message.startsWith(tag)) throw err;
+        throw new Error(this.remoteProxyFailureMessage(nodeId, getErrorMessage(err, 'Remote proxy request failed')));
+    }
+
    private async resolveContainerId(task: ScheduledTask): Promise<{ id: string; name: string }> {
        if (!task.target_id || task.node_id == null) {
            throw new Error('Container operations require target_id and node_id');
@@ -878,12 +918,10 @@ export class SchedulerService {
        State?: string;
        Image?: string;
    }>> {
-        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
-        if (!proxyTarget) {
-            throw new Error('Remote node is not configured or missing API credentials');
-        }
+        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
+        try {
            const response = await fetch(`${baseUrl}/api/containers?all=true`, {
                headers: {
                    'Authorization': `Bearer ${proxyTarget.apiToken}`,
@@ -892,8 +930,7 @@ export class SchedulerService {
                signal: AbortSignal.timeout(60_000),
            });
            if (!response.ok) {
-            const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
-            throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
+                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
            }
            return excludeSelfContainers(await response.json() as Array<{
                Id: string;
@@ -902,6 +939,9 @@ export class SchedulerService {
                Image?: string;
                ImageID?: string;
            }>);
+        } catch (err) {
+            this.rethrowRemoteProxyError(nodeId, err);
+        }
    }

    private async postToRemoteContainer(
@@ -909,12 +949,10 @@ export class SchedulerService {
        containerId: string,
        action: 'start' | 'stop' | 'restart',
    ): Promise<void> {
-        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
-        if (!proxyTarget) {
-            throw new Error('Remote node is not configured or missing API credentials');
-        }
+        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
+        try {
            const response = await fetch(
                `${baseUrl}/api/containers/${encodeURIComponent(containerId)}/${action}`,
                {
@@ -928,21 +966,21 @@ export class SchedulerService {
                },
            );
            if (!response.ok) {
-            const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
-            throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
+                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
+            }
+        } catch (err) {
+            this.rethrowRemoteProxyError(nodeId, err);
        }
    }

    private async postToRemoteStack(nodeId: number, routeSuffix: string): Promise<void> {
-        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
-        if (!proxyTarget) {
-            throw new Error('Remote node is not configured or missing API credentials');
-        }
+        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
        if (isDebugEnabled()) {
            console.log(`[SchedulerService:debug] postToRemoteStack: node=${nodeId} route=${routeSuffix}`);
        }
+        try {
            const response = await fetch(`${baseUrl}/api/stacks/${routeSuffix}`, {
                method: 'POST',
                headers: {
@@ -953,8 +991,10 @@ export class SchedulerService {
                signal: AbortSignal.timeout(300_000),
            });
            if (!response.ok) {
-            const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
-            throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
+                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
+            }
+        } catch (err) {
+            this.rethrowRemoteProxyError(nodeId, err);
        }
    }