feat(scheduler): include node identifier in remote proxy error messages (#1609)

* feat(scheduler): include node identifier in remote proxy error messages

Remote lifecycle proxy errors from postToRemoteStack,
executeUpdateRemote, postToRemoteContainer, and getRemoteContainers
now include a stable node label (name and numeric id) so a hub
operator reading run history can correlate failures to a node without
cross-referencing the task table.

The prefix matches the existing containerNotFoundMessage pattern in
the same class. Transport-level failures (DNS, ECONNREFUSED, timeout)
also receive the node label. A startsWith guard prevents double-
prefixing within the same method. The restart fan-out catch relies
on the inner postToRemoteStack error for node context so the label
appears exactly once.

* fix(scheduler): pilot-aware remote proxy no-target error context

Use formatNoTargetError for null proxy targets so pilot tunnel disconnects
are not misdiagnosed as missing credentials. Standardize the failure prefix
to Remote node name/id, share require/rethrow helpers, and cover pilot
no-target plus remote container action failures in tests.

backend/src/__tests__/scheduler-service.test.ts

@@ -131,6 +131,17 @@ vi.mock('../services/DockerController', () => ({
   },
 }));
 
+vi.mock('../services/SelfIdentityService', () => ({
+  default: {
+    getInstance: () => ({
+      initialize: vi.fn().mockResolvedValue(undefined),
+      isOwnContainer: vi.fn().mockReturnValue(false),
+      isOwnImage: vi.fn().mockReturnValue(false),
+      resetForTesting: vi.fn(),
+    }),
+  },
+}));
+
 vi.mock('../services/ComposeService', () => ({
   ComposeService: {
     getInstance: () => ({
@@ -1741,7 +1752,7 @@ describe('SchedulerService - executeUpdateRemote', () => {
 
     expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
       1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Internal error') })
+      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): Internal error') })
     );
   });
 });
@@ -1946,7 +1957,7 @@ describe('SchedulerService - lifecycle remote proxy', () => {
     await SchedulerService.getInstance().triggerTask(300);
     expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
       1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Docker daemon is unreachable') }),
+      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): Docker daemon is unreachable') }),
     );
   });
 
@@ -1962,13 +1973,13 @@ describe('SchedulerService - lifecycle remote proxy', () => {
     await SchedulerService.getInstance().triggerTask(300);
     expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
       1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('HTTP 502') }),
+      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): HTTP 502') }),
     );
   });
 
-  it('records failure when the remote node has no proxy credentials', async () => {
-    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
-    // getProxyTarget defaults to null (no credentials configured).
+  it('records failure when the remote proxy target is unavailable', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online', mode: 'proxy' });
+    // getProxyTarget defaults to null (proxy credentials missing / unreachable).
     const fetchMock = vi.fn();
     vi.stubGlobal('fetch', fetchMock);
     mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
@@ -1976,7 +1987,34 @@ describe('SchedulerService - lifecycle remote proxy', () => {
     expect(fetchMock).not.toHaveBeenCalled();
     expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
       1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('not configured or missing API credentials') }),
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): Remote node not configured'),
+      }),
+    );
+  });
+
+  it('records pilot-aware no-target messaging when the tunnel is disconnected', async () => {
+    mockGetNode.mockReturnValue({
+      id: 2,
+      name: 'edge-pilot',
+      type: 'remote',
+      status: 'online',
+      mode: 'pilot_agent',
+    });
+    const fetchMock = vi.fn();
+    vi.stubGlobal('fetch', fetchMock);
+    mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
+    await SchedulerService.getInstance().triggerTask(300);
+    expect(fetchMock).not.toHaveBeenCalled();
+    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining(
+          'Remote node "edge-pilot" (id=2): Pilot tunnel to "edge-pilot" is disconnected',
+        ),
+      }),
     );
   });
 
@@ -1993,11 +2031,91 @@ describe('SchedulerService - lifecycle remote proxy', () => {
     await SchedulerService.getInstance().triggerTask(300);
     // Third service is never reached after the second fails.
     expect(fetchMock).toHaveBeenCalledTimes(2);
+    const call = (mockUpdateScheduledTaskRun as ReturnType<typeof vi.fn>).mock.calls.at(-1)!;
+    const errorMsg: string = call[1].error;
+    // Node context appears exactly once (via postToRemoteStack's inner prefix, not duplicated
+    // by executeRestartRemote's catch wrapper).
+    const nodeLabel = 'Remote node "remote" (id=2)';
+    expect(errorMsg).toContain(nodeLabel);
+    expect(errorMsg.indexOf(nodeLabel)).toBe(errorMsg.lastIndexOf(nodeLabel));
+    expect(errorMsg).toContain('already restarted: api');
+    expect(errorMsg).toContain('worker');
+    expect(errorMsg).toContain('boom');
+  });
+});
+
+// ── Remote container proxy error context ─────────────────────────────
+
+describe('SchedulerService - remote container proxy error context', () => {
+  it('prefixes transport failures with node context', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
+    mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
+    vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('connect ECONNREFUSED')));
+    mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
+    await SchedulerService.getInstance().triggerTask(300);
     expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
       1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('already restarted: api') }),
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): connect ECONNREFUSED'),
+      }),
     );
   });
+
+  it('prefixes remote container list failures with node context', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
+    mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
+      ok: false,
+      status: 503,
+      json: async () => ({ error: 'Docker daemon is unreachable' }),
+    }));
+    mockGetScheduledTask.mockReturnValue({
+      ...makeLifecycleTask('restart', { node_id: 2 }),
+      target_type: 'container',
+      target_services: null,
+    });
+    await SchedulerService.getInstance().triggerTask(300);
+    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): Docker daemon is unreachable'),
+      }),
+    );
+  });
+
+  it('prefixes remote container action failures with node context', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
+    mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
+    const fetchMock = vi.fn()
+      .mockResolvedValueOnce({
+        ok: true,
+        json: async () => ([{ Id: 'ctrdeadbeef01', Names: ['/watchtower'], State: 'running', Image: 'containrrr/watchtower' }]),
+      })
+      .mockResolvedValueOnce({
+        ok: false,
+        status: 500,
+        json: async () => ({ error: 'container stop failed' }),
+      });
+    vi.stubGlobal('fetch', fetchMock);
+    mockGetScheduledTask.mockReturnValue({
+      ...makeLifecycleTask('auto_stop', { node_id: 2, target_id: 'watchtower' }),
+      target_type: 'container',
+      target_services: null,
+    });
+    await SchedulerService.getInstance().triggerTask(300);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(fetchMock.mock.calls[1][0]).toContain('/api/containers/ctrdeadbeef01/stop');
+    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): container stop failed'),
+      }),
+    );
+  });
+
 });
 
 // ── Unpaid-tier guard in executeTask ────────────────────────────────────

backend/src/services/SchedulerService.ts

@@ -12,6 +12,7 @@ import { ImageUpdateService } from './ImageUpdateService';
 import type { ImageCheckResult } from './ImageUpdateService';
 import { isDebugEnabled } from '../utils/debug';
 import { getErrorMessage } from '../utils/errors';
+import { formatNoTargetError } from '../utils/remoteTarget';
 import { sanitizeForLog } from '../utils/safeLog';
 import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, type SnapshotNodeData } from '../utils/snapshot-capture';
 import { NodeRegistry } from './NodeRegistry';
@@ -771,38 +772,37 @@ export class SchedulerService {
      * The remote node runs the image checks and compose update locally.
      */
     private async executeUpdateRemote(nodeId: number, target: string): Promise<string> {
-        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
-        if (!proxyTarget) {
-            throw new Error('Remote node is not configured or missing API credentials');
-        }
-
+        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
         const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
         const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
         if (isDebugEnabled()) {
             console.log(`[SchedulerService] executeUpdateRemote: node=${nodeId} target=${target}`);
         }
         const startTime = Date.now();
-        const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${proxyTarget.apiToken}`,
-                [PROXY_TIER_HEADER]: proxyHeaders.tier,
-            },
-            body: JSON.stringify({ target }),
-            signal: AbortSignal.timeout(300_000), // 5 minute timeout for long updates
-        });
+        try {
+            const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${proxyTarget.apiToken}`,
+                    [PROXY_TIER_HEADER]: proxyHeaders.tier,
+                },
+                body: JSON.stringify({ target }),
+                signal: AbortSignal.timeout(300_000), // 5 minute timeout for long updates
+            });
 
-        if (!response.ok) {
-            const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
-            throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
-        }
+            if (!response.ok) {
+                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
+            }
 
-        const body = await response.json() as { result?: string };
-        if (isDebugEnabled()) {
-            console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`);
+            const body = await response.json() as { result?: string };
+            if (isDebugEnabled()) {
+                console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`);
+            }
+            return body.result || 'Remote auto-update completed (no details returned).';
+        } catch (err) {
+            this.rethrowRemoteProxyError(nodeId, err);
         }
-        return body.result || 'Remote auto-update completed (no details returned).';
     }
 
     /**
@@ -818,6 +818,46 @@ export class SchedulerService {
         return `Container "${containerName}" not found on node "${nodeName}". It may have been renamed or removed.`;
     }
 
+    /** Hub target tag used for startsWith rethrow guards (no trailing detail). */
+    private remoteProxyNodeTag(nodeId: number): string {
+        const nodeName = NodeRegistry.getInstance().getNode(nodeId)?.name ?? String(nodeId);
+        return `Remote node "${nodeName}" (id=${nodeId})`;
+    }
+
+    /** Operator-facing remote proxy failure with stable node correlation. */
+    private remoteProxyFailureMessage(nodeId: number, detail: string): string {
+        return `${this.remoteProxyNodeTag(nodeId)}: ${detail}`;
+    }
+
+    private noProxyTargetDetail(nodeId: number): string {
+        const node = NodeRegistry.getInstance().getNode(nodeId);
+        if (!node) return 'Remote node is not configured or missing API credentials';
+        return formatNoTargetError(node);
+    }
+
+    private async remoteResponseDetail(response: Response): Promise<string> {
+        try {
+            const body = await response.json() as { error?: string };
+            return body.error || `Remote node returned ${response.status}`;
+        } catch {
+            return `HTTP ${response.status}`;
+        }
+    }
+
+    private requireRemoteProxyTarget(nodeId: number): { apiUrl: string; apiToken: string } {
+        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
+        if (!proxyTarget) {
+            throw new Error(this.remoteProxyFailureMessage(nodeId, this.noProxyTargetDetail(nodeId)));
+        }
+        return proxyTarget;
+    }
+
+    private rethrowRemoteProxyError(nodeId: number, err: unknown): never {
+        const tag = this.remoteProxyNodeTag(nodeId);
+        if (err instanceof Error && err.message.startsWith(tag)) throw err;
+        throw new Error(this.remoteProxyFailureMessage(nodeId, getErrorMessage(err, 'Remote proxy request failed')));
+    }
+
     private async resolveContainerId(task: ScheduledTask): Promise<{ id: string; name: string }> {
         if (!task.target_id || task.node_id == null) {
             throw new Error('Container operations require target_id and node_id');
@@ -878,30 +918,30 @@ export class SchedulerService {
         State?: string;
         Image?: string;
     }>> {
-        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
-        if (!proxyTarget) {
-            throw new Error('Remote node is not configured or missing API credentials');
-        }
+        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
         const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
         const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
-        const response = await fetch(`${baseUrl}/api/containers?all=true`, {
-            headers: {
-                'Authorization': `Bearer ${proxyTarget.apiToken}`,
-                [PROXY_TIER_HEADER]: proxyHeaders.tier,
-            },
-            signal: AbortSignal.timeout(60_000),
-        });
-        if (!response.ok) {
-            const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
-            throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
+        try {
+            const response = await fetch(`${baseUrl}/api/containers?all=true`, {
+                headers: {
+                    'Authorization': `Bearer ${proxyTarget.apiToken}`,
+                    [PROXY_TIER_HEADER]: proxyHeaders.tier,
+                },
+                signal: AbortSignal.timeout(60_000),
+            });
+            if (!response.ok) {
+                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
+            }
+            return excludeSelfContainers(await response.json() as Array<{
+                Id: string;
+                Names?: string[];
+                State?: string;
+                Image?: string;
+                ImageID?: string;
+            }>);
+        } catch (err) {
+            this.rethrowRemoteProxyError(nodeId, err);
         }
-        return excludeSelfContainers(await response.json() as Array<{
-            Id: string;
-            Names?: string[];
-            State?: string;
-            Image?: string;
-            ImageID?: string;
-        }>);
     }
 
     private async postToRemoteContainer(
@@ -909,15 +949,39 @@ export class SchedulerService {
         containerId: string,
         action: 'start' | 'stop' | 'restart',
     ): Promise<void> {
-        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
-        if (!proxyTarget) {
-            throw new Error('Remote node is not configured or missing API credentials');
-        }
+        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
         const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
         const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
-        const response = await fetch(
-            `${baseUrl}/api/containers/${encodeURIComponent(containerId)}/${action}`,
-            {
+        try {
+            const response = await fetch(
+                `${baseUrl}/api/containers/${encodeURIComponent(containerId)}/${action}`,
+                {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'Authorization': `Bearer ${proxyTarget.apiToken}`,
+                        [PROXY_TIER_HEADER]: proxyHeaders.tier,
+                    },
+                    signal: AbortSignal.timeout(300_000),
+                },
+            );
+            if (!response.ok) {
+                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
+            }
+        } catch (err) {
+            this.rethrowRemoteProxyError(nodeId, err);
+        }
+    }
+
+    private async postToRemoteStack(nodeId: number, routeSuffix: string): Promise<void> {
+        const proxyTarget = this.requireRemoteProxyTarget(nodeId);
+        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
+        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
+        if (isDebugEnabled()) {
+            console.log(`[SchedulerService:debug] postToRemoteStack: node=${nodeId} route=${routeSuffix}`);
+        }
+        try {
+            const response = await fetch(`${baseUrl}/api/stacks/${routeSuffix}`, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
@@ -925,36 +989,12 @@ export class SchedulerService {
                     [PROXY_TIER_HEADER]: proxyHeaders.tier,
                 },
                 signal: AbortSignal.timeout(300_000),
-            },
-        );
-        if (!response.ok) {
-            const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
-            throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
-        }
-    }
-
-    private async postToRemoteStack(nodeId: number, routeSuffix: string): Promise<void> {
-        const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
-        if (!proxyTarget) {
-            throw new Error('Remote node is not configured or missing API credentials');
-        }
-        const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
-        const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
-        if (isDebugEnabled()) {
-            console.log(`[SchedulerService:debug] postToRemoteStack: node=${nodeId} route=${routeSuffix}`);
-        }
-        const response = await fetch(`${baseUrl}/api/stacks/${routeSuffix}`, {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${proxyTarget.apiToken}`,
-                [PROXY_TIER_HEADER]: proxyHeaders.tier,
-            },
-            signal: AbortSignal.timeout(300_000),
-        });
-        if (!response.ok) {
-            const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` }));
-            throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`);
+            });
+            if (!response.ok) {
+                throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
+            }
+        } catch (err) {
+            this.rethrowRemoteProxyError(nodeId, err);
         }
     }