feat(scheduler): include node identifier in remote proxy error messages (#1609)

* feat(scheduler): include node identifier in remote proxy error messages Remote lifecycle proxy errors from postToRemoteStack, executeUpdateRemote, postToRemoteContainer, and getRemoteContainers now include a stable node label (name and numeric id) so a hub operator reading run history can correlate failures to a node without cross-referencing the task table. The prefix matches the existing containerNotFoundMessage pattern in the same class. Transport-level failures (DNS, ECONNREFUSED, timeout) also receive the node label. A startsWith guard prevents double- prefixing within the same method. The restart fan-out catch relies on the inner postToRemoteStack error for node context so the label appears exactly once. * fix(scheduler): pilot-aware remote proxy no-target error context Use formatNoTargetError for null proxy targets so pilot tunnel disconnects are not misdiagnosed as missing credentials. Standardize the failure prefix to Remote node name/id, share require/rethrow helpers, and cover pilot no-target plus remote container action failures in tests.
2026-08-21 07:36:40 +00:00 · 2026-07-10 13:05:03 -04:00
parent ba2e7bded9
commit 8fd526ba05
2 changed files with 247 additions and 89 deletions
@@ -131,6 +131,17 @@ vi.mock('../services/DockerController', () => ({
  },
 }));

+vi.mock('../services/SelfIdentityService', () => ({
+  default: {
+    getInstance: () => ({
+      initialize: vi.fn().mockResolvedValue(undefined),
+      isOwnContainer: vi.fn().mockReturnValue(false),
+      isOwnImage: vi.fn().mockReturnValue(false),
+      resetForTesting: vi.fn(),
+    }),
+  },
+}));
+
 vi.mock('../services/ComposeService', () => ({
  ComposeService: {
    getInstance: () => ({
@@ -1741,7 +1752,7 @@ describe('SchedulerService - executeUpdateRemote', () => {

    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Internal error') })
+      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): Internal error') })
    );
  });
 });
@@ -1946,7 +1957,7 @@ describe('SchedulerService - lifecycle remote proxy', () => {
    await SchedulerService.getInstance().triggerTask(300);
    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Docker daemon is unreachable') }),
+      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): Docker daemon is unreachable') }),
    );
  });

@@ -1962,13 +1973,13 @@ describe('SchedulerService - lifecycle remote proxy', () => {
    await SchedulerService.getInstance().triggerTask(300);
    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('HTTP 502') }),
+      expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): HTTP 502') }),
    );
  });

-  it('records failure when the remote node has no proxy credentials', async () => {
-    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
-    // getProxyTarget defaults to null (no credentials configured).
+  it('records failure when the remote proxy target is unavailable', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online', mode: 'proxy' });
+    // getProxyTarget defaults to null (proxy credentials missing / unreachable).
    const fetchMock = vi.fn();
    vi.stubGlobal('fetch', fetchMock);
    mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
@@ -1976,7 +1987,34 @@ describe('SchedulerService - lifecycle remote proxy', () => {
    expect(fetchMock).not.toHaveBeenCalled();
    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('not configured or missing API credentials') }),
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): Remote node not configured'),
+      }),
+    );
+  });
+
+  it('records pilot-aware no-target messaging when the tunnel is disconnected', async () => {
+    mockGetNode.mockReturnValue({
+      id: 2,
+      name: 'edge-pilot',
+      type: 'remote',
+      status: 'online',
+      mode: 'pilot_agent',
+    });
+    const fetchMock = vi.fn();
+    vi.stubGlobal('fetch', fetchMock);
+    mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
+    await SchedulerService.getInstance().triggerTask(300);
+    expect(fetchMock).not.toHaveBeenCalled();
+    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining(
+          'Remote node "edge-pilot" (id=2): Pilot tunnel to "edge-pilot" is disconnected',
+        ),
+      }),
    );
  });

@@ -1993,11 +2031,91 @@ describe('SchedulerService - lifecycle remote proxy', () => {
    await SchedulerService.getInstance().triggerTask(300);
    // Third service is never reached after the second fails.
    expect(fetchMock).toHaveBeenCalledTimes(2);
+    const call = (mockUpdateScheduledTaskRun as ReturnType<typeof vi.fn>).mock.calls.at(-1)!;
+    const errorMsg: string = call[1].error;
+    // Node context appears exactly once (via postToRemoteStack's inner prefix, not duplicated
+    // by executeRestartRemote's catch wrapper).
+    const nodeLabel = 'Remote node "remote" (id=2)';
+    expect(errorMsg).toContain(nodeLabel);
+    expect(errorMsg.indexOf(nodeLabel)).toBe(errorMsg.lastIndexOf(nodeLabel));
+    expect(errorMsg).toContain('already restarted: api');
+    expect(errorMsg).toContain('worker');
+    expect(errorMsg).toContain('boom');
+  });
+});
+
+// ── Remote container proxy error context ─────────────────────────────
+
+describe('SchedulerService - remote container proxy error context', () => {
+  it('prefixes transport failures with node context', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
+    mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
+    vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('connect ECONNREFUSED')));
+    mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 }));
+    await SchedulerService.getInstance().triggerTask(300);
    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
      1,
-      expect.objectContaining({ status: 'failure', error: expect.stringContaining('already restarted: api') }),
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): connect ECONNREFUSED'),
+      }),
    );
  });
+
+  it('prefixes remote container list failures with node context', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
+    mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
+      ok: false,
+      status: 503,
+      json: async () => ({ error: 'Docker daemon is unreachable' }),
+    }));
+    mockGetScheduledTask.mockReturnValue({
+      ...makeLifecycleTask('restart', { node_id: 2 }),
+      target_type: 'container',
+      target_services: null,
+    });
+    await SchedulerService.getInstance().triggerTask(300);
+    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): Docker daemon is unreachable'),
+      }),
+    );
+  });
+
+  it('prefixes remote container action failures with node context', async () => {
+    mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' });
+    mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' });
+    const fetchMock = vi.fn()
+      .mockResolvedValueOnce({
+        ok: true,
+        json: async () => ([{ Id: 'ctrdeadbeef01', Names: ['/watchtower'], State: 'running', Image: 'containrrr/watchtower' }]),
+      })
+      .mockResolvedValueOnce({
+        ok: false,
+        status: 500,
+        json: async () => ({ error: 'container stop failed' }),
+      });
+    vi.stubGlobal('fetch', fetchMock);
+    mockGetScheduledTask.mockReturnValue({
+      ...makeLifecycleTask('auto_stop', { node_id: 2, target_id: 'watchtower' }),
+      target_type: 'container',
+      target_services: null,
+    });
+    await SchedulerService.getInstance().triggerTask(300);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(fetchMock.mock.calls[1][0]).toContain('/api/containers/ctrdeadbeef01/stop');
+    expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith(
+      1,
+      expect.objectContaining({
+        status: 'failure',
+        error: expect.stringContaining('Remote node "remote" (id=2): container stop failed'),
+      }),
+    );
+  });
+
 });

 // ── Unpaid-tier guard in executeTask ────────────────────────────────────