From 8fd526ba0532cc891efe0366b0f9ad8e658e8319 Mon Sep 17 00:00:00 2001 From: Anso Date: Fri, 10 Jul 2026 13:05:03 -0400 Subject: [PATCH] feat(scheduler): include node identifier in remote proxy error messages (#1609) * feat(scheduler): include node identifier in remote proxy error messages Remote lifecycle proxy errors from postToRemoteStack, executeUpdateRemote, postToRemoteContainer, and getRemoteContainers now include a stable node label (name and numeric id) so a hub operator reading run history can correlate failures to a node without cross-referencing the task table. The prefix matches the existing containerNotFoundMessage pattern in the same class. Transport-level failures (DNS, ECONNREFUSED, timeout) also receive the node label. A startsWith guard prevents double- prefixing within the same method. The restart fan-out catch relies on the inner postToRemoteStack error for node context so the label appears exactly once. * fix(scheduler): pilot-aware remote proxy no-target error context Use formatNoTargetError for null proxy targets so pilot tunnel disconnects are not misdiagnosed as missing credentials. Standardize the failure prefix to Remote node name/id, share require/rethrow helpers, and cover pilot no-target plus remote container action failures in tests. --- .../src/__tests__/scheduler-service.test.ts | 134 +++++++++++- backend/src/services/SchedulerService.ts | 202 +++++++++++------- 2 files changed, 247 insertions(+), 89 deletions(-) diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index 5d4cd312..b2cd041e 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -131,6 +131,17 @@ vi.mock('../services/DockerController', () => ({ }, })); +vi.mock('../services/SelfIdentityService', () => ({ + default: { + getInstance: () => ({ + initialize: vi.fn().mockResolvedValue(undefined), + isOwnContainer: vi.fn().mockReturnValue(false), + isOwnImage: vi.fn().mockReturnValue(false), + resetForTesting: vi.fn(), + }), + }, +})); + vi.mock('../services/ComposeService', () => ({ ComposeService: { getInstance: () => ({ @@ -1741,7 +1752,7 @@ describe('SchedulerService - executeUpdateRemote', () => { expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, - expect.objectContaining({ status: 'failure', error: expect.stringContaining('Internal error') }) + expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): Internal error') }) ); }); }); @@ -1946,7 +1957,7 @@ describe('SchedulerService - lifecycle remote proxy', () => { await SchedulerService.getInstance().triggerTask(300); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, - expect.objectContaining({ status: 'failure', error: expect.stringContaining('Docker daemon is unreachable') }), + expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): Docker daemon is unreachable') }), ); }); @@ -1962,13 +1973,13 @@ describe('SchedulerService - lifecycle remote proxy', () => { await SchedulerService.getInstance().triggerTask(300); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, - expect.objectContaining({ status: 'failure', error: expect.stringContaining('HTTP 502') }), + expect.objectContaining({ status: 'failure', error: expect.stringContaining('Remote node "remote" (id=2): HTTP 502') }), ); }); - it('records failure when the remote node has no proxy credentials', async () => { - mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); - // getProxyTarget defaults to null (no credentials configured). + it('records failure when the remote proxy target is unavailable', async () => { + mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online', mode: 'proxy' }); + // getProxyTarget defaults to null (proxy credentials missing / unreachable). const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 })); @@ -1976,7 +1987,34 @@ describe('SchedulerService - lifecycle remote proxy', () => { expect(fetchMock).not.toHaveBeenCalled(); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, - expect.objectContaining({ status: 'failure', error: expect.stringContaining('not configured or missing API credentials') }), + expect.objectContaining({ + status: 'failure', + error: expect.stringContaining('Remote node "remote" (id=2): Remote node not configured'), + }), + ); + }); + + it('records pilot-aware no-target messaging when the tunnel is disconnected', async () => { + mockGetNode.mockReturnValue({ + id: 2, + name: 'edge-pilot', + type: 'remote', + status: 'online', + mode: 'pilot_agent', + }); + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); + expect(fetchMock).not.toHaveBeenCalled(); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ + status: 'failure', + error: expect.stringContaining( + 'Remote node "edge-pilot" (id=2): Pilot tunnel to "edge-pilot" is disconnected', + ), + }), ); }); @@ -1993,11 +2031,91 @@ describe('SchedulerService - lifecycle remote proxy', () => { await SchedulerService.getInstance().triggerTask(300); // Third service is never reached after the second fails. expect(fetchMock).toHaveBeenCalledTimes(2); + const call = (mockUpdateScheduledTaskRun as ReturnType).mock.calls.at(-1)!; + const errorMsg: string = call[1].error; + // Node context appears exactly once (via postToRemoteStack's inner prefix, not duplicated + // by executeRestartRemote's catch wrapper). + const nodeLabel = 'Remote node "remote" (id=2)'; + expect(errorMsg).toContain(nodeLabel); + expect(errorMsg.indexOf(nodeLabel)).toBe(errorMsg.lastIndexOf(nodeLabel)); + expect(errorMsg).toContain('already restarted: api'); + expect(errorMsg).toContain('worker'); + expect(errorMsg).toContain('boom'); + }); +}); + +// ── Remote container proxy error context ───────────────────────────── + +describe('SchedulerService - remote container proxy error context', () => { + it('prefixes transport failures with node context', async () => { + mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); + mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('connect ECONNREFUSED'))); + mockGetScheduledTask.mockReturnValue(makeLifecycleTask('auto_stop', { node_id: 2 })); + await SchedulerService.getInstance().triggerTask(300); expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( 1, - expect.objectContaining({ status: 'failure', error: expect.stringContaining('already restarted: api') }), + expect.objectContaining({ + status: 'failure', + error: expect.stringContaining('Remote node "remote" (id=2): connect ECONNREFUSED'), + }), ); }); + + it('prefixes remote container list failures with node context', async () => { + mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); + mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + ok: false, + status: 503, + json: async () => ({ error: 'Docker daemon is unreachable' }), + })); + mockGetScheduledTask.mockReturnValue({ + ...makeLifecycleTask('restart', { node_id: 2 }), + target_type: 'container', + target_services: null, + }); + await SchedulerService.getInstance().triggerTask(300); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ + status: 'failure', + error: expect.stringContaining('Remote node "remote" (id=2): Docker daemon is unreachable'), + }), + ); + }); + + it('prefixes remote container action failures with node context', async () => { + mockGetNode.mockReturnValue({ id: 2, name: 'remote', type: 'remote', status: 'online' }); + mockGetProxyTarget.mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tkn' }); + const fetchMock = vi.fn() + .mockResolvedValueOnce({ + ok: true, + json: async () => ([{ Id: 'ctrdeadbeef01', Names: ['/watchtower'], State: 'running', Image: 'containrrr/watchtower' }]), + }) + .mockResolvedValueOnce({ + ok: false, + status: 500, + json: async () => ({ error: 'container stop failed' }), + }); + vi.stubGlobal('fetch', fetchMock); + mockGetScheduledTask.mockReturnValue({ + ...makeLifecycleTask('auto_stop', { node_id: 2, target_id: 'watchtower' }), + target_type: 'container', + target_services: null, + }); + await SchedulerService.getInstance().triggerTask(300); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fetchMock.mock.calls[1][0]).toContain('/api/containers/ctrdeadbeef01/stop'); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ + status: 'failure', + error: expect.stringContaining('Remote node "remote" (id=2): container stop failed'), + }), + ); + }); + }); // ── Unpaid-tier guard in executeTask ──────────────────────────────────── diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index 8db8ee54..8e124143 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -12,6 +12,7 @@ import { ImageUpdateService } from './ImageUpdateService'; import type { ImageCheckResult } from './ImageUpdateService'; import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; +import { formatNoTargetError } from '../utils/remoteTarget'; import { sanitizeForLog } from '../utils/safeLog'; import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, type SnapshotNodeData } from '../utils/snapshot-capture'; import { NodeRegistry } from './NodeRegistry'; @@ -771,38 +772,37 @@ export class SchedulerService { * The remote node runs the image checks and compose update locally. */ private async executeUpdateRemote(nodeId: number, target: string): Promise { - const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId); - if (!proxyTarget) { - throw new Error('Remote node is not configured or missing API credentials'); - } - + const proxyTarget = this.requireRemoteProxyTarget(nodeId); const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); if (isDebugEnabled()) { console.log(`[SchedulerService] executeUpdateRemote: node=${nodeId} target=${target}`); } const startTime = Date.now(); - const response = await fetch(`${baseUrl}/api/auto-update/execute`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${proxyTarget.apiToken}`, - [PROXY_TIER_HEADER]: proxyHeaders.tier, - }, - body: JSON.stringify({ target }), - signal: AbortSignal.timeout(300_000), // 5 minute timeout for long updates - }); + try { + const response = await fetch(`${baseUrl}/api/auto-update/execute`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${proxyTarget.apiToken}`, + [PROXY_TIER_HEADER]: proxyHeaders.tier, + }, + body: JSON.stringify({ target }), + signal: AbortSignal.timeout(300_000), // 5 minute timeout for long updates + }); - if (!response.ok) { - const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` })); - throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`); - } + if (!response.ok) { + throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response))); + } - const body = await response.json() as { result?: string }; - if (isDebugEnabled()) { - console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`); + const body = await response.json() as { result?: string }; + if (isDebugEnabled()) { + console.log(`[SchedulerService] executeUpdateRemote: completed in ${Date.now() - startTime}ms`); + } + return body.result || 'Remote auto-update completed (no details returned).'; + } catch (err) { + this.rethrowRemoteProxyError(nodeId, err); } - return body.result || 'Remote auto-update completed (no details returned).'; } /** @@ -818,6 +818,46 @@ export class SchedulerService { return `Container "${containerName}" not found on node "${nodeName}". It may have been renamed or removed.`; } + /** Hub target tag used for startsWith rethrow guards (no trailing detail). */ + private remoteProxyNodeTag(nodeId: number): string { + const nodeName = NodeRegistry.getInstance().getNode(nodeId)?.name ?? String(nodeId); + return `Remote node "${nodeName}" (id=${nodeId})`; + } + + /** Operator-facing remote proxy failure with stable node correlation. */ + private remoteProxyFailureMessage(nodeId: number, detail: string): string { + return `${this.remoteProxyNodeTag(nodeId)}: ${detail}`; + } + + private noProxyTargetDetail(nodeId: number): string { + const node = NodeRegistry.getInstance().getNode(nodeId); + if (!node) return 'Remote node is not configured or missing API credentials'; + return formatNoTargetError(node); + } + + private async remoteResponseDetail(response: Response): Promise { + try { + const body = await response.json() as { error?: string }; + return body.error || `Remote node returned ${response.status}`; + } catch { + return `HTTP ${response.status}`; + } + } + + private requireRemoteProxyTarget(nodeId: number): { apiUrl: string; apiToken: string } { + const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId); + if (!proxyTarget) { + throw new Error(this.remoteProxyFailureMessage(nodeId, this.noProxyTargetDetail(nodeId))); + } + return proxyTarget; + } + + private rethrowRemoteProxyError(nodeId: number, err: unknown): never { + const tag = this.remoteProxyNodeTag(nodeId); + if (err instanceof Error && err.message.startsWith(tag)) throw err; + throw new Error(this.remoteProxyFailureMessage(nodeId, getErrorMessage(err, 'Remote proxy request failed'))); + } + private async resolveContainerId(task: ScheduledTask): Promise<{ id: string; name: string }> { if (!task.target_id || task.node_id == null) { throw new Error('Container operations require target_id and node_id'); @@ -878,30 +918,30 @@ export class SchedulerService { State?: string; Image?: string; }>> { - const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId); - if (!proxyTarget) { - throw new Error('Remote node is not configured or missing API credentials'); - } + const proxyTarget = this.requireRemoteProxyTarget(nodeId); const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); - const response = await fetch(`${baseUrl}/api/containers?all=true`, { - headers: { - 'Authorization': `Bearer ${proxyTarget.apiToken}`, - [PROXY_TIER_HEADER]: proxyHeaders.tier, - }, - signal: AbortSignal.timeout(60_000), - }); - if (!response.ok) { - const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` })); - throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`); + try { + const response = await fetch(`${baseUrl}/api/containers?all=true`, { + headers: { + 'Authorization': `Bearer ${proxyTarget.apiToken}`, + [PROXY_TIER_HEADER]: proxyHeaders.tier, + }, + signal: AbortSignal.timeout(60_000), + }); + if (!response.ok) { + throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response))); + } + return excludeSelfContainers(await response.json() as Array<{ + Id: string; + Names?: string[]; + State?: string; + Image?: string; + ImageID?: string; + }>); + } catch (err) { + this.rethrowRemoteProxyError(nodeId, err); } - return excludeSelfContainers(await response.json() as Array<{ - Id: string; - Names?: string[]; - State?: string; - Image?: string; - ImageID?: string; - }>); } private async postToRemoteContainer( @@ -909,15 +949,39 @@ export class SchedulerService { containerId: string, action: 'start' | 'stop' | 'restart', ): Promise { - const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId); - if (!proxyTarget) { - throw new Error('Remote node is not configured or missing API credentials'); - } + const proxyTarget = this.requireRemoteProxyTarget(nodeId); const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); - const response = await fetch( - `${baseUrl}/api/containers/${encodeURIComponent(containerId)}/${action}`, - { + try { + const response = await fetch( + `${baseUrl}/api/containers/${encodeURIComponent(containerId)}/${action}`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${proxyTarget.apiToken}`, + [PROXY_TIER_HEADER]: proxyHeaders.tier, + }, + signal: AbortSignal.timeout(300_000), + }, + ); + if (!response.ok) { + throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response))); + } + } catch (err) { + this.rethrowRemoteProxyError(nodeId, err); + } + } + + private async postToRemoteStack(nodeId: number, routeSuffix: string): Promise { + const proxyTarget = this.requireRemoteProxyTarget(nodeId); + const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); + const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); + if (isDebugEnabled()) { + console.log(`[SchedulerService:debug] postToRemoteStack: node=${nodeId} route=${routeSuffix}`); + } + try { + const response = await fetch(`${baseUrl}/api/stacks/${routeSuffix}`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -925,36 +989,12 @@ export class SchedulerService { [PROXY_TIER_HEADER]: proxyHeaders.tier, }, signal: AbortSignal.timeout(300_000), - }, - ); - if (!response.ok) { - const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` })); - throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`); - } - } - - private async postToRemoteStack(nodeId: number, routeSuffix: string): Promise { - const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId); - if (!proxyTarget) { - throw new Error('Remote node is not configured or missing API credentials'); - } - const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); - const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); - if (isDebugEnabled()) { - console.log(`[SchedulerService:debug] postToRemoteStack: node=${nodeId} route=${routeSuffix}`); - } - const response = await fetch(`${baseUrl}/api/stacks/${routeSuffix}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${proxyTarget.apiToken}`, - [PROXY_TIER_HEADER]: proxyHeaders.tier, - }, - signal: AbortSignal.timeout(300_000), - }); - if (!response.ok) { - const body = await response.json().catch(() => ({ error: `HTTP ${response.status}` })); - throw new Error((body as { error?: string }).error || `Remote node returned ${response.status}`); + }); + if (!response.ok) { + throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response))); + } + } catch (err) { + this.rethrowRemoteProxyError(nodeId, err); } }