diff --git a/backend/src/__tests__/blueprints.test.ts b/backend/src/__tests__/blueprints.test.ts index 434d3439..2a90a822 100644 --- a/backend/src/__tests__/blueprints.test.ts +++ b/backend/src/__tests__/blueprints.test.ts @@ -469,3 +469,106 @@ describe('BlueprintService marker parsing + name-conflict guard', () => { expect(BlueprintService.parseMarker(JSON.stringify({ blueprintId: 'nope', revision: 1 }))).toBeNull(); }); }); + +describe('BlueprintReconciler drift alert node wording', () => { + type ReconcilerWithDrift = { + handleDrift: (blueprint: Blueprint, node: Node, reason: string) => Promise; + }; + + function seedRemoteNode(name: string): number { + counter += 1; + const db = DatabaseService.getInstance().getDb(); + const result = db.prepare( + `INSERT INTO nodes (name, type, mode, compose_dir, is_default, status, created_at) + VALUES (?, 'remote', 'proxy', '/tmp/compose', 0, 'online', ?)` + ).run(name, Date.now()); + return result.lastInsertRowid as number; + } + + it('suggest-mode local drift uses on this node', async () => { + const { NotificationService } = await import('../services/NotificationService'); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const nodeId = seedNode(); + const bp = seedBlueprint({ name: 'drift-local', drift_mode: 'suggest', nodeIds: [nodeId] }); + const node = DatabaseService.getInstance().getNode(nodeId)!; + const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithDrift; + + await reconciler.handleDrift(bp, node, 'compose changed'); + + expect(dispatchSpy).toHaveBeenCalledWith( + 'warning', + 'blueprint_drift_detected', + 'Blueprint "drift-local" drifted on this node: compose changed', + { stackName: 'drift-local', actor: 'system:blueprint' }, + ); + }); + + it('suggest-mode remote drift keeps the authoritative node name', async () => { + const { NotificationService } = await import('../services/NotificationService'); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const nodeId = seedRemoteNode('sencho-test-02'); + const bp = seedBlueprint({ name: 'drift-remote', drift_mode: 'suggest', nodeIds: [nodeId] }); + const node = DatabaseService.getInstance().getNode(nodeId)!; + const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithDrift; + + await reconciler.handleDrift(bp, node, 'compose changed'); + + expect(dispatchSpy).toHaveBeenCalledWith( + 'warning', + 'blueprint_drift_detected', + 'Blueprint "drift-remote" drifted on node "sencho-test-02": compose changed', + { stackName: 'drift-remote', actor: 'system:blueprint' }, + ); + }); + + it('stateful marker-loss on local uses on this node', async () => { + const { NotificationService } = await import('../services/NotificationService'); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null); + const nodeId = seedNode(); + const bp = seedBlueprint({ + name: 'marker-local', + drift_mode: 'enforce', + classification: 'stateful', + nodeIds: [nodeId], + }); + const node = DatabaseService.getInstance().getNode(nodeId)!; + const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithDrift; + + await reconciler.handleDrift(bp, node, 'volumes diverged'); + + expect(dispatchSpy).toHaveBeenCalledWith( + 'warning', + 'blueprint_drift_detected', + 'Blueprint "marker-local" lost its marker on this node; auto-fix declined to avoid stomping unowned data. Reason: volumes diverged', + { stackName: 'marker-local', actor: 'system:blueprint' }, + ); + }); + + it('enforce correction-failure on local uses on this node', async () => { + const { NotificationService } = await import('../services/NotificationService'); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({ + status: 'failed', + error: 'compose up failed', + } as Awaited>); + const nodeId = seedNode(); + const bp = seedBlueprint({ + name: 'fix-local', + drift_mode: 'enforce', + classification: 'stateless', + nodeIds: [nodeId], + }); + const node = DatabaseService.getInstance().getNode(nodeId)!; + const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithDrift; + + await reconciler.handleDrift(bp, node, 'compose changed'); + + expect(dispatchSpy).toHaveBeenCalledWith( + 'error', + 'blueprint_drift_correction_failed', + 'Auto-fix for "fix-local" on this node failed: compose up failed', + { stackName: 'fix-local', actor: 'system:blueprint' }, + ); + }); +}); diff --git a/backend/src/__tests__/docker-event-service.test.ts b/backend/src/__tests__/docker-event-service.test.ts index 652516ce..2fb418e3 100644 --- a/backend/src/__tests__/docker-event-service.test.ts +++ b/backend/src/__tests__/docker-event-service.test.ts @@ -137,7 +137,7 @@ describe('DockerEventService - die classification', () => { expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', 'monitor_alert', - expect.stringContaining('Container Crash Detected'), + 'Container Crash Detected: web exited unexpectedly (Code: 1).', expect.objectContaining({ containerName: 'web' }), ); }); @@ -588,8 +588,8 @@ describe('DockerEventService - reconnect', () => { const warn = mockDispatchAlert.mock.calls.find(c => c[0] === 'warning'); const info = mockDispatchAlert.mock.calls.find(c => c[0] === 'info'); - expect(warn?.[2]).toContain('Lost connection'); - expect(info?.[2]).toContain('Reconnected'); + expect(warn?.[2]).toBe('Lost connection to Docker daemon; monitoring paused.'); + expect(info?.[2]).toBe('Reconnected to Docker daemon.'); }); it('shutdown cancels pending reconnect', async () => { diff --git a/backend/src/__tests__/image-update-service.test.ts b/backend/src/__tests__/image-update-service.test.ts index 6e3e6683..6994f129 100644 --- a/backend/src/__tests__/image-update-service.test.ts +++ b/backend/src/__tests__/image-update-service.test.ts @@ -452,13 +452,13 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImage(service, true); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockDispatchAlert).toHaveBeenCalledTimes(1); expect(mockDispatchAlert).toHaveBeenCalledWith( 'info', 'image_update_available', - expect.stringContaining('stackA'), + 'Stack "stackA" has image updates available.', { stackName: 'stackA', actor: 'system:image-update' }, ); expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number), 'ok', null); @@ -469,7 +469,7 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImage(service, true); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockDispatchAlert).not.toHaveBeenCalled(); }); @@ -483,7 +483,7 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImage(service, true); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockDispatchAlert).toHaveBeenCalledTimes(2); const dispatched = mockDispatchAlert.mock.calls.map(call => (call[3] as any)?.stackName); @@ -497,7 +497,7 @@ services: mockGetStackUpdateStatus.mockReturnValue({ stackA: true, stackB: true }); stubCheckImage(service, true); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockDispatchAlert).not.toHaveBeenCalled(); }); @@ -508,11 +508,11 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImage(service, true); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockAddNotificationHistory).toHaveBeenCalledWith(1, expect.objectContaining({ level: 'error', - message: expect.stringContaining('webhook timeout'), + message: 'Failed to notify about image updates for stack "stackA": webhook timeout', })); }); }); @@ -567,7 +567,7 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImageByRef(service, { 'nginx:latest': { hasUpdate: false, error: 'Registry unreachable for registry-1.docker.io/library/nginx:latest' } }); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockRecordStackCheckFailure).toHaveBeenCalledWith(1, 'stackA', expect.stringContaining('Registry unreachable'), expect.any(Number)); expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); @@ -584,7 +584,7 @@ services: 'postgres:15': { hasUpdate: false, error: 'Registry unreachable for registry-1.docker.io/library/postgres:15' }, }); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number), 'partial', expect.stringContaining('Registry unreachable')); expect(mockRecordStackCheckFailure).not.toHaveBeenCalled(); @@ -605,7 +605,7 @@ services: 'postgres:15': { hasUpdate: false }, }); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number), 'partial', expect.stringContaining('Registry unreachable')); expect(mockRecordStackCheckFailure).not.toHaveBeenCalled(); @@ -619,7 +619,7 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImageByRef(service, { 'nginx:latest': { hasUpdate: false, notCheckable: true } }); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', false, expect.any(Number), 'ok', null); expect(mockRecordStackCheckFailure).not.toHaveBeenCalled(); @@ -665,7 +665,7 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImage(service, false); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockEnvExists).toHaveBeenCalledWith('stackA'); expect(mockGetEnvContent).not.toHaveBeenCalled(); @@ -677,7 +677,7 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImage(service, false); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockEnvExists).toHaveBeenCalledWith('stackA'); expect(mockGetEnvContent).toHaveBeenCalledWith('stackA'); @@ -689,7 +689,7 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImage(service, false); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); // Should not throw; should still complete and write status expect(mockUpsertStackUpdateStatus).toHaveBeenCalled(); @@ -1070,7 +1070,7 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImage(service, false); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockClearStackUpdateStatus).toHaveBeenCalledWith(1, 'stackB'); }); @@ -1080,7 +1080,7 @@ services: const service = ImageUpdateService.getInstance(); stubCheckImage(service, false); - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); expect(mockClearStackUpdateStatus).not.toHaveBeenCalled(); }); @@ -1125,7 +1125,7 @@ services: const checkImageSpy = vi.fn().mockResolvedValue({ hasUpdate: false }); (service as any).checkImage = checkImageSpy; - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); // Should check both the compose image and the container image const checkedImages = checkImageSpy.mock.calls.map((c: any[]) => c[1]); @@ -1143,7 +1143,7 @@ services: const checkImageSpy = vi.fn().mockResolvedValue({ hasUpdate: false }); (service as any).checkImage = checkImageSpy; - await (service as any).checkNode(1, 'local', fakeDb()); + await (service as any).checkNode(1, fakeDb()); const checkedImages = checkImageSpy.mock.calls.map((c: any[]) => c[1]); expect(checkedImages).not.toContain('someapp:v2'); diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index b522cada..4d51b855 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -895,7 +895,12 @@ describe('MonitorService - breach state machine', () => { const svc = MonitorService.getInstance(); await (svc as any).evaluate(); - expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: 'my-stack', actor: 'system:monitor' }); + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + 'The **CPU usage** for **my-stack** has exceeded your threshold of **80%** (Currently: 90%).', + { stackName: 'my-stack', actor: 'system:monitor' }, + ); expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); }); diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index b2cd041e..029b7e6a 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -2159,3 +2159,97 @@ describe('SchedulerService - delete_after_run', () => { expect(mockUpdateScheduledTask).toHaveBeenCalledWith(300, expect.objectContaining({ last_status: 'success' })); }); }); + +// ── Node-neutral failure alerts (local vs remote) ────────────────────── + +describe('SchedulerService - node-neutral failure alerts', () => { + function makeContainerTask(overrides: Partial & { id: number; name: string }) { + return { + action: 'restart', + target_type: 'container', + target_id: 'web', + cron_expression: '0 * * * *', + enabled: true, + node_id: 1, + created_by: 'admin', + last_status: null, + ...overrides, + }; + } + + it('dispatches local offline failure without the seed node name', async () => { + mockGetScheduledTask.mockReturnValue(makeContainerTask({ id: 400, name: 'offline-local', node_id: 1 })); + mockGetNode.mockReturnValue({ id: 1, name: 'Local', type: 'local', status: 'offline' }); + + await SchedulerService.getInstance().triggerTask(400); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'error', + 'system', + 'Scheduled task "offline-local" (restart) failed: Target node is offline', + { stackName: 'web', actor: 'system:scheduler' }, + ); + }); + + it('dispatches remote offline failure with the authoritative node name', async () => { + mockGetScheduledTask.mockReturnValue(makeContainerTask({ id: 401, name: 'offline-remote', node_id: 2 })); + mockGetNode.mockReturnValue({ id: 2, name: 'sencho-test-02', type: 'remote', status: 'offline' }); + + await SchedulerService.getInstance().triggerTask(401); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'error', + 'system', + 'Scheduled task "offline-remote" (restart) failed: Target node "sencho-test-02" is offline', + { stackName: 'web', actor: 'system:scheduler' }, + ); + }); + + it('dispatches local missing-container failure with this-node wording', async () => { + mockGetScheduledTask.mockReturnValue(makeContainerTask({ id: 402, name: 'missing-local', node_id: 1 })); + mockGetNode.mockReturnValue({ id: 1, name: 'Local', type: 'local', status: 'online' }); + mockFindContainerByName.mockResolvedValue(null); + + await SchedulerService.getInstance().triggerTask(402); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'error', + 'system', + 'Scheduled task "missing-local" (restart) failed: Container "web" not found on this node. It may have been renamed or removed.', + { stackName: 'web', actor: 'system:scheduler' }, + ); + }); + + it('dispatches remote missing-container failure with the authoritative node name', async () => { + mockGetScheduledTask.mockReturnValue(makeContainerTask({ id: 403, name: 'missing-remote', node_id: 2 })); + mockGetNode.mockReturnValue({ id: 2, name: 'sencho-test-02', type: 'remote', status: 'online' }); + mockGetProxyTarget.mockReturnValue({ + apiUrl: 'http://remote:1852', + apiToken: 'test-token', + }); + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ([]), + }); + vi.stubGlobal('fetch', mockFetch); + + await SchedulerService.getInstance().triggerTask(403); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'error', + 'system', + 'Scheduled task "missing-remote" (restart) failed: Container "web" not found on node "sencho-test-02". It may have been renamed or removed.', + { stackName: 'web', actor: 'system:scheduler' }, + ); + }); + + it('missing node lookup keeps the id fallback, not this-node wording', () => { + mockGetNode.mockReturnValue(undefined); + const svc = SchedulerService.getInstance() as unknown as { + containerNotFoundMessage: (containerName: string, nodeId: number) => string; + }; + expect(svc.containerNotFoundMessage('web', 99)).toBe( + 'Container "web" not found on node "99". It may have been renamed or removed.', + ); + }); +}); diff --git a/backend/src/services/BlueprintReconciler.ts b/backend/src/services/BlueprintReconciler.ts index 4f8df5ce..c048a49d 100644 --- a/backend/src/services/BlueprintReconciler.ts +++ b/backend/src/services/BlueprintReconciler.ts @@ -29,6 +29,11 @@ function diagnosticLog(message: string, fields: Record { - return this.notifier.dispatchAlert('error', category, this.prefix(message), this.buildAlertOptions(stackName, containerName, systemOnly)); + return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly)); } private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { - return this.notifier.dispatchAlert('warning', category, this.prefix(message), this.buildAlertOptions(stackName, containerName)); + return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName)); } private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { - return this.notifier.dispatchAlert('info', category, this.prefix(message), this.buildAlertOptions(stackName, containerName)); - } - - private prefix(message: string): string { - return `[Node: ${this.nodeName}] ${message}`; + return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName)); } // ======================================================================== diff --git a/backend/src/services/ImageUpdateService.ts b/backend/src/services/ImageUpdateService.ts index e4ca1444..719b9016 100644 --- a/backend/src/services/ImageUpdateService.ts +++ b/backend/src/services/ImageUpdateService.ts @@ -523,7 +523,7 @@ export class ImageUpdateService { for (const node of db.getNodes()) { if (node.type !== 'local' || !node.id) continue; try { - await this.checkNode(node.id, node.name, db); + await this.checkNode(node.id, db); } catch (e) { console.error(`[ImageUpdateService] Error on node ${node.name}:`, e); } @@ -536,7 +536,7 @@ export class ImageUpdateService { } } - private async checkNode(nodeId: number, nodeName: string, db: DatabaseService) { + private async checkNode(nodeId: number, db: DatabaseService) { const docker = DockerController.getInstance(nodeId); const fs = FileSystemService.getInstance(nodeId); const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId)); @@ -708,7 +708,8 @@ export class ImageUpdateService { await notifier.dispatchAlert( 'info', 'image_update_available', - `[Node: ${nodeName}] Stack "${stackName}" has image updates available.`, + // Node-neutral body: the hub bell badge attributes remotes. + `Stack "${stackName}" has image updates available.`, { stackName, actor: 'system:image-update' }, ); } catch (e) { @@ -722,7 +723,7 @@ export class ImageUpdateService { level: 'error', category: 'system', message: sanitizeNotificationMessage( - `[Node: ${nodeName}] Failed to notify about image updates for stack "${stackName}": ${getErrorMessage(e, String(e))}`, + `Failed to notify about image updates for stack "${stackName}": ${getErrorMessage(e, String(e))}`, { composeDir: NodeRegistry.getInstance().getComposeDir(localNodeId) }, ), timestamp: Date.now(), diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index 608a9208..4840b08c 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -765,7 +765,8 @@ export class MonitorService { const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue; const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold; - const message = `[Node: ${node.name}] The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`; + // Node-neutral body: the hub bell badge attributes remotes. + const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`; console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`); await NotificationService.getInstance().dispatchAlert( diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index b2ad6d37..cb3f05d5 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -298,7 +298,15 @@ export class SchedulerService { if (task.node_id != null && task.action !== 'snapshot') { const node = db.getNode(task.node_id); if (!node) throw new Error(`Target node (id=${task.node_id}) no longer exists`); - if (node.status === 'offline') throw new Error(`Target node "${node.name}" is offline`); + if (node.status === 'offline') { + // Local seed name ("Local") must not reach fleet-aggregated alerts; + // remote roster names stay for diagnosis. + throw new Error( + node.type === 'local' + ? 'Target node is offline' + : `Target node "${node.name}" is offline`, + ); + } } if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} pre-checks passed, executing ${task.action}`); @@ -813,8 +821,12 @@ export class SchedulerService { * segment. */ private containerNotFoundMessage(containerName: string, nodeId: number): string { - const nodeName = NodeRegistry.getInstance().getNode(nodeId)?.name ?? String(nodeId); - return `Container "${containerName}" not found on node "${nodeName}". It may have been renamed or removed.`; + const node = NodeRegistry.getInstance().getNode(nodeId); + // Only an explicit local node is neutralized; missing lookup keeps the id fallback. + const location = node?.type === 'local' + ? 'on this node' + : `on node "${node?.name ?? String(nodeId)}"`; + return `Container "${containerName}" not found ${location}. It may have been renamed or removed.`; } /** Hub target tag used for startsWith rethrow guards (no trailing detail). */ diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index f8f24d92..d7ad204e 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -35,7 +35,7 @@ Use the Generic Webhook tab when you have your own receiver: a Mattermost or Tea ```json { "level": "warning", - "message": "[Node: Local] The cpu_percent for plex has exceeded your threshold of 80% (Currently: 91%).", + "message": "The **CPU usage** for **plex** has exceeded your threshold of **80%** (Currently: 91%).", "timestamp": "2026-05-08T22:14:09.812Z", "source": "sencho" } @@ -287,13 +287,13 @@ The bell aggregates across the entire fleet by hitting `/api/notifications` agai Per-stack alert rules and channel configuration are **stored on the node where the stack runs**. To configure a rule on a remote, switch the active node via the picker, open the stack's **Monitor** sheet, and the form's POST is forwarded to the remote. -Crash detection runs only on local Docker; remote nodes run their own copy of `DockerEventService` and emit through the proxy. Each emitted message is prefixed with `[Node: ]` so the source is unambiguous in the channel. +Crash detection runs only on local Docker; remote nodes run their own copy of `DockerEventService` and emit through the proxy. On a multi-node fleet the bell attributes remotes with a node-name badge rather than embedding the satellite-local name in the message. External channel payloads (Slack, Discord, custom webhook) receive the same sanitized body with no structured source-node id. Switching the active node only affects per-stack rule editing and the Settings panels. The bell aggregates every node regardless of which one is active. ## Alerts emitted by the system -Sencho dispatches notifications from many code paths. The list below covers every emitter in current code, organized by source. Each message is prefixed with `[Node: ]` when emitted by a node-scoped service. +Sencho dispatches notifications from many code paths. The list below covers every emitter in current code, organized by source. Message bodies are node-neutral for local targets so fleet aggregation does not contradict the bell badge; remote-target names stay when a hub names an authoritative remote. ### Container crash and health @@ -332,7 +332,7 @@ The entire host threshold evaluation can be silenced per node from **Settings · ### Image update availability -`info`/`image_update_available`: `Stack "" has image updates available.` Polled every 6 hours, with a 2-minute startup delay and a 2-minute cooldown on manual refresh. Notifies on **state transition only**; pre-existing `has_update` rows are backfilled silently on first run. +`info`/`image_update_available`: `Stack "" has image updates available.` By default, checks every two hours in interval mode, with a two-minute startup delay. The cadence can be changed or replaced with a cron schedule. Manual refresh carries a two-minute cooldown. Notifies on **state transition only**; the first run also emits catch-up notifications for stacks already known to have updates. ### Auto-update execution @@ -362,14 +362,16 @@ See [Vulnerability scanning](/features/vulnerability-scanning). - `error`/`system`: `Scheduled task "" () failed: ` - `info`/`system` recovery: `Scheduled task "" () recovered successfully` +Local-target failure reasons stay node-neutral (`Target node is offline`, `Container "" not found on this node. ...`). When the task targets a remote node, the failure reason keeps that node's roster name. + ### Recovery Vault upload failure `warning`/`system`: `Cloud backup failed for scheduled snapshot : `. See [Fleet backups](/features/fleet-backups). ### Blueprints -- `warning`/`blueprint_drift_detected`: `Blueprint "" drifted on node "": ` (suggest mode), with stateful-safeguard variants when enforce mode declines to redeploy. -- `error`/`blueprint_drift_correction_failed`: `Auto-fix for "" on node "" failed: `. +- `warning`/`blueprint_drift_detected`: `Blueprint "" drifted on this node: ` for a local target, or `Blueprint "" drifted on node "": ` for a remote. Stateful marker-loss uses the same local/remote clause. +- `error`/`blueprint_drift_correction_failed`: `Auto-fix for "" on this node failed: ` (local) or `Auto-fix for "" on node "" failed: ` (remote). See [Blueprints](/features/blueprint-model). @@ -459,6 +461,6 @@ Switching the active node tears down per-stack rule editors and reloads channel Fleet and the notification path share the same version lookup. When a newer published release is in that cache, Monitor dispatches a single `info`/`node_update_available` alert and records it under `last_sencho_update_notified_version`, so each release produces one alert per node. If you already upgraded to (or past) that version, the dedup self-heals and the alert does not fire. Check **Settings · Notifications · Mute Rules** for a rule that mutes `node_update_available` in the bell. A newly published release may take up to the cache TTL (about 30 minutes when published, or about 3 minutes while registry publish is still pending) before Fleet and the bell both observe it. - The image-update service emits on **state transitions only**. The first run of the service backfills a flag against rows that already had `has_update: true` so they do not all re-fire on first install. Once the flag is set, only the false-to-true transition triggers an alert. If a stack was already showing the indicator before backfill ran, a notification will only appear the next time it goes from up-to-date back to behind. + The image-update service emits on **state transitions only** after its first run. The first run also emits catch-up notifications for stacks already known to have updates, then sets a backfill flag. Once that flag is set, only a false-to-true transition triggers another alert. If you dismissed the catch-up notification, a new one appears the next time the stack goes from up-to-date back to behind. diff --git a/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts b/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts index fe16a4de..5550164e 100644 --- a/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts @@ -193,6 +193,27 @@ describe('useNotifications', () => { await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 2)); }); + it('stamps roster nodeName on remote REST notifications without rewriting the body', async () => { + const remote = makeRemoteNode('online', { id: 2, name: 'sencho-sat-qa' }); + const neutralBody = + 'Scheduled task "qa-missing-container" (restart) failed: Container "web" not found on this node. It may have been renamed or removed.'; + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + (fetchForNode as ReturnType).mockResolvedValue({ + ok: true, + json: async () => [{ id: 9, level: 'error', message: neutralBody, timestamp: 2000, is_read: 0 }], + }); + + const { result } = renderHook(() => + useNotifications({ nodes: [localNode, remote], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ); + + await waitFor(() => expect(result.current.notifications).toHaveLength(1)); + expect(result.current.notifications[0].message).toBe(neutralBody); + expect(result.current.notifications[0].nodeId).toBe(2); + expect(result.current.notifications[0].nodeName).toBe('sencho-sat-qa'); + expect(result.current.notifications[0].message).not.toMatch(/\[Node:|Local/); + }); + it('subscribes to the online node and skips the offline one in a mixed fleet', async () => { renderHook(() => useNotifications({