mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
fix(notifications): neutralize satellite-local node names in alert bodies (#1640)
* fix(notifications): neutralize satellite-local node names in alert bodies Fleet-aggregated alerts embedded each instance seed name (often Local) while the hub badge already named the remote. Drop identity prefixes and use type-aware local wording so attribution stays on the badge. * fix(docs): correct image-update default check cadence Operator docs still said six-hour polling; the seeded default is two hours in interval mode, and the cadence is configurable or cron-based. * test(notifications): assert hub stamps roster name on neutral remote bodies Cover the fan-in path that attaches hub roster identity while leaving the satellite message body unchanged.
This commit is contained in:
Anso
@@ -469,3 +469,106 @@ describe('BlueprintService marker parsing + name-conflict guard', () => {
|
||||
expect(BlueprintService.parseMarker(JSON.stringify({ blueprintId: 'nope', revision: 1 }))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('BlueprintReconciler drift alert node wording', () => {
|
||||
type ReconcilerWithDrift = {
|
||||
handleDrift: (blueprint: Blueprint, node: Node, reason: string) => Promise<void>;
|
||||
};
|
||||
|
||||
function seedRemoteNode(name: string): number {
|
||||
counter += 1;
|
||||
const db = DatabaseService.getInstance().getDb();
|
||||
const result = db.prepare(
|
||||
`INSERT INTO nodes (name, type, mode, compose_dir, is_default, status, created_at)
|
||||
VALUES (?, 'remote', 'proxy', '/tmp/compose', 0, 'online', ?)`
|
||||
).run(name, Date.now());
|
||||
return result.lastInsertRowid as number;
|
||||
}
|
||||
|
||||
it('suggest-mode local drift uses on this node', async () => {
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
const nodeId = seedNode();
|
||||
const bp = seedBlueprint({ name: 'drift-local', drift_mode: 'suggest', nodeIds: [nodeId] });
|
||||
const node = DatabaseService.getInstance().getNode(nodeId)!;
|
||||
const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithDrift;
|
||||
|
||||
await reconciler.handleDrift(bp, node, 'compose changed');
|
||||
|
||||
expect(dispatchSpy).toHaveBeenCalledWith(
|
||||
'warning',
|
||||
'blueprint_drift_detected',
|
||||
'Blueprint "drift-local" drifted on this node: compose changed',
|
||||
{ stackName: 'drift-local', actor: 'system:blueprint' },
|
||||
);
|
||||
});
|
||||
|
||||
it('suggest-mode remote drift keeps the authoritative node name', async () => {
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
const nodeId = seedRemoteNode('sencho-test-02');
|
||||
const bp = seedBlueprint({ name: 'drift-remote', drift_mode: 'suggest', nodeIds: [nodeId] });
|
||||
const node = DatabaseService.getInstance().getNode(nodeId)!;
|
||||
const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithDrift;
|
||||
|
||||
await reconciler.handleDrift(bp, node, 'compose changed');
|
||||
|
||||
expect(dispatchSpy).toHaveBeenCalledWith(
|
||||
'warning',
|
||||
'blueprint_drift_detected',
|
||||
'Blueprint "drift-remote" drifted on node "sencho-test-02": compose changed',
|
||||
{ stackName: 'drift-remote', actor: 'system:blueprint' },
|
||||
);
|
||||
});
|
||||
|
||||
it('stateful marker-loss on local uses on this node', async () => {
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null);
|
||||
const nodeId = seedNode();
|
||||
const bp = seedBlueprint({
|
||||
name: 'marker-local',
|
||||
drift_mode: 'enforce',
|
||||
classification: 'stateful',
|
||||
nodeIds: [nodeId],
|
||||
});
|
||||
const node = DatabaseService.getInstance().getNode(nodeId)!;
|
||||
const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithDrift;
|
||||
|
||||
await reconciler.handleDrift(bp, node, 'volumes diverged');
|
||||
|
||||
expect(dispatchSpy).toHaveBeenCalledWith(
|
||||
'warning',
|
||||
'blueprint_drift_detected',
|
||||
'Blueprint "marker-local" lost its marker on this node; auto-fix declined to avoid stomping unowned data. Reason: volumes diverged',
|
||||
{ stackName: 'marker-local', actor: 'system:blueprint' },
|
||||
);
|
||||
});
|
||||
|
||||
it('enforce correction-failure on local uses on this node', async () => {
|
||||
const { NotificationService } = await import('../services/NotificationService');
|
||||
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
|
||||
vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({
|
||||
status: 'failed',
|
||||
error: 'compose up failed',
|
||||
} as Awaited<ReturnType<typeof BlueprintService.prototype.deployToNode>>);
|
||||
const nodeId = seedNode();
|
||||
const bp = seedBlueprint({
|
||||
name: 'fix-local',
|
||||
drift_mode: 'enforce',
|
||||
classification: 'stateless',
|
||||
nodeIds: [nodeId],
|
||||
});
|
||||
const node = DatabaseService.getInstance().getNode(nodeId)!;
|
||||
const reconciler = BlueprintReconciler.getInstance() as unknown as ReconcilerWithDrift;
|
||||
|
||||
await reconciler.handleDrift(bp, node, 'compose changed');
|
||||
|
||||
expect(dispatchSpy).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'blueprint_drift_correction_failed',
|
||||
'Auto-fix for "fix-local" on this node failed: compose up failed',
|
||||
{ stackName: 'fix-local', actor: 'system:blueprint' },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -137,7 +137,7 @@ describe('DockerEventService - die classification', () => {
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'monitor_alert',
|
||||
expect.stringContaining('Container Crash Detected'),
|
||||
'Container Crash Detected: web exited unexpectedly (Code: 1).',
|
||||
expect.objectContaining({ containerName: 'web' }),
|
||||
);
|
||||
});
|
||||
@@ -588,8 +588,8 @@ describe('DockerEventService - reconnect', () => {
|
||||
|
||||
const warn = mockDispatchAlert.mock.calls.find(c => c[0] === 'warning');
|
||||
const info = mockDispatchAlert.mock.calls.find(c => c[0] === 'info');
|
||||
expect(warn?.[2]).toContain('Lost connection');
|
||||
expect(info?.[2]).toContain('Reconnected');
|
||||
expect(warn?.[2]).toBe('Lost connection to Docker daemon; monitoring paused.');
|
||||
expect(info?.[2]).toBe('Reconnected to Docker daemon.');
|
||||
});
|
||||
|
||||
it('shutdown cancels pending reconnect', async () => {
|
||||
|
||||
@@ -452,13 +452,13 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImage(service, true);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'info',
|
||||
'image_update_available',
|
||||
expect.stringContaining('stackA'),
|
||||
'Stack "stackA" has image updates available.',
|
||||
{ stackName: 'stackA', actor: 'system:image-update' },
|
||||
);
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number), 'ok', null);
|
||||
@@ -469,7 +469,7 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImage(service, true);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -483,7 +483,7 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImage(service, true);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
|
||||
const dispatched = mockDispatchAlert.mock.calls.map(call => (call[3] as any)?.stackName);
|
||||
@@ -497,7 +497,7 @@ services:
|
||||
mockGetStackUpdateStatus.mockReturnValue({ stackA: true, stackB: true });
|
||||
stubCheckImage(service, true);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -508,11 +508,11 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImage(service, true);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockAddNotificationHistory).toHaveBeenCalledWith(1, expect.objectContaining({
|
||||
level: 'error',
|
||||
message: expect.stringContaining('webhook timeout'),
|
||||
message: 'Failed to notify about image updates for stack "stackA": webhook timeout',
|
||||
}));
|
||||
});
|
||||
});
|
||||
@@ -567,7 +567,7 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImageByRef(service, { 'nginx:latest': { hasUpdate: false, error: 'Registry unreachable for registry-1.docker.io/library/nginx:latest' } });
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockRecordStackCheckFailure).toHaveBeenCalledWith(1, 'stackA', expect.stringContaining('Registry unreachable'), expect.any(Number));
|
||||
expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled();
|
||||
@@ -584,7 +584,7 @@ services:
|
||||
'postgres:15': { hasUpdate: false, error: 'Registry unreachable for registry-1.docker.io/library/postgres:15' },
|
||||
});
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number), 'partial', expect.stringContaining('Registry unreachable'));
|
||||
expect(mockRecordStackCheckFailure).not.toHaveBeenCalled();
|
||||
@@ -605,7 +605,7 @@ services:
|
||||
'postgres:15': { hasUpdate: false },
|
||||
});
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', true, expect.any(Number), 'partial', expect.stringContaining('Registry unreachable'));
|
||||
expect(mockRecordStackCheckFailure).not.toHaveBeenCalled();
|
||||
@@ -619,7 +619,7 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImageByRef(service, { 'nginx:latest': { hasUpdate: false, notCheckable: true } });
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith(1, 'stackA', false, expect.any(Number), 'ok', null);
|
||||
expect(mockRecordStackCheckFailure).not.toHaveBeenCalled();
|
||||
@@ -665,7 +665,7 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImage(service, false);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockEnvExists).toHaveBeenCalledWith('stackA');
|
||||
expect(mockGetEnvContent).not.toHaveBeenCalled();
|
||||
@@ -677,7 +677,7 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImage(service, false);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockEnvExists).toHaveBeenCalledWith('stackA');
|
||||
expect(mockGetEnvContent).toHaveBeenCalledWith('stackA');
|
||||
@@ -689,7 +689,7 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImage(service, false);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
// Should not throw; should still complete and write status
|
||||
expect(mockUpsertStackUpdateStatus).toHaveBeenCalled();
|
||||
@@ -1070,7 +1070,7 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImage(service, false);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockClearStackUpdateStatus).toHaveBeenCalledWith(1, 'stackB');
|
||||
});
|
||||
@@ -1080,7 +1080,7 @@ services:
|
||||
const service = ImageUpdateService.getInstance();
|
||||
stubCheckImage(service, false);
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
expect(mockClearStackUpdateStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -1125,7 +1125,7 @@ services:
|
||||
const checkImageSpy = vi.fn().mockResolvedValue({ hasUpdate: false });
|
||||
(service as any).checkImage = checkImageSpy;
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
// Should check both the compose image and the container image
|
||||
const checkedImages = checkImageSpy.mock.calls.map((c: any[]) => c[1]);
|
||||
@@ -1143,7 +1143,7 @@ services:
|
||||
const checkImageSpy = vi.fn().mockResolvedValue({ hasUpdate: false });
|
||||
(service as any).checkImage = checkImageSpy;
|
||||
|
||||
await (service as any).checkNode(1, 'local', fakeDb());
|
||||
await (service as any).checkNode(1, fakeDb());
|
||||
|
||||
const checkedImages = checkImageSpy.mock.calls.map((c: any[]) => c[1]);
|
||||
expect(checkedImages).not.toContain('someapp:v2');
|
||||
|
||||
@@ -895,7 +895,12 @@ describe('MonitorService - breach state machine', () => {
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('CPU'), { stackName: 'my-stack', actor: 'system:monitor' });
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'warning',
|
||||
'monitor_alert',
|
||||
'The **CPU usage** for **my-stack** has exceeded your threshold of **80%** (Currently: 90%).',
|
||||
{ stackName: 'my-stack', actor: 'system:monitor' },
|
||||
);
|
||||
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
|
||||
});
|
||||
|
||||
|
||||
@@ -2159,3 +2159,97 @@ describe('SchedulerService - delete_after_run', () => {
|
||||
expect(mockUpdateScheduledTask).toHaveBeenCalledWith(300, expect.objectContaining({ last_status: 'success' }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── Node-neutral failure alerts (local vs remote) ──────────────────────
|
||||
|
||||
describe('SchedulerService - node-neutral failure alerts', () => {
|
||||
function makeContainerTask(overrides: Partial<ScheduledTask> & { id: number; name: string }) {
|
||||
return {
|
||||
action: 'restart',
|
||||
target_type: 'container',
|
||||
target_id: 'web',
|
||||
cron_expression: '0 * * * *',
|
||||
enabled: true,
|
||||
node_id: 1,
|
||||
created_by: 'admin',
|
||||
last_status: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it('dispatches local offline failure without the seed node name', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeContainerTask({ id: 400, name: 'offline-local', node_id: 1 }));
|
||||
mockGetNode.mockReturnValue({ id: 1, name: 'Local', type: 'local', status: 'offline' });
|
||||
|
||||
await SchedulerService.getInstance().triggerTask(400);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'system',
|
||||
'Scheduled task "offline-local" (restart) failed: Target node is offline',
|
||||
{ stackName: 'web', actor: 'system:scheduler' },
|
||||
);
|
||||
});
|
||||
|
||||
it('dispatches remote offline failure with the authoritative node name', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeContainerTask({ id: 401, name: 'offline-remote', node_id: 2 }));
|
||||
mockGetNode.mockReturnValue({ id: 2, name: 'sencho-test-02', type: 'remote', status: 'offline' });
|
||||
|
||||
await SchedulerService.getInstance().triggerTask(401);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'system',
|
||||
'Scheduled task "offline-remote" (restart) failed: Target node "sencho-test-02" is offline',
|
||||
{ stackName: 'web', actor: 'system:scheduler' },
|
||||
);
|
||||
});
|
||||
|
||||
it('dispatches local missing-container failure with this-node wording', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeContainerTask({ id: 402, name: 'missing-local', node_id: 1 }));
|
||||
mockGetNode.mockReturnValue({ id: 1, name: 'Local', type: 'local', status: 'online' });
|
||||
mockFindContainerByName.mockResolvedValue(null);
|
||||
|
||||
await SchedulerService.getInstance().triggerTask(402);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'system',
|
||||
'Scheduled task "missing-local" (restart) failed: Container "web" not found on this node. It may have been renamed or removed.',
|
||||
{ stackName: 'web', actor: 'system:scheduler' },
|
||||
);
|
||||
});
|
||||
|
||||
it('dispatches remote missing-container failure with the authoritative node name', async () => {
|
||||
mockGetScheduledTask.mockReturnValue(makeContainerTask({ id: 403, name: 'missing-remote', node_id: 2 }));
|
||||
mockGetNode.mockReturnValue({ id: 2, name: 'sencho-test-02', type: 'remote', status: 'online' });
|
||||
mockGetProxyTarget.mockReturnValue({
|
||||
apiUrl: 'http://remote:1852',
|
||||
apiToken: 'test-token',
|
||||
});
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ([]),
|
||||
});
|
||||
vi.stubGlobal('fetch', mockFetch);
|
||||
|
||||
await SchedulerService.getInstance().triggerTask(403);
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'system',
|
||||
'Scheduled task "missing-remote" (restart) failed: Container "web" not found on node "sencho-test-02". It may have been renamed or removed.',
|
||||
{ stackName: 'web', actor: 'system:scheduler' },
|
||||
);
|
||||
});
|
||||
|
||||
it('missing node lookup keeps the id fallback, not this-node wording', () => {
|
||||
mockGetNode.mockReturnValue(undefined);
|
||||
const svc = SchedulerService.getInstance() as unknown as {
|
||||
containerNotFoundMessage: (containerName: string, nodeId: number) => string;
|
||||
};
|
||||
expect(svc.containerNotFoundMessage('web', 99)).toBe(
|
||||
'Container "web" not found on node "99". It may have been renamed or removed.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -29,6 +29,11 @@ function diagnosticLog(message: string, fields: Record<string, string | number |
|
||||
console.info(`[BlueprintReconciler:diag] ${message}`, safeFields);
|
||||
}
|
||||
|
||||
/** Local seed name must not reach fleet alerts; remote roster names stay. */
|
||||
function nodeLocationClause(node: Node): string {
|
||||
return node.type === 'local' ? 'on this node' : `on node "${node.name}"`;
|
||||
}
|
||||
|
||||
export interface ReconcileDecision {
|
||||
deploy: Node[];
|
||||
withdraw: Node[];
|
||||
@@ -287,7 +292,7 @@ export class BlueprintReconciler {
|
||||
notifications.dispatchAlert(
|
||||
'warning',
|
||||
'blueprint_drift_detected',
|
||||
`Blueprint "${blueprint.name}" drifted on node "${node.name}": ${reason}`,
|
||||
`Blueprint "${blueprint.name}" drifted ${nodeLocationClause(node)}: ${reason}`,
|
||||
{ stackName: blueprint.name, actor: 'system:blueprint' },
|
||||
);
|
||||
return;
|
||||
@@ -301,7 +306,7 @@ export class BlueprintReconciler {
|
||||
notifications.dispatchAlert(
|
||||
'warning',
|
||||
'blueprint_drift_detected',
|
||||
`Blueprint "${blueprint.name}" lost its marker on node "${node.name}"; auto-fix declined to avoid stomping unowned data. Reason: ${reason}`,
|
||||
`Blueprint "${blueprint.name}" lost its marker ${nodeLocationClause(node)}; auto-fix declined to avoid stomping unowned data. Reason: ${reason}`,
|
||||
{ stackName: blueprint.name, actor: 'system:blueprint' },
|
||||
);
|
||||
return;
|
||||
@@ -318,7 +323,7 @@ export class BlueprintReconciler {
|
||||
notifications.dispatchAlert(
|
||||
'error',
|
||||
'blueprint_drift_correction_failed',
|
||||
`Auto-fix for "${blueprint.name}" on node "${node.name}" failed: ${result.error ?? 'unknown error'}`,
|
||||
`Auto-fix for "${blueprint.name}" ${nodeLocationClause(node)} failed: ${result.error ?? 'unknown error'}`,
|
||||
{ stackName: blueprint.name, actor: 'system:blueprint' },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -721,7 +721,7 @@ export class DockerEventService {
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Notification wrappers (prefix with node name for multi-node clarity)
|
||||
// Notification wrappers (node-neutral bodies; hub bell badges remotes)
|
||||
// ========================================================================
|
||||
|
||||
private buildAlertOptions(
|
||||
@@ -735,19 +735,15 @@ export class DockerEventService {
|
||||
}
|
||||
|
||||
private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise<void> {
|
||||
return this.notifier.dispatchAlert('error', category, this.prefix(message), this.buildAlertOptions(stackName, containerName, systemOnly));
|
||||
return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly));
|
||||
}
|
||||
|
||||
private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('warning', category, this.prefix(message), this.buildAlertOptions(stackName, containerName));
|
||||
return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
}
|
||||
|
||||
private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('info', category, this.prefix(message), this.buildAlertOptions(stackName, containerName));
|
||||
}
|
||||
|
||||
private prefix(message: string): string {
|
||||
return `[Node: ${this.nodeName}] ${message}`;
|
||||
return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
|
||||
@@ -523,7 +523,7 @@ export class ImageUpdateService {
|
||||
for (const node of db.getNodes()) {
|
||||
if (node.type !== 'local' || !node.id) continue;
|
||||
try {
|
||||
await this.checkNode(node.id, node.name, db);
|
||||
await this.checkNode(node.id, db);
|
||||
} catch (e) {
|
||||
console.error(`[ImageUpdateService] Error on node ${node.name}:`, e);
|
||||
}
|
||||
@@ -536,7 +536,7 @@ export class ImageUpdateService {
|
||||
}
|
||||
}
|
||||
|
||||
private async checkNode(nodeId: number, nodeName: string, db: DatabaseService) {
|
||||
private async checkNode(nodeId: number, db: DatabaseService) {
|
||||
const docker = DockerController.getInstance(nodeId);
|
||||
const fs = FileSystemService.getInstance(nodeId);
|
||||
const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
|
||||
@@ -708,7 +708,8 @@ export class ImageUpdateService {
|
||||
await notifier.dispatchAlert(
|
||||
'info',
|
||||
'image_update_available',
|
||||
`[Node: ${nodeName}] Stack "${stackName}" has image updates available.`,
|
||||
// Node-neutral body: the hub bell badge attributes remotes.
|
||||
`Stack "${stackName}" has image updates available.`,
|
||||
{ stackName, actor: 'system:image-update' },
|
||||
);
|
||||
} catch (e) {
|
||||
@@ -722,7 +723,7 @@ export class ImageUpdateService {
|
||||
level: 'error',
|
||||
category: 'system',
|
||||
message: sanitizeNotificationMessage(
|
||||
`[Node: ${nodeName}] Failed to notify about image updates for stack "${stackName}": ${getErrorMessage(e, String(e))}`,
|
||||
`Failed to notify about image updates for stack "${stackName}": ${getErrorMessage(e, String(e))}`,
|
||||
{ composeDir: NodeRegistry.getInstance().getComposeDir(localNodeId) },
|
||||
),
|
||||
timestamp: Date.now(),
|
||||
|
||||
@@ -765,7 +765,8 @@ export class MonitorService {
|
||||
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
|
||||
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
|
||||
|
||||
const message = `[Node: ${node.name}] The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
|
||||
// Node-neutral body: the hub bell badge attributes remotes.
|
||||
const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
|
||||
|
||||
console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`);
|
||||
await NotificationService.getInstance().dispatchAlert(
|
||||
|
||||
@@ -298,7 +298,15 @@ export class SchedulerService {
|
||||
if (task.node_id != null && task.action !== 'snapshot') {
|
||||
const node = db.getNode(task.node_id);
|
||||
if (!node) throw new Error(`Target node (id=${task.node_id}) no longer exists`);
|
||||
if (node.status === 'offline') throw new Error(`Target node "${node.name}" is offline`);
|
||||
if (node.status === 'offline') {
|
||||
// Local seed name ("Local") must not reach fleet-aggregated alerts;
|
||||
// remote roster names stay for diagnosis.
|
||||
throw new Error(
|
||||
node.type === 'local'
|
||||
? 'Target node is offline'
|
||||
: `Target node "${node.name}" is offline`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) console.log(`[SchedulerService:debug] Task ${task.id} pre-checks passed, executing ${task.action}`);
|
||||
@@ -813,8 +821,12 @@ export class SchedulerService {
|
||||
* segment.
|
||||
*/
|
||||
private containerNotFoundMessage(containerName: string, nodeId: number): string {
|
||||
const nodeName = NodeRegistry.getInstance().getNode(nodeId)?.name ?? String(nodeId);
|
||||
return `Container "${containerName}" not found on node "${nodeName}". It may have been renamed or removed.`;
|
||||
const node = NodeRegistry.getInstance().getNode(nodeId);
|
||||
// Only an explicit local node is neutralized; missing lookup keeps the id fallback.
|
||||
const location = node?.type === 'local'
|
||||
? 'on this node'
|
||||
: `on node "${node?.name ?? String(nodeId)}"`;
|
||||
return `Container "${containerName}" not found ${location}. It may have been renamed or removed.`;
|
||||
}
|
||||
|
||||
/** Hub target tag used for startsWith rethrow guards (no trailing detail). */
|
||||
|
||||
@@ -35,7 +35,7 @@ Use the Generic Webhook tab when you have your own receiver: a Mattermost or Tea
|
||||
```json
|
||||
{
|
||||
"level": "warning",
|
||||
"message": "[Node: Local] The cpu_percent for plex has exceeded your threshold of 80% (Currently: 91%).",
|
||||
"message": "The **CPU usage** for **plex** has exceeded your threshold of **80%** (Currently: 91%).",
|
||||
"timestamp": "2026-05-08T22:14:09.812Z",
|
||||
"source": "sencho"
|
||||
}
|
||||
@@ -287,13 +287,13 @@ The bell aggregates across the entire fleet by hitting `/api/notifications` agai
|
||||
|
||||
Per-stack alert rules and channel configuration are **stored on the node where the stack runs**. To configure a rule on a remote, switch the active node via the picker, open the stack's **Monitor** sheet, and the form's POST is forwarded to the remote.
|
||||
|
||||
Crash detection runs only on local Docker; remote nodes run their own copy of `DockerEventService` and emit through the proxy. Each emitted message is prefixed with `[Node: <nodeName>]` so the source is unambiguous in the channel.
|
||||
Crash detection runs only on local Docker; remote nodes run their own copy of `DockerEventService` and emit through the proxy. On a multi-node fleet the bell attributes remotes with a node-name badge rather than embedding the satellite-local name in the message. External channel payloads (Slack, Discord, custom webhook) receive the same sanitized body with no structured source-node id.
|
||||
|
||||
Switching the active node only affects per-stack rule editing and the Settings panels. The bell aggregates every node regardless of which one is active.
|
||||
|
||||
## Alerts emitted by the system
|
||||
|
||||
Sencho dispatches notifications from many code paths. The list below covers every emitter in current code, organized by source. Each message is prefixed with `[Node: <nodeName>]` when emitted by a node-scoped service.
|
||||
Sencho dispatches notifications from many code paths. The list below covers every emitter in current code, organized by source. Message bodies are node-neutral for local targets so fleet aggregation does not contradict the bell badge; remote-target names stay when a hub names an authoritative remote.
|
||||
|
||||
### Container crash and health
|
||||
|
||||
@@ -332,7 +332,7 @@ The entire host threshold evaluation can be silenced per node from **Settings ·
|
||||
|
||||
### Image update availability
|
||||
|
||||
`info`/`image_update_available`: `Stack "<name>" has image updates available.` Polled every 6 hours, with a 2-minute startup delay and a 2-minute cooldown on manual refresh. Notifies on **state transition only**; pre-existing `has_update` rows are backfilled silently on first run.
|
||||
`info`/`image_update_available`: `Stack "<name>" has image updates available.` By default, checks every two hours in interval mode, with a two-minute startup delay. The cadence can be changed or replaced with a cron schedule. Manual refresh carries a two-minute cooldown. Notifies on **state transition only**; the first run also emits catch-up notifications for stacks already known to have updates.
|
||||
|
||||
### Auto-update execution
|
||||
|
||||
@@ -362,14 +362,16 @@ See [Vulnerability scanning](/features/vulnerability-scanning).
|
||||
- `error`/`system`: `Scheduled task "<name>" (<action>) failed: <err>`
|
||||
- `info`/`system` recovery: `Scheduled task "<name>" (<action>) recovered successfully`
|
||||
|
||||
Local-target failure reasons stay node-neutral (`Target node is offline`, `Container "<n>" not found on this node. ...`). When the task targets a remote node, the failure reason keeps that node's roster name.
|
||||
|
||||
### Recovery Vault upload failure
|
||||
|
||||
`warning`/`system`: `Cloud backup failed for scheduled snapshot <id>: <message>`. See [Fleet backups](/features/fleet-backups).
|
||||
|
||||
### Blueprints
|
||||
|
||||
- `warning`/`blueprint_drift_detected`: `Blueprint "<n>" drifted on node "<n2>": <reason>` (suggest mode), with stateful-safeguard variants when enforce mode declines to redeploy.
|
||||
- `error`/`blueprint_drift_correction_failed`: `Auto-fix for "<n>" on node "<n2>" failed: <err>`.
|
||||
- `warning`/`blueprint_drift_detected`: `Blueprint "<n>" drifted on this node: <reason>` for a local target, or `Blueprint "<n>" drifted on node "<remote>": <reason>` for a remote. Stateful marker-loss uses the same local/remote clause.
|
||||
- `error`/`blueprint_drift_correction_failed`: `Auto-fix for "<n>" on this node failed: <err>` (local) or `Auto-fix for "<n>" on node "<remote>" failed: <err>` (remote).
|
||||
|
||||
See [Blueprints](/features/blueprint-model).
|
||||
|
||||
@@ -459,6 +461,6 @@ Switching the active node tears down per-stack rule editors and reloads channel
|
||||
Fleet and the notification path share the same version lookup. When a newer published release is in that cache, Monitor dispatches a single `info`/`node_update_available` alert and records it under `last_sencho_update_notified_version`, so each release produces one alert per node. If you already upgraded to (or past) that version, the dedup self-heals and the alert does not fire. Check **Settings · Notifications · Mute Rules** for a rule that mutes `node_update_available` in the bell. A newly published release may take up to the cache TTL (about 30 minutes when published, or about 3 minutes while registry publish is still pending) before Fleet and the bell both observe it.
|
||||
</Accordion>
|
||||
<Accordion title="A stack shows the blue update indicator but no notification was received">
|
||||
The image-update service emits on **state transitions only**. The first run of the service backfills a flag against rows that already had `has_update: true` so they do not all re-fire on first install. Once the flag is set, only the false-to-true transition triggers an alert. If a stack was already showing the indicator before backfill ran, a notification will only appear the next time it goes from up-to-date back to behind.
|
||||
The image-update service emits on **state transitions only** after its first run. The first run also emits catch-up notifications for stacks already known to have updates, then sets a backfill flag. Once that flag is set, only a false-to-true transition triggers another alert. If you dismissed the catch-up notification, a new one appears the next time the stack goes from up-to-date back to behind.
|
||||
</Accordion>
|
||||
</AccordionGroup>
|
||||
|
||||
@@ -193,6 +193,27 @@ describe('useNotifications', () => {
|
||||
await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 2));
|
||||
});
|
||||
|
||||
it('stamps roster nodeName on remote REST notifications without rewriting the body', async () => {
|
||||
const remote = makeRemoteNode('online', { id: 2, name: 'sencho-sat-qa' });
|
||||
const neutralBody =
|
||||
'Scheduled task "qa-missing-container" (restart) failed: Container "web" not found on this node. It may have been renamed or removed.';
|
||||
(apiFetch as ReturnType<typeof vi.fn>).mockResolvedValue({ ok: true, json: async () => [] });
|
||||
(fetchForNode as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => [{ id: 9, level: 'error', message: neutralBody, timestamp: 2000, is_read: 0 }],
|
||||
});
|
||||
|
||||
const { result } = renderHook(() =>
|
||||
useNotifications({ nodes: [localNode, remote], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }),
|
||||
);
|
||||
|
||||
await waitFor(() => expect(result.current.notifications).toHaveLength(1));
|
||||
expect(result.current.notifications[0].message).toBe(neutralBody);
|
||||
expect(result.current.notifications[0].nodeId).toBe(2);
|
||||
expect(result.current.notifications[0].nodeName).toBe('sencho-sat-qa');
|
||||
expect(result.current.notifications[0].message).not.toMatch(/\[Node:|Local/);
|
||||
});
|
||||
|
||||
it('subscribes to the online node and skips the offline one in a mixed fleet', async () => {
|
||||
renderHook(() =>
|
||||
useNotifications({
|
||||
|
||||
Reference in New Issue
Block a user