mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-24 08:57:25 +00:00
fix(alerts): harden with security fixes, design compliance, and test coverage (#570)
* fix(alerts): harden with security fixes, design compliance, and test coverage Add authMiddleware to all alert endpoints, validate notification test dispatch inputs, fix restart_count metric via Docker inspect, correct network metric units, replace any types with DockerContainerStats interface, add webhook timeouts and dispatch error tracking. Frontend: migrate Select to Combobox, add ScrollArea and delete confirmation AlertDialog, fix icon strokeWidth to 1.5. Add update availability notifications for both Sencho version updates (6-hour check in MonitorService) and stack image updates (state transition detection in ImageUpdateService). Extract shared version fetch logic into utils/version-check.ts. Add diagnostic logging gated behind developer_mode for MonitorService breach state machine and NotificationService dispatch routing. Tests: 24 new alert API integration tests, restart_count and version check unit tests (688 total passing). Docs updated with HTTPS requirement, update notifications section, and troubleshooting guide. * fix(alerts): remove unused TEST_USERNAME import in alerts-api tests
This commit is contained in:
@@ -0,0 +1,282 @@
|
||||
/**
|
||||
* Integration tests for Alert CRUD endpoints, notification test dispatch
|
||||
* validation, and auth enforcement on all alert/notification routes.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import bcrypt from 'bcrypt';
|
||||
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let authCookie: string;
|
||||
let viewerCookie: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
|
||||
// Mock LicenseService so Admiral-gated routes are accessible
|
||||
const { LicenseService } = await import('../services/LicenseService');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getSeatLimits').mockReturnValue({ maxAdmins: null, maxViewers: null });
|
||||
|
||||
({ app } = await import('../index'));
|
||||
authCookie = await loginAsTestAdmin(app);
|
||||
|
||||
// Create a viewer user for non-admin tests
|
||||
const viewerHash = await bcrypt.hash('viewerpass', 1);
|
||||
DatabaseService.getInstance().addUser({ username: 'viewer', password_hash: viewerHash, role: 'viewer' });
|
||||
const viewerRes = await request(app)
|
||||
.post('/api/auth/login')
|
||||
.send({ username: 'viewer', password: 'viewerpass' });
|
||||
const cookies = viewerRes.headers['set-cookie'] as string | string[];
|
||||
viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
// --- GET /api/alerts ---
|
||||
|
||||
describe('GET /api/alerts', () => {
|
||||
it('rejects unauthenticated requests with 401', async () => {
|
||||
const res = await request(app).get('/api/alerts');
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('returns empty array when no alerts exist', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/alerts')
|
||||
.set('Cookie', authCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual([]);
|
||||
});
|
||||
|
||||
it('filters alerts by stackName query param', async () => {
|
||||
// Seed two alerts for different stacks
|
||||
const db = DatabaseService.getInstance();
|
||||
db.addStackAlert({ stack_name: 'web', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
|
||||
db.addStackAlert({ stack_name: 'api', metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 });
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/alerts?stackName=web')
|
||||
.set('Cookie', authCookie);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.length).toBe(1);
|
||||
expect(res.body[0].stack_name).toBe('web');
|
||||
});
|
||||
});
|
||||
|
||||
// --- POST /api/alerts ---
|
||||
|
||||
describe('POST /api/alerts', () => {
|
||||
it('rejects unauthenticated requests with 401', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('rejects non-admin users with 403', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', viewerCookie)
|
||||
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
|
||||
expect(res.status).toBe(403);
|
||||
});
|
||||
|
||||
it('creates alert and returns 201 with created resource', async () => {
|
||||
const payload = {
|
||||
stack_name: 'new-stack',
|
||||
metric: 'memory_percent',
|
||||
operator: '>=',
|
||||
threshold: 85,
|
||||
duration_mins: 10,
|
||||
cooldown_mins: 30,
|
||||
};
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send(payload);
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(res.body.id).toBeDefined();
|
||||
expect(res.body.stack_name).toBe('new-stack');
|
||||
expect(res.body.metric).toBe('memory_percent');
|
||||
expect(res.body.threshold).toBe(85);
|
||||
});
|
||||
|
||||
it('validates required fields and returns 400 for missing data', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ stack_name: 'test' }); // missing metric, operator, threshold, etc.
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe('Invalid alert data');
|
||||
});
|
||||
|
||||
it('rejects invalid metric values', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ stack_name: 'test', metric: 'invalid_metric', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects negative threshold', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: -1, duration_mins: 5, cooldown_mins: 60 });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects empty stack_name', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ stack_name: '', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects stack_name exceeding 255 characters', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ stack_name: 'a'.repeat(256), metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects duration_mins exceeding 1440', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 1441, cooldown_mins: 60 });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects cooldown_mins exceeding 10080', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 10081 });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects invalid operator', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/alerts')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '!=', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
// --- DELETE /api/alerts/:id ---
|
||||
|
||||
describe('DELETE /api/alerts/:id', () => {
|
||||
it('rejects unauthenticated requests with 401', async () => {
|
||||
const res = await request(app).delete('/api/alerts/1');
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('rejects non-admin users with 403', async () => {
|
||||
const res = await request(app)
|
||||
.delete('/api/alerts/1')
|
||||
.set('Cookie', viewerCookie);
|
||||
expect(res.status).toBe(403);
|
||||
});
|
||||
|
||||
it('deletes existing alert rule', async () => {
|
||||
// Create an alert to delete
|
||||
const created = DatabaseService.getInstance().addStackAlert({
|
||||
stack_name: 'delete-me',
|
||||
metric: 'cpu_percent',
|
||||
operator: '>',
|
||||
threshold: 90,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 0,
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.delete(`/api/alerts/${created.id}`)
|
||||
.set('Cookie', authCookie);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// --- POST /api/notifications/test ---
|
||||
|
||||
describe('POST /api/notifications/test', () => {
|
||||
it('rejects unauthenticated requests with 401', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/notifications/test')
|
||||
.send({ type: 'discord', url: 'https://discord.com/api/webhooks/123/abc' });
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('rejects non-admin users with 403', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/notifications/test')
|
||||
.set('Cookie', viewerCookie)
|
||||
.send({ type: 'discord', url: 'https://discord.com/api/webhooks/123/abc' });
|
||||
expect(res.status).toBe(403);
|
||||
});
|
||||
|
||||
it('rejects invalid type with 400', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/notifications/test')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ type: 'telegram', url: 'https://example.com' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toContain('discord, slack, or webhook');
|
||||
});
|
||||
|
||||
it('rejects missing type with 400', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/notifications/test')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ url: 'https://example.com' });
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects non-HTTPS url with 400', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/notifications/test')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ type: 'discord', url: 'http://example.com' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toContain('HTTPS');
|
||||
});
|
||||
|
||||
it('rejects missing url with 400', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/notifications/test')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ type: 'discord' });
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects malformed url with 400', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/notifications/test')
|
||||
.set('Cookie', authCookie)
|
||||
.send({ type: 'discord', url: 'https://' });
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
@@ -10,9 +10,11 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
|
||||
mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs,
|
||||
mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState,
|
||||
mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream,
|
||||
mockGetContainerRestartCount,
|
||||
mockDispatchAlert,
|
||||
mockCurrentLoad, mockMem, mockFsSize,
|
||||
mockExecAsync,
|
||||
mockFetchLatestSenchoVersion,
|
||||
} = vi.hoisted(() => ({
|
||||
mockGetGlobalSettings: vi.fn().mockReturnValue({}),
|
||||
mockGetNodes: vi.fn().mockReturnValue([]),
|
||||
@@ -27,11 +29,13 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
|
||||
mockGetRunningContainers: vi.fn().mockResolvedValue([]),
|
||||
mockGetAllContainers: vi.fn().mockResolvedValue([]),
|
||||
mockGetContainerStatsStream: vi.fn().mockResolvedValue('{}'),
|
||||
mockGetContainerRestartCount: vi.fn().mockResolvedValue(0),
|
||||
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
|
||||
mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }),
|
||||
mockMem: vi.fn().mockResolvedValue({ used: 4e9, total: 16e9 }),
|
||||
mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
|
||||
mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }),
|
||||
mockFetchLatestSenchoVersion: vi.fn().mockRejectedValue(new Error('not configured')),
|
||||
}));
|
||||
|
||||
vi.mock('../services/DatabaseService', () => ({
|
||||
@@ -57,10 +61,15 @@ vi.mock('../services/DockerController', () => ({
|
||||
getRunningContainers: mockGetRunningContainers,
|
||||
getAllContainers: mockGetAllContainers,
|
||||
getContainerStatsStream: mockGetContainerStatsStream,
|
||||
getContainerRestartCount: mockGetContainerRestartCount,
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../utils/version-check', () => ({
|
||||
fetchLatestSenchoVersion: (...args: unknown[]) => mockFetchLatestSenchoVersion(...args),
|
||||
}));
|
||||
|
||||
vi.mock('../services/NotificationService', () => ({
|
||||
NotificationService: {
|
||||
getInstance: () => ({
|
||||
@@ -522,3 +531,135 @@ describe('MonitorService - isProcessing guard', () => {
|
||||
expect((svc as any).isProcessing).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── restart_count metric ──────────────────────────────────────────────
|
||||
|
||||
describe('MonitorService - restart_count metric', () => {
|
||||
function setupRestartScenario(restartCount: number, hasRestartRule: boolean) {
|
||||
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
|
||||
mockGetRunningContainers.mockResolvedValue([{
|
||||
Id: 'c1',
|
||||
Labels: { 'com.docker.compose.project': 'my-stack' },
|
||||
}]);
|
||||
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
|
||||
cpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000, online_cpus: 1 },
|
||||
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
|
||||
memory_stats: { usage: 100e6, limit: 1e9 },
|
||||
}));
|
||||
mockGetContainerRestartCount.mockResolvedValue(restartCount);
|
||||
const alerts = [];
|
||||
if (hasRestartRule) {
|
||||
alerts.push({
|
||||
id: 100,
|
||||
stack_name: 'my-stack',
|
||||
metric: 'restart_count',
|
||||
operator: '>',
|
||||
threshold: 3,
|
||||
duration_mins: 0,
|
||||
cooldown_mins: 60,
|
||||
last_fired_at: 0,
|
||||
});
|
||||
}
|
||||
mockGetStackAlerts.mockReturnValue(alerts);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
}
|
||||
|
||||
it('fetches restart count from Docker when a restart_count rule exists', async () => {
|
||||
setupRestartScenario(5, true);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
|
||||
// restart_count=5 > threshold=3, should fire
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Restart count'), 'my-stack');
|
||||
});
|
||||
|
||||
it('skips Docker inspect when no restart_count rules exist', async () => {
|
||||
setupRestartScenario(5, false);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockGetContainerRestartCount).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not fire when restart count is below threshold', async () => {
|
||||
setupRestartScenario(2, true);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('Restart count'), expect.anything());
|
||||
});
|
||||
});
|
||||
|
||||
// ── Sencho version update check ───────────────────────────────────────
|
||||
|
||||
describe('MonitorService - Sencho version check', () => {
|
||||
it('dispatches notification when newer version available', async () => {
|
||||
// Set current version
|
||||
process.env.npm_package_version = '0.45.0';
|
||||
mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
|
||||
mockGetSystemState.mockReturnValue(null); // No previous notification
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetNodes.mockReturnValue([]);
|
||||
mockGetStackAlerts.mockReturnValue([]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
// Reset the version check timer so it runs immediately
|
||||
(svc as any).lastVersionCheckAt = 0;
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
|
||||
expect(mockSetSystemState).toHaveBeenCalledWith('last_sencho_update_notified_version', '0.46.0');
|
||||
});
|
||||
|
||||
it('does not re-notify for the same version', async () => {
|
||||
process.env.npm_package_version = '0.45.0';
|
||||
mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
|
||||
mockGetSystemState.mockReturnValue('0.46.0'); // Already notified for this version
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetNodes.mockReturnValue([]);
|
||||
mockGetStackAlerts.mockReturnValue([]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
(svc as any).lastVersionCheckAt = 0;
|
||||
await (svc as any).evaluate();
|
||||
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
|
||||
});
|
||||
|
||||
it('handles version check failure gracefully', async () => {
|
||||
process.env.npm_package_version = '0.45.0';
|
||||
mockFetchLatestSenchoVersion.mockRejectedValue(new Error('Network down'));
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetNodes.mockReturnValue([]);
|
||||
mockGetStackAlerts.mockReturnValue([]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
(svc as any).lastVersionCheckAt = 0;
|
||||
|
||||
// Should not throw
|
||||
await expect((svc as any).evaluate()).resolves.toBeUndefined();
|
||||
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('available'));
|
||||
});
|
||||
|
||||
it('respects the 6-hour cooldown interval', async () => {
|
||||
process.env.npm_package_version = '0.45.0';
|
||||
mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
|
||||
mockGetSystemState.mockReturnValue(null);
|
||||
mockGetGlobalSettings.mockReturnValue({});
|
||||
mockGetNodes.mockReturnValue([]);
|
||||
mockGetStackAlerts.mockReturnValue([]);
|
||||
|
||||
const svc = MonitorService.getInstance();
|
||||
// Simulate the check ran 1 hour ago (within 6-hour window)
|
||||
(svc as any).lastVersionCheckAt = Date.now() - 1 * 60 * 60 * 1000;
|
||||
await (svc as any).evaluate();
|
||||
|
||||
// fetchLatestSenchoVersion should not have been called since we're within cooldown
|
||||
expect(mockFetchLatestSenchoVersion).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
+22
-53
@@ -59,6 +59,7 @@ function invalidateNodeCaches(nodeId: number): void {
|
||||
}
|
||||
|
||||
import { isDebugEnabled } from './utils/debug';
|
||||
import { fetchLatestSenchoVersion } from './utils/version-check';
|
||||
import { getErrorMessage } from './utils/errors';
|
||||
import { captureLocalNodeFiles, captureRemoteNodeFiles, SnapshotNodeData } from './utils/snapshot-capture';
|
||||
import { GlobalLogEntry, normalizeContainerName, parseLogTimestamp, detectLogLevel, demuxDockerLog } from './utils/log-parsing';
|
||||
@@ -1285,50 +1286,8 @@ const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull
|
||||
const LATEST_VERSION_CACHE_KEY = 'latest-version';
|
||||
const LATEST_VERSION_CACHE_TTL = 30 * 60 * 1000; // 30 minutes
|
||||
|
||||
async function fetchFromGitHub(): Promise<string | null> {
|
||||
const res = await fetch('https://api.github.com/repos/AnsoCode/Sencho/releases/latest', {
|
||||
headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'Sencho' },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const data = await res.json() as { tag_name?: string };
|
||||
const tag = data.tag_name?.replace(/^v/, '') ?? null;
|
||||
return tag && semver.valid(tag) ? tag : null;
|
||||
}
|
||||
|
||||
async function fetchFromDockerHub(): Promise<string | null> {
|
||||
const res = await fetch(
|
||||
'https://hub.docker.com/v2/repositories/saelix/sencho/tags/?page_size=50&ordering=last_updated',
|
||||
{ headers: { 'User-Agent': 'Sencho' }, signal: AbortSignal.timeout(10000) },
|
||||
);
|
||||
if (!res.ok) return null;
|
||||
const data = await res.json() as { results?: { name: string }[] };
|
||||
const tags = (data.results ?? [])
|
||||
.map(t => t.name)
|
||||
.filter(n => semver.valid(n));
|
||||
if (tags.length === 0) return null;
|
||||
tags.sort(semver.rcompare);
|
||||
return tags[0];
|
||||
}
|
||||
|
||||
async function fetchLatestSenchoVersion(): Promise<string> {
|
||||
try {
|
||||
const gh = await fetchFromGitHub();
|
||||
if (gh) return gh;
|
||||
} catch (err) {
|
||||
// GitHub API fails for private repos or rate limits; try Docker Hub
|
||||
console.warn('[VersionCheck] GitHub fetch failed:', (err as Error).message);
|
||||
}
|
||||
try {
|
||||
const hub = await fetchFromDockerHub();
|
||||
if (hub) return hub;
|
||||
} catch (err) {
|
||||
console.warn('[VersionCheck] Docker Hub fetch failed:', (err as Error).message);
|
||||
}
|
||||
// Throw so CacheService falls back to a stale value if one exists,
|
||||
// and so we do not poison the cache with null.
|
||||
throw new Error('Both GitHub and Docker Hub version lookups failed');
|
||||
}
|
||||
// Version fetch logic lives in utils/version-check.ts; imported below for getLatestVersion().
|
||||
// fetchFromGitHub + fetchFromDockerHub + fetchLatestSenchoVersion are shared with MonitorService.
|
||||
|
||||
async function getLatestVersion(forceRefresh = false): Promise<string | null> {
|
||||
if (forceRefresh) {
|
||||
@@ -4354,7 +4313,7 @@ app.patch('/api/settings', async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/api/alerts', async (req: Request, res: Response) => {
|
||||
app.get('/api/alerts', authMiddleware, async (req: Request, res: Response) => {
|
||||
try {
|
||||
let stackName = req.query.stackName as string | undefined;
|
||||
if (Array.isArray(stackName)) stackName = stackName[0] as string;
|
||||
@@ -4375,7 +4334,7 @@ const AlertCreateSchema = z.object({
|
||||
cooldown_mins: z.coerce.number().int().min(0).max(10080),
|
||||
});
|
||||
|
||||
app.post('/api/alerts', async (req: Request, res: Response) => {
|
||||
app.post('/api/alerts', authMiddleware, async (req: Request, res: Response) => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
const parsed = AlertCreateSchema.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
@@ -4383,15 +4342,15 @@ app.post('/api/alerts', async (req: Request, res: Response) => {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
DatabaseService.getInstance().addStackAlert(parsed.data);
|
||||
res.json({ success: true });
|
||||
const created = DatabaseService.getInstance().addStackAlert(parsed.data);
|
||||
res.status(201).json(created);
|
||||
} catch (error) {
|
||||
console.error('Failed to add alert:', error);
|
||||
res.status(500).json({ error: 'Failed to add alert' });
|
||||
}
|
||||
});
|
||||
|
||||
app.delete('/api/alerts/:id', async (req: Request, res: Response) => {
|
||||
app.delete('/api/alerts/:id', authMiddleware, async (req: Request, res: Response) => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
try {
|
||||
const id = parseInt(req.params.id as string, 10);
|
||||
@@ -4439,21 +4398,31 @@ app.delete('/api/notifications', authMiddleware, async (req: Request, res: Respo
|
||||
}
|
||||
});
|
||||
|
||||
const NOTIFICATION_CHANNEL_TYPES = ['discord', 'slack', 'webhook'] as const;
|
||||
|
||||
app.post('/api/notifications/test', authMiddleware, async (req: Request, res: Response) => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
try {
|
||||
const { type, url } = req.body;
|
||||
if (!type || !NOTIFICATION_CHANNEL_TYPES.includes(type)) {
|
||||
res.status(400).json({ error: 'type must be discord, slack, or webhook' });
|
||||
return;
|
||||
}
|
||||
if (!url || typeof url !== 'string' || !url.startsWith('https://')) {
|
||||
res.status(400).json({ error: 'url must be a valid HTTPS URL' });
|
||||
return;
|
||||
}
|
||||
try { new URL(url); } catch { res.status(400).json({ error: 'url is not a valid URL' }); return; }
|
||||
await NotificationService.getInstance().testDispatch(type, url);
|
||||
res.json({ success: true });
|
||||
} catch (error: any) {
|
||||
res.status(500).json({ error: 'Test failed', details: error.message });
|
||||
} catch (error: unknown) {
|
||||
const msg = error instanceof Error ? error.message : String(error);
|
||||
res.status(500).json({ error: 'Test failed', details: msg });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Notification Routes (Admiral) ---
|
||||
|
||||
const NOTIFICATION_CHANNEL_TYPES = ['discord', 'slack', 'webhook'] as const;
|
||||
|
||||
app.get('/api/notification-routes', authMiddleware, (req: Request, res: Response): void => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
if (!requireAdmiral(req, res)) return;
|
||||
|
||||
@@ -655,6 +655,8 @@ export class DatabaseService {
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_notification_routes_priority ON notification_routes(priority);
|
||||
`);
|
||||
// Track external dispatch errors on notification records
|
||||
try { this.db.prepare('ALTER TABLE notification_history ADD COLUMN dispatch_error TEXT').run(); } catch { /* already exists */ }
|
||||
}
|
||||
|
||||
// --- Agents ---
|
||||
@@ -796,11 +798,11 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
public addStackAlert(alert: StackAlert): void {
|
||||
public addStackAlert(alert: StackAlert): StackAlert {
|
||||
const stmt = this.db.prepare(
|
||||
'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
|
||||
);
|
||||
stmt.run(
|
||||
const result = stmt.run(
|
||||
alert.stack_name,
|
||||
alert.metric,
|
||||
alert.operator,
|
||||
@@ -809,6 +811,7 @@ export class DatabaseService {
|
||||
alert.cooldown_mins,
|
||||
alert.last_fired_at || 0
|
||||
);
|
||||
return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert;
|
||||
}
|
||||
|
||||
public deleteStackAlert(id: number): void {
|
||||
@@ -866,6 +869,10 @@ export class DatabaseService {
|
||||
stmt.run();
|
||||
}
|
||||
|
||||
public updateNotificationDispatchError(id: number, error: string): void {
|
||||
this.db.prepare('UPDATE notification_history SET dispatch_error = ? WHERE id = ?').run(error, id);
|
||||
}
|
||||
|
||||
// --- Container Metrics ---
|
||||
|
||||
public addContainerMetric(metric: Omit<any, 'id'>): void {
|
||||
|
||||
@@ -1028,6 +1028,13 @@ class DockerController {
|
||||
return typeof stats === 'string' ? stats : JSON.stringify(stats);
|
||||
}
|
||||
|
||||
/** Return the cumulative restart count for a container via inspect(). */
|
||||
public async getContainerRestartCount(containerId: string): Promise<number> {
|
||||
const container = this.docker.getContainer(containerId);
|
||||
const info = await container.inspect();
|
||||
return info.RestartCount ?? 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exec into a container with full session isolation.
|
||||
* All state (exec instance, stream) lives in this closure - no singleton traps.
|
||||
|
||||
@@ -7,6 +7,7 @@ import { DatabaseService } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { RegistryService } from './RegistryService';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { NotificationService } from './NotificationService';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
|
||||
// ─── Image ref parsing ────────────────────────────────────────────────────────
|
||||
@@ -284,7 +285,7 @@ export class ImageUpdateService {
|
||||
for (const node of db.getNodes()) {
|
||||
if (node.type !== 'local' || !node.id) continue;
|
||||
try {
|
||||
await this.checkNode(node.id, db);
|
||||
await this.checkNode(node.id, node.name, db);
|
||||
} catch (e) {
|
||||
console.error(`[ImageUpdateService] Error on node ${node.name}:`, e);
|
||||
}
|
||||
@@ -297,7 +298,7 @@ export class ImageUpdateService {
|
||||
}
|
||||
}
|
||||
|
||||
private async checkNode(nodeId: number, db: DatabaseService) {
|
||||
private async checkNode(nodeId: number, nodeName: string, db: DatabaseService) {
|
||||
const docker = DockerController.getInstance(nodeId);
|
||||
const fs = FileSystemService.getInstance(nodeId);
|
||||
const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
|
||||
@@ -373,20 +374,45 @@ export class ImageUpdateService {
|
||||
await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS);
|
||||
}
|
||||
|
||||
// Read previous state to detect new updates for notifications
|
||||
const previousState = db.getStackUpdateStatus(nodeId);
|
||||
|
||||
// Write status for ALL stacks (including those with no pullable images)
|
||||
const now = Date.now();
|
||||
let updatesFound = 0;
|
||||
const newlyUpdated: string[] = [];
|
||||
for (const [stackName, images] of stackImages) {
|
||||
const hasUpdate = Array.from(images).some(img => imageUpdateMap.get(img)?.hasUpdate === true);
|
||||
if (hasUpdate) updatesFound++;
|
||||
if (hasUpdate) {
|
||||
updatesFound++;
|
||||
// Notify only on state transition: was false/absent, now true
|
||||
if (!previousState[stackName]) {
|
||||
newlyUpdated.push(stackName);
|
||||
}
|
||||
}
|
||||
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now);
|
||||
}
|
||||
|
||||
// Dispatch notifications for stacks that newly have updates
|
||||
if (newlyUpdated.length > 0) {
|
||||
const notifier = NotificationService.getInstance();
|
||||
for (const stackName of newlyUpdated) {
|
||||
try {
|
||||
await notifier.dispatchAlert(
|
||||
'info',
|
||||
`[Node: ${nodeName}] Stack "${stackName}" has image updates available.`,
|
||||
stackName,
|
||||
);
|
||||
} catch (e) {
|
||||
console.error(`[ImageUpdateService] Failed to dispatch update notification for "${stackName}":`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[ImageUpdateService] Node ${nodeId}: checked ${allImages.size} image(s), ${updatesFound} stack(s) with updates`);
|
||||
|
||||
// Prune stale entries for stacks no longer on disk
|
||||
const existing = db.getStackUpdateStatus(nodeId);
|
||||
for (const staleStack of Object.keys(existing)) {
|
||||
// Prune stale entries for stacks no longer on disk (reuse previousState to avoid extra DB read)
|
||||
for (const staleStack of Object.keys(previousState)) {
|
||||
if (!stackImages.has(staleStack)) {
|
||||
db.clearStackUpdateStatus(nodeId, staleStack);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
import si from 'systeminformation';
|
||||
import { exec } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import semver from 'semver';
|
||||
import DockerController from './DockerController';
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { NotificationService } from './NotificationService';
|
||||
import { isValidVersion } from './CapabilityRegistry';
|
||||
import { fetchLatestSenchoVersion } from '../utils/version-check';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
@@ -12,8 +16,8 @@ const getMetricDetails = (metric: string): { name: string, unit: string } => {
|
||||
case 'cpu_percent': return { name: 'CPU usage', unit: '%' };
|
||||
case 'memory_percent': return { name: 'Memory usage', unit: '%' };
|
||||
case 'memory_mb': return { name: 'Memory allocation', unit: ' MB' };
|
||||
case 'net_rx': return { name: 'Inbound network traffic', unit: ' MB/s' };
|
||||
case 'net_tx': return { name: 'Outbound network traffic', unit: ' MB/s' };
|
||||
case 'net_rx': return { name: 'Inbound network traffic', unit: ' MB' };
|
||||
case 'net_tx': return { name: 'Outbound network traffic', unit: ' MB' };
|
||||
case 'restart_count': return { name: 'Restart count', unit: ' restarts' };
|
||||
default: return { name: metric, unit: '' };
|
||||
}
|
||||
@@ -26,6 +30,25 @@ const getOperatorPhrase = (operator: string): string => {
|
||||
return `triggered the operator ${operator}`;
|
||||
};
|
||||
|
||||
/** Shape of the JSON returned by Docker container stats (stream: false). */
|
||||
interface DockerContainerStats {
|
||||
cpu_stats?: {
|
||||
cpu_usage?: { total_usage: number; percpu_usage?: number[] };
|
||||
system_cpu_usage?: number;
|
||||
online_cpus?: number;
|
||||
};
|
||||
precpu_stats?: {
|
||||
cpu_usage?: { total_usage: number };
|
||||
system_cpu_usage?: number;
|
||||
};
|
||||
memory_stats?: {
|
||||
usage?: number;
|
||||
limit?: number;
|
||||
stats?: { cache?: number };
|
||||
};
|
||||
networks?: Record<string, { rx_bytes: number; tx_bytes: number }>;
|
||||
}
|
||||
|
||||
interface AlertState {
|
||||
breachStartedAt: number; // timestamp when the rule first breached
|
||||
}
|
||||
@@ -51,6 +74,10 @@ export class MonitorService {
|
||||
private alertedCrashes = new Map<string, number>();
|
||||
private static readonly CRASH_ALERT_TTL_MS = 60 * 60 * 1000; // 1 hour
|
||||
|
||||
// Sencho version check cooldown (6 hours between external API calls)
|
||||
private lastVersionCheckAt = 0;
|
||||
private static readonly VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000;
|
||||
|
||||
private constructor() { }
|
||||
|
||||
public static getInstance(): MonitorService {
|
||||
@@ -62,6 +89,7 @@ export class MonitorService {
|
||||
|
||||
public start() {
|
||||
if (this.intervalId) return;
|
||||
if (isDebugEnabled()) console.log('[Monitor:diag] Starting evaluation loop (30s interval)');
|
||||
|
||||
// Run every 30 seconds
|
||||
this.intervalId = setInterval(() => {
|
||||
@@ -76,6 +104,7 @@ export class MonitorService {
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
this.intervalId = null;
|
||||
if (isDebugEnabled()) console.log('[Monitor:diag] Evaluation loop stopped');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -232,12 +261,43 @@ export class MonitorService {
|
||||
} catch (e) {
|
||||
console.error('Error checking docker janitor limits', e);
|
||||
}
|
||||
|
||||
// 4. Sencho version update check (runs once per VERSION_CHECK_INTERVAL_MS)
|
||||
if (Date.now() - this.lastVersionCheckAt > MonitorService.VERSION_CHECK_INTERVAL_MS) {
|
||||
this.lastVersionCheckAt = Date.now();
|
||||
try {
|
||||
const currentVersion = process.env.npm_package_version || '0.0.0';
|
||||
const latest = await fetchLatestSenchoVersion();
|
||||
if (isValidVersion(latest) && isValidVersion(currentVersion) && semver.gt(latest, currentVersion)) {
|
||||
const db = DatabaseService.getInstance();
|
||||
const stateKey = 'last_sencho_update_notified_version';
|
||||
const lastNotified = db.getSystemState(stateKey) || '';
|
||||
if (lastNotified !== latest) {
|
||||
const notifier = NotificationService.getInstance();
|
||||
await notifier.dispatchAlert('info',
|
||||
`Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`);
|
||||
db.setSystemState(stateKey, latest);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Network errors are expected; do not spam logs
|
||||
if (isDebugEnabled()) console.debug('[Monitor:diag] Sencho version check failed:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async evaluateStackAlerts(db: DatabaseService) {
|
||||
const alerts = db.getStackAlerts();
|
||||
const nodes = db.getNodes();
|
||||
|
||||
// Pre-group alerts by stack name to avoid O(containers * alerts) scanning
|
||||
const alertsByStack = new Map<string, typeof alerts>();
|
||||
for (const a of alerts) {
|
||||
const list = alertsByStack.get(a.stack_name);
|
||||
if (list) list.push(a);
|
||||
else alertsByStack.set(a.stack_name, [a]);
|
||||
}
|
||||
|
||||
for (const node of nodes) {
|
||||
if (!node.id) continue;
|
||||
// Remote nodes are self-monitoring - skip direct Docker access
|
||||
@@ -250,16 +310,24 @@ export class MonitorService {
|
||||
|
||||
try {
|
||||
const rawStats = await docker.getContainerStatsStream(container.Id);
|
||||
const stats = JSON.parse(rawStats);
|
||||
const stats: DockerContainerStats = JSON.parse(rawStats);
|
||||
|
||||
const usedMemory = (stats.memory_stats?.usage || 0) - (stats.memory_stats?.stats?.cache || 0);
|
||||
|
||||
// Only fetch restart count when at least one rule for this stack uses it
|
||||
const stackAlerts = alertsByStack.get(stackName) || [];
|
||||
const needsRestartCount = stackAlerts.some(a => a.metric === 'restart_count');
|
||||
const restartCount = needsRestartCount
|
||||
? await docker.getContainerRestartCount(container.Id)
|
||||
: 0;
|
||||
|
||||
const metrics = {
|
||||
cpu_percent: this.calculateCpuPercent(stats),
|
||||
memory_percent: this.calculateMemoryPercent(stats),
|
||||
memory_mb: Math.max(0, usedMemory) / (1024 * 1024),
|
||||
net_rx: this.calculateNetwork(stats, 'rx'),
|
||||
net_tx: this.calculateNetwork(stats, 'tx'),
|
||||
restart_count: 0 // Simplification since ContainerInfo doesn't have it natively
|
||||
restart_count: restartCount,
|
||||
};
|
||||
|
||||
db.addContainerMetric({
|
||||
@@ -272,7 +340,6 @@ export class MonitorService {
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const stackAlerts = alerts.filter(a => a.stack_name === stackName);
|
||||
for (const rule of stackAlerts) {
|
||||
const ruleId = rule.id!;
|
||||
const currentValue = metrics[rule.metric as keyof typeof metrics];
|
||||
@@ -284,6 +351,7 @@ export class MonitorService {
|
||||
if (isBreaching) {
|
||||
if (!this.activeBreaches.has(ruleId)) {
|
||||
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`);
|
||||
}
|
||||
|
||||
const breachState = this.activeBreaches.get(ruleId)!;
|
||||
@@ -305,6 +373,7 @@ export class MonitorService {
|
||||
|
||||
const message = `[Node: ${node.name}] The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
|
||||
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Duration met for rule ${ruleId}, dispatching alert`);
|
||||
await NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
message,
|
||||
@@ -313,11 +382,14 @@ export class MonitorService {
|
||||
|
||||
// Update last fired
|
||||
db.updateStackAlertLastFired(ruleId, Date.now());
|
||||
} else if (isDebugEnabled()) {
|
||||
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Rule isn't breaching anymore, reset tracker
|
||||
if (this.activeBreaches.has(ruleId)) {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`);
|
||||
this.activeBreaches.delete(ruleId);
|
||||
}
|
||||
}
|
||||
@@ -347,6 +419,7 @@ export class MonitorService {
|
||||
db.cleanupOldNotifications(isNaN(retentionDays) ? 30 : retentionDays);
|
||||
const auditRetentionDays = parseInt(settings['audit_retention_days'] || '90', 10);
|
||||
db.cleanupOldAuditLogs(isNaN(auditRetentionDays) ? 90 : auditRetentionDays);
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Cleanup: metrics ${isNaN(retentionHours) ? 24 : retentionHours}h, notifications ${isNaN(retentionDays) ? 30 : retentionDays}d, audit ${isNaN(auditRetentionDays) ? 90 : auditRetentionDays}d`);
|
||||
} catch (e) {
|
||||
console.error('MonitorService: failed to cleanup old data', e);
|
||||
}
|
||||
@@ -376,7 +449,7 @@ export class MonitorService {
|
||||
}
|
||||
}
|
||||
|
||||
private calculateCpuPercent(stats: any): number {
|
||||
private calculateCpuPercent(stats: DockerContainerStats): number {
|
||||
let cpuPercent = 0.0;
|
||||
if (!stats?.cpu_stats?.cpu_usage || !stats?.precpu_stats?.cpu_usage) return 0.0;
|
||||
|
||||
@@ -390,7 +463,7 @@ export class MonitorService {
|
||||
return cpuPercent;
|
||||
}
|
||||
|
||||
private calculateMemoryPercent(stats: any): number {
|
||||
private calculateMemoryPercent(stats: DockerContainerStats): number {
|
||||
if (!stats?.memory_stats?.usage || !stats?.memory_stats?.limit) return 0.0;
|
||||
|
||||
const used_memory = stats.memory_stats.usage - (stats.memory_stats.stats?.cache || 0);
|
||||
@@ -401,11 +474,12 @@ export class MonitorService {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
private calculateNetwork(stats: any, direction: 'rx' | 'tx'): number {
|
||||
private calculateNetwork(stats: DockerContainerStats, direction: 'rx' | 'tx'): number {
|
||||
let bytes = 0;
|
||||
if (stats.networks) {
|
||||
const key = direction === 'rx' ? 'rx_bytes' : 'tx_bytes';
|
||||
for (const iface in stats.networks) {
|
||||
bytes += stats.networks[iface][`${direction}_bytes`];
|
||||
bytes += stats.networks[iface][key];
|
||||
}
|
||||
}
|
||||
return bytes / (1024 * 1024); // Return in MB
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
import { DatabaseService, NotificationHistory } from './DatabaseService';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
/** Webhook timeout: 10 seconds per external dispatch call. */
|
||||
const WEBHOOK_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Valid notification channel types for defense-in-depth validation. */
|
||||
const ALLOWED_CHANNEL_TYPES = new Set(['discord', 'slack', 'webhook']);
|
||||
|
||||
export class NotificationService {
|
||||
private static instance: NotificationService;
|
||||
@@ -21,6 +29,16 @@ export class NotificationService {
|
||||
this.broadcaster = fn;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch an alert: log to history, push via WebSocket, and route to
|
||||
* external channels.
|
||||
*
|
||||
* Routing uses two tiers that coexist intentionally:
|
||||
* - notification_routes (Admiral tier): per-stack pattern-based routing
|
||||
* with priority ordering. If any route matches, global agents are skipped.
|
||||
* - agents table (all tiers): global fallback channels used when no
|
||||
* notification_routes match or when no stackName is provided.
|
||||
*/
|
||||
public async dispatchAlert(level: 'info' | 'warning' | 'error', message: string, stackName?: string) {
|
||||
// 1. Log to history and get the full inserted record (with id)
|
||||
const notification = this.dbService.addNotificationHistory({
|
||||
@@ -35,16 +53,26 @@ export class NotificationService {
|
||||
}
|
||||
|
||||
// 3. Check notification routing rules if a stack context is available
|
||||
const errors: string[] = [];
|
||||
|
||||
if (stackName) {
|
||||
const routes = this.dbService.getEnabledNotificationRoutes();
|
||||
const matched = routes.filter(r => r.stack_patterns.includes(stackName));
|
||||
if (matched.length > 0) {
|
||||
if (isDebugEnabled()) console.log(`[Notify:diag] Matched ${matched.length} route(s) for stack "${stackName}"`);
|
||||
await Promise.allSettled(
|
||||
matched.map(route =>
|
||||
this.sendToChannel(route.channel_type, route.channel_url, level, message)
|
||||
.catch(error => console.error(`Failed to dispatch notification via route "${route.name}":`, error))
|
||||
.then(() => {
|
||||
if (isDebugEnabled()) console.log(`[Notify:diag] Dispatched ${level} via route "${route.name}" (${route.channel_type})`);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error(`Failed to dispatch notification via route "${route.name}":`, error);
|
||||
errors.push(`Route "${route.name}": ${getErrorMessage(error, String(error))}`);
|
||||
})
|
||||
)
|
||||
);
|
||||
this.recordDispatchErrors(notification.id!, errors);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -52,16 +80,35 @@ export class NotificationService {
|
||||
// 4. Fall back to global agents
|
||||
const agents = this.dbService.getEnabledAgents();
|
||||
if (agents.length === 0) {
|
||||
console.log('No active notification agents found. Skipping external dispatch.');
|
||||
if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch');
|
||||
return;
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`);
|
||||
await Promise.allSettled(
|
||||
agents.map(agent =>
|
||||
this.sendToChannel(agent.type, agent.url, level, message)
|
||||
.catch(error => console.error(`Failed to dispatch notification to ${agent.type}:`, error))
|
||||
.then(() => {
|
||||
if (isDebugEnabled()) console.log(`[Notify:diag] Dispatched ${level} via global agent (${agent.type})`);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error(`Failed to dispatch notification to ${agent.type}:`, error);
|
||||
errors.push(`${agent.type}: ${getErrorMessage(error, String(error))}`);
|
||||
})
|
||||
)
|
||||
);
|
||||
this.recordDispatchErrors(notification.id!, errors);
|
||||
}
|
||||
|
||||
/** Persist dispatch errors to the notification record for user visibility. */
|
||||
private recordDispatchErrors(notificationId: number, errors: string[]) {
|
||||
if (errors.length > 0) {
|
||||
try {
|
||||
this.dbService.updateNotificationDispatchError(notificationId, errors.join('; '));
|
||||
} catch (e) {
|
||||
console.error('[Notify] Failed to record dispatch error:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async sendToChannel(type: string, url: string, level: 'info' | 'warning' | 'error', message: string): Promise<void> {
|
||||
@@ -71,10 +118,14 @@ export class NotificationService {
|
||||
await this.sendSlackWebhook(url, level, message);
|
||||
} else if (type === 'webhook') {
|
||||
await this.sendCustomWebhook(url, level, message);
|
||||
} else {
|
||||
throw new Error(`Unsupported channel type: ${type}`);
|
||||
}
|
||||
}
|
||||
|
||||
public async testDispatch(type: 'discord' | 'slack' | 'webhook', url: string) {
|
||||
if (!ALLOWED_CHANNEL_TYPES.has(type)) throw new Error(`Invalid notification type: ${type}`);
|
||||
if (!url || !url.startsWith('https://')) throw new Error('URL must use HTTPS');
|
||||
await this.sendToChannel(type, url, 'info', '🔌 Test Notification from Sencho!');
|
||||
}
|
||||
|
||||
@@ -97,7 +148,8 @@ export class NotificationService {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -119,7 +171,8 @@ export class NotificationService {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -138,7 +191,8 @@ export class NotificationService {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import semver from 'semver';
|
||||
|
||||
/**
|
||||
* Fetches the latest Sencho release version from GitHub or Docker Hub.
|
||||
* Extracted from index.ts so both the fleet endpoint and MonitorService
|
||||
* can share the same lookup logic.
|
||||
*/
|
||||
|
||||
async function fetchFromGitHub(): Promise<string | null> {
|
||||
const res = await fetch('https://api.github.com/repos/AnsoCode/Sencho/releases/latest', {
|
||||
headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'Sencho' },
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const data = await res.json() as { tag_name?: string };
|
||||
const tag = data.tag_name?.replace(/^v/, '') ?? null;
|
||||
return tag && semver.valid(tag) ? tag : null;
|
||||
}
|
||||
|
||||
async function fetchFromDockerHub(): Promise<string | null> {
|
||||
const res = await fetch(
|
||||
'https://hub.docker.com/v2/repositories/saelix/sencho/tags/?page_size=50&ordering=last_updated',
|
||||
{ headers: { 'User-Agent': 'Sencho' }, signal: AbortSignal.timeout(10000) },
|
||||
);
|
||||
if (!res.ok) return null;
|
||||
const data = await res.json() as { results?: { name: string }[] };
|
||||
const tags = (data.results ?? [])
|
||||
.map(t => t.name)
|
||||
.filter(n => semver.valid(n));
|
||||
if (tags.length === 0) return null;
|
||||
tags.sort(semver.rcompare);
|
||||
return tags[0];
|
||||
}
|
||||
|
||||
export async function fetchLatestSenchoVersion(): Promise<string> {
|
||||
try {
|
||||
const gh = await fetchFromGitHub();
|
||||
if (gh) return gh;
|
||||
} catch (err) {
|
||||
// GitHub API fails for private repos or rate limits; try Docker Hub
|
||||
console.warn('[VersionCheck] GitHub fetch failed:', (err as Error).message);
|
||||
}
|
||||
try {
|
||||
const hub = await fetchFromDockerHub();
|
||||
if (hub) return hub;
|
||||
} catch (err) {
|
||||
console.warn('[VersionCheck] Docker Hub fetch failed:', (err as Error).message);
|
||||
}
|
||||
// Throw so CacheService falls back to a stale value if one exists,
|
||||
// and so we do not poison the cache with null.
|
||||
throw new Error('Both GitHub and Docker Hub version lookups failed');
|
||||
}
|
||||
Reference in New Issue
Block a user