fix(alerts): harden with security fixes, design compliance, and test coverage (#570)

* fix(alerts): harden with security fixes, design compliance, and test coverage

Add authMiddleware to all alert endpoints, validate notification test
dispatch inputs, fix restart_count metric via Docker inspect, correct
network metric units, replace any types with DockerContainerStats
interface, add webhook timeouts and dispatch error tracking.

Frontend: migrate Select to Combobox, add ScrollArea and delete
confirmation AlertDialog, fix icon strokeWidth to 1.5.

Add update availability notifications for both Sencho version updates
(6-hour check in MonitorService) and stack image updates (state
transition detection in ImageUpdateService). Extract shared version
fetch logic into utils/version-check.ts.

Add diagnostic logging gated behind developer_mode for MonitorService
breach state machine and NotificationService dispatch routing.

Tests: 24 new alert API integration tests, restart_count and version
check unit tests (688 total passing). Docs updated with HTTPS
requirement, update notifications section, and troubleshooting guide.

* fix(alerts): remove unused TEST_USERNAME import in alerts-api tests
This commit is contained in:
Anso
2026-04-13 21:40:53 -04:00
committed by GitHub
parent 4cf6949e9f
commit 0a94df318a
11 changed files with 963 additions and 282 deletions
+282
View File
@@ -0,0 +1,282 @@
/**
* Integration tests for Alert CRUD endpoints, notification test dispatch
* validation, and auth enforcement on all alert/notification routes.
*/
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import request from 'supertest';
import bcrypt from 'bcrypt';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let authCookie: string;
let viewerCookie: string;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
// Mock LicenseService so Admiral-gated routes are accessible
const { LicenseService } = await import('../services/LicenseService');
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral');
vi.spyOn(LicenseService.getInstance(), 'getSeatLimits').mockReturnValue({ maxAdmins: null, maxViewers: null });
({ app } = await import('../index'));
authCookie = await loginAsTestAdmin(app);
// Create a viewer user for non-admin tests
const viewerHash = await bcrypt.hash('viewerpass', 1);
DatabaseService.getInstance().addUser({ username: 'viewer', password_hash: viewerHash, role: 'viewer' });
const viewerRes = await request(app)
.post('/api/auth/login')
.send({ username: 'viewer', password: 'viewerpass' });
const cookies = viewerRes.headers['set-cookie'] as string | string[];
viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies;
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
// --- GET /api/alerts ---
describe('GET /api/alerts', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app).get('/api/alerts');
expect(res.status).toBe(401);
});
it('returns empty array when no alerts exist', async () => {
const res = await request(app)
.get('/api/alerts')
.set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body).toEqual([]);
});
it('filters alerts by stackName query param', async () => {
// Seed two alerts for different stacks
const db = DatabaseService.getInstance();
db.addStackAlert({ stack_name: 'web', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
db.addStackAlert({ stack_name: 'api', metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 });
const res = await request(app)
.get('/api/alerts?stackName=web')
.set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body.length).toBe(1);
expect(res.body[0].stack_name).toBe('web');
});
});
// --- POST /api/alerts ---
describe('POST /api/alerts', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app)
.post('/api/alerts')
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(401);
});
it('rejects non-admin users with 403', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', viewerCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(403);
});
it('creates alert and returns 201 with created resource', async () => {
const payload = {
stack_name: 'new-stack',
metric: 'memory_percent',
operator: '>=',
threshold: 85,
duration_mins: 10,
cooldown_mins: 30,
};
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send(payload);
expect(res.status).toBe(201);
expect(res.body.id).toBeDefined();
expect(res.body.stack_name).toBe('new-stack');
expect(res.body.metric).toBe('memory_percent');
expect(res.body.threshold).toBe(85);
});
it('validates required fields and returns 400 for missing data', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test' }); // missing metric, operator, threshold, etc.
expect(res.status).toBe(400);
expect(res.body.error).toBe('Invalid alert data');
});
it('rejects invalid metric values', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'invalid_metric', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects negative threshold', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: -1, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects empty stack_name', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: '', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects stack_name exceeding 255 characters', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'a'.repeat(256), metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects duration_mins exceeding 1440', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 1441, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects cooldown_mins exceeding 10080', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 10081 });
expect(res.status).toBe(400);
});
it('rejects invalid operator', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '!=', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
});
// --- DELETE /api/alerts/:id ---
describe('DELETE /api/alerts/:id', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app).delete('/api/alerts/1');
expect(res.status).toBe(401);
});
it('rejects non-admin users with 403', async () => {
const res = await request(app)
.delete('/api/alerts/1')
.set('Cookie', viewerCookie);
expect(res.status).toBe(403);
});
it('deletes existing alert rule', async () => {
// Create an alert to delete
const created = DatabaseService.getInstance().addStackAlert({
stack_name: 'delete-me',
metric: 'cpu_percent',
operator: '>',
threshold: 90,
duration_mins: 0,
cooldown_mins: 0,
});
const res = await request(app)
.delete(`/api/alerts/${created.id}`)
.set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body.success).toBe(true);
});
});
// --- POST /api/notifications/test ---
describe('POST /api/notifications/test', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app)
.post('/api/notifications/test')
.send({ type: 'discord', url: 'https://discord.com/api/webhooks/123/abc' });
expect(res.status).toBe(401);
});
it('rejects non-admin users with 403', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', viewerCookie)
.send({ type: 'discord', url: 'https://discord.com/api/webhooks/123/abc' });
expect(res.status).toBe(403);
});
it('rejects invalid type with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ type: 'telegram', url: 'https://example.com' });
expect(res.status).toBe(400);
expect(res.body.error).toContain('discord, slack, or webhook');
});
it('rejects missing type with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ url: 'https://example.com' });
expect(res.status).toBe(400);
});
it('rejects non-HTTPS url with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ type: 'discord', url: 'http://example.com' });
expect(res.status).toBe(400);
expect(res.body.error).toContain('HTTPS');
});
it('rejects missing url with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ type: 'discord' });
expect(res.status).toBe(400);
});
it('rejects malformed url with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ type: 'discord', url: 'https://' });
expect(res.status).toBe(400);
});
});
@@ -10,9 +10,11 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs,
mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState,
mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream,
mockGetContainerRestartCount,
mockDispatchAlert,
mockCurrentLoad, mockMem, mockFsSize,
mockExecAsync,
mockFetchLatestSenchoVersion,
} = vi.hoisted(() => ({
mockGetGlobalSettings: vi.fn().mockReturnValue({}),
mockGetNodes: vi.fn().mockReturnValue([]),
@@ -27,11 +29,13 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
mockGetRunningContainers: vi.fn().mockResolvedValue([]),
mockGetAllContainers: vi.fn().mockResolvedValue([]),
mockGetContainerStatsStream: vi.fn().mockResolvedValue('{}'),
mockGetContainerRestartCount: vi.fn().mockResolvedValue(0),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }),
mockMem: vi.fn().mockResolvedValue({ used: 4e9, total: 16e9 }),
mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }),
mockFetchLatestSenchoVersion: vi.fn().mockRejectedValue(new Error('not configured')),
}));
vi.mock('../services/DatabaseService', () => ({
@@ -57,10 +61,15 @@ vi.mock('../services/DockerController', () => ({
getRunningContainers: mockGetRunningContainers,
getAllContainers: mockGetAllContainers,
getContainerStatsStream: mockGetContainerStatsStream,
getContainerRestartCount: mockGetContainerRestartCount,
}),
},
}));
vi.mock('../utils/version-check', () => ({
fetchLatestSenchoVersion: (...args: unknown[]) => mockFetchLatestSenchoVersion(...args),
}));
vi.mock('../services/NotificationService', () => ({
NotificationService: {
getInstance: () => ({
@@ -522,3 +531,135 @@ describe('MonitorService - isProcessing guard', () => {
expect((svc as any).isProcessing).toBe(false);
});
});
// ── restart_count metric ──────────────────────────────────────────────
describe('MonitorService - restart_count metric', () => {
function setupRestartScenario(restartCount: number, hasRestartRule: boolean) {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetRunningContainers.mockResolvedValue([{
Id: 'c1',
Labels: { 'com.docker.compose.project': 'my-stack' },
}]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetContainerRestartCount.mockResolvedValue(restartCount);
const alerts = [];
if (hasRestartRule) {
alerts.push({
id: 100,
stack_name: 'my-stack',
metric: 'restart_count',
operator: '>',
threshold: 3,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: 0,
});
}
mockGetStackAlerts.mockReturnValue(alerts);
mockGetGlobalSettings.mockReturnValue({});
}
it('fetches restart count from Docker when a restart_count rule exists', async () => {
setupRestartScenario(5, true);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
// restart_count=5 > threshold=3, should fire
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Restart count'), 'my-stack');
});
it('skips Docker inspect when no restart_count rules exist', async () => {
setupRestartScenario(5, false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockGetContainerRestartCount).not.toHaveBeenCalled();
});
it('does not fire when restart count is below threshold', async () => {
setupRestartScenario(2, true);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('Restart count'), expect.anything());
});
});
// ── Sencho version update check ───────────────────────────────────────
describe('MonitorService - Sencho version check', () => {
it('dispatches notification when newer version available', async () => {
// Set current version
process.env.npm_package_version = '0.45.0';
mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
mockGetSystemState.mockReturnValue(null); // No previous notification
mockGetGlobalSettings.mockReturnValue({});
mockGetNodes.mockReturnValue([]);
mockGetStackAlerts.mockReturnValue([]);
const svc = MonitorService.getInstance();
// Reset the version check timer so it runs immediately
(svc as any).lastVersionCheckAt = 0;
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
expect(mockSetSystemState).toHaveBeenCalledWith('last_sencho_update_notified_version', '0.46.0');
});
it('does not re-notify for the same version', async () => {
process.env.npm_package_version = '0.45.0';
mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
mockGetSystemState.mockReturnValue('0.46.0'); // Already notified for this version
mockGetGlobalSettings.mockReturnValue({});
mockGetNodes.mockReturnValue([]);
mockGetStackAlerts.mockReturnValue([]);
const svc = MonitorService.getInstance();
(svc as any).lastVersionCheckAt = 0;
await (svc as any).evaluate();
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
});
it('handles version check failure gracefully', async () => {
process.env.npm_package_version = '0.45.0';
mockFetchLatestSenchoVersion.mockRejectedValue(new Error('Network down'));
mockGetGlobalSettings.mockReturnValue({});
mockGetNodes.mockReturnValue([]);
mockGetStackAlerts.mockReturnValue([]);
const svc = MonitorService.getInstance();
(svc as any).lastVersionCheckAt = 0;
// Should not throw
await expect((svc as any).evaluate()).resolves.toBeUndefined();
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('available'));
});
it('respects the 6-hour cooldown interval', async () => {
process.env.npm_package_version = '0.45.0';
mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
mockGetSystemState.mockReturnValue(null);
mockGetGlobalSettings.mockReturnValue({});
mockGetNodes.mockReturnValue([]);
mockGetStackAlerts.mockReturnValue([]);
const svc = MonitorService.getInstance();
// Simulate the check ran 1 hour ago (within 6-hour window)
(svc as any).lastVersionCheckAt = Date.now() - 1 * 60 * 60 * 1000;
await (svc as any).evaluate();
// fetchLatestSenchoVersion should not have been called since we're within cooldown
expect(mockFetchLatestSenchoVersion).not.toHaveBeenCalled();
});
});
+22 -53
View File
@@ -59,6 +59,7 @@ function invalidateNodeCaches(nodeId: number): void {
}
import { isDebugEnabled } from './utils/debug';
import { fetchLatestSenchoVersion } from './utils/version-check';
import { getErrorMessage } from './utils/errors';
import { captureLocalNodeFiles, captureRemoteNodeFiles, SnapshotNodeData } from './utils/snapshot-capture';
import { GlobalLogEntry, normalizeContainerName, parseLogTimestamp, detectLogLevel, demuxDockerLog } from './utils/log-parsing';
@@ -1285,50 +1286,8 @@ const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull
const LATEST_VERSION_CACHE_KEY = 'latest-version';
const LATEST_VERSION_CACHE_TTL = 30 * 60 * 1000; // 30 minutes
async function fetchFromGitHub(): Promise<string | null> {
const res = await fetch('https://api.github.com/repos/AnsoCode/Sencho/releases/latest', {
headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'Sencho' },
signal: AbortSignal.timeout(10000),
});
if (!res.ok) return null;
const data = await res.json() as { tag_name?: string };
const tag = data.tag_name?.replace(/^v/, '') ?? null;
return tag && semver.valid(tag) ? tag : null;
}
async function fetchFromDockerHub(): Promise<string | null> {
const res = await fetch(
'https://hub.docker.com/v2/repositories/saelix/sencho/tags/?page_size=50&ordering=last_updated',
{ headers: { 'User-Agent': 'Sencho' }, signal: AbortSignal.timeout(10000) },
);
if (!res.ok) return null;
const data = await res.json() as { results?: { name: string }[] };
const tags = (data.results ?? [])
.map(t => t.name)
.filter(n => semver.valid(n));
if (tags.length === 0) return null;
tags.sort(semver.rcompare);
return tags[0];
}
async function fetchLatestSenchoVersion(): Promise<string> {
try {
const gh = await fetchFromGitHub();
if (gh) return gh;
} catch (err) {
// GitHub API fails for private repos or rate limits; try Docker Hub
console.warn('[VersionCheck] GitHub fetch failed:', (err as Error).message);
}
try {
const hub = await fetchFromDockerHub();
if (hub) return hub;
} catch (err) {
console.warn('[VersionCheck] Docker Hub fetch failed:', (err as Error).message);
}
// Throw so CacheService falls back to a stale value if one exists,
// and so we do not poison the cache with null.
throw new Error('Both GitHub and Docker Hub version lookups failed');
}
// Version fetch logic lives in utils/version-check.ts; imported below for getLatestVersion().
// fetchFromGitHub + fetchFromDockerHub + fetchLatestSenchoVersion are shared with MonitorService.
async function getLatestVersion(forceRefresh = false): Promise<string | null> {
if (forceRefresh) {
@@ -4354,7 +4313,7 @@ app.patch('/api/settings', async (req: Request, res: Response) => {
}
});
app.get('/api/alerts', async (req: Request, res: Response) => {
app.get('/api/alerts', authMiddleware, async (req: Request, res: Response) => {
try {
let stackName = req.query.stackName as string | undefined;
if (Array.isArray(stackName)) stackName = stackName[0] as string;
@@ -4375,7 +4334,7 @@ const AlertCreateSchema = z.object({
cooldown_mins: z.coerce.number().int().min(0).max(10080),
});
app.post('/api/alerts', async (req: Request, res: Response) => {
app.post('/api/alerts', authMiddleware, async (req: Request, res: Response) => {
if (!requireAdmin(req, res)) return;
const parsed = AlertCreateSchema.safeParse(req.body);
if (!parsed.success) {
@@ -4383,15 +4342,15 @@ app.post('/api/alerts', async (req: Request, res: Response) => {
return;
}
try {
DatabaseService.getInstance().addStackAlert(parsed.data);
res.json({ success: true });
const created = DatabaseService.getInstance().addStackAlert(parsed.data);
res.status(201).json(created);
} catch (error) {
console.error('Failed to add alert:', error);
res.status(500).json({ error: 'Failed to add alert' });
}
});
app.delete('/api/alerts/:id', async (req: Request, res: Response) => {
app.delete('/api/alerts/:id', authMiddleware, async (req: Request, res: Response) => {
if (!requireAdmin(req, res)) return;
try {
const id = parseInt(req.params.id as string, 10);
@@ -4439,21 +4398,31 @@ app.delete('/api/notifications', authMiddleware, async (req: Request, res: Respo
}
});
const NOTIFICATION_CHANNEL_TYPES = ['discord', 'slack', 'webhook'] as const;
app.post('/api/notifications/test', authMiddleware, async (req: Request, res: Response) => {
if (!requireAdmin(req, res)) return;
try {
const { type, url } = req.body;
if (!type || !NOTIFICATION_CHANNEL_TYPES.includes(type)) {
res.status(400).json({ error: 'type must be discord, slack, or webhook' });
return;
}
if (!url || typeof url !== 'string' || !url.startsWith('https://')) {
res.status(400).json({ error: 'url must be a valid HTTPS URL' });
return;
}
try { new URL(url); } catch { res.status(400).json({ error: 'url is not a valid URL' }); return; }
await NotificationService.getInstance().testDispatch(type, url);
res.json({ success: true });
} catch (error: any) {
res.status(500).json({ error: 'Test failed', details: error.message });
} catch (error: unknown) {
const msg = error instanceof Error ? error.message : String(error);
res.status(500).json({ error: 'Test failed', details: msg });
}
});
// --- Notification Routes (Admiral) ---
const NOTIFICATION_CHANNEL_TYPES = ['discord', 'slack', 'webhook'] as const;
app.get('/api/notification-routes', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requireAdmiral(req, res)) return;
+9 -2
View File
@@ -655,6 +655,8 @@ export class DatabaseService {
);
CREATE INDEX IF NOT EXISTS idx_notification_routes_priority ON notification_routes(priority);
`);
// Track external dispatch errors on notification records
try { this.db.prepare('ALTER TABLE notification_history ADD COLUMN dispatch_error TEXT').run(); } catch { /* already exists */ }
}
// --- Agents ---
@@ -796,11 +798,11 @@ export class DatabaseService {
}
}
public addStackAlert(alert: StackAlert): void {
public addStackAlert(alert: StackAlert): StackAlert {
const stmt = this.db.prepare(
'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
);
stmt.run(
const result = stmt.run(
alert.stack_name,
alert.metric,
alert.operator,
@@ -809,6 +811,7 @@ export class DatabaseService {
alert.cooldown_mins,
alert.last_fired_at || 0
);
return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert;
}
public deleteStackAlert(id: number): void {
@@ -866,6 +869,10 @@ export class DatabaseService {
stmt.run();
}
public updateNotificationDispatchError(id: number, error: string): void {
this.db.prepare('UPDATE notification_history SET dispatch_error = ? WHERE id = ?').run(error, id);
}
// --- Container Metrics ---
public addContainerMetric(metric: Omit<any, 'id'>): void {
+7
View File
@@ -1028,6 +1028,13 @@ class DockerController {
return typeof stats === 'string' ? stats : JSON.stringify(stats);
}
/** Return the cumulative restart count for a container via inspect(). */
public async getContainerRestartCount(containerId: string): Promise<number> {
const container = this.docker.getContainer(containerId);
const info = await container.inspect();
return info.RestartCount ?? 0;
}
/**
* Exec into a container with full session isolation.
* All state (exec instance, stream) lives in this closure - no singleton traps.
+32 -6
View File
@@ -7,6 +7,7 @@ import { DatabaseService } from './DatabaseService';
import { FileSystemService } from './FileSystemService';
import { RegistryService } from './RegistryService';
import { NodeRegistry } from './NodeRegistry';
import { NotificationService } from './NotificationService';
import { isDebugEnabled } from '../utils/debug';
// ─── Image ref parsing ────────────────────────────────────────────────────────
@@ -284,7 +285,7 @@ export class ImageUpdateService {
for (const node of db.getNodes()) {
if (node.type !== 'local' || !node.id) continue;
try {
await this.checkNode(node.id, db);
await this.checkNode(node.id, node.name, db);
} catch (e) {
console.error(`[ImageUpdateService] Error on node ${node.name}:`, e);
}
@@ -297,7 +298,7 @@ export class ImageUpdateService {
}
}
private async checkNode(nodeId: number, db: DatabaseService) {
private async checkNode(nodeId: number, nodeName: string, db: DatabaseService) {
const docker = DockerController.getInstance(nodeId);
const fs = FileSystemService.getInstance(nodeId);
const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
@@ -373,20 +374,45 @@ export class ImageUpdateService {
await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS);
}
// Read previous state to detect new updates for notifications
const previousState = db.getStackUpdateStatus(nodeId);
// Write status for ALL stacks (including those with no pullable images)
const now = Date.now();
let updatesFound = 0;
const newlyUpdated: string[] = [];
for (const [stackName, images] of stackImages) {
const hasUpdate = Array.from(images).some(img => imageUpdateMap.get(img)?.hasUpdate === true);
if (hasUpdate) updatesFound++;
if (hasUpdate) {
updatesFound++;
// Notify only on state transition: was false/absent, now true
if (!previousState[stackName]) {
newlyUpdated.push(stackName);
}
}
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now);
}
// Dispatch notifications for stacks that newly have updates
if (newlyUpdated.length > 0) {
const notifier = NotificationService.getInstance();
for (const stackName of newlyUpdated) {
try {
await notifier.dispatchAlert(
'info',
`[Node: ${nodeName}] Stack "${stackName}" has image updates available.`,
stackName,
);
} catch (e) {
console.error(`[ImageUpdateService] Failed to dispatch update notification for "${stackName}":`, e);
}
}
}
console.log(`[ImageUpdateService] Node ${nodeId}: checked ${allImages.size} image(s), ${updatesFound} stack(s) with updates`);
// Prune stale entries for stacks no longer on disk
const existing = db.getStackUpdateStatus(nodeId);
for (const staleStack of Object.keys(existing)) {
// Prune stale entries for stacks no longer on disk (reuse previousState to avoid extra DB read)
for (const staleStack of Object.keys(previousState)) {
if (!stackImages.has(staleStack)) {
db.clearStackUpdateStatus(nodeId, staleStack);
}
+83 -9
View File
@@ -1,9 +1,13 @@
import si from 'systeminformation';
import { exec } from 'child_process';
import { promisify } from 'util';
import semver from 'semver';
import DockerController from './DockerController';
import { DatabaseService } from './DatabaseService';
import { NotificationService } from './NotificationService';
import { isValidVersion } from './CapabilityRegistry';
import { fetchLatestSenchoVersion } from '../utils/version-check';
import { isDebugEnabled } from '../utils/debug';
const execAsync = promisify(exec);
@@ -12,8 +16,8 @@ const getMetricDetails = (metric: string): { name: string, unit: string } => {
case 'cpu_percent': return { name: 'CPU usage', unit: '%' };
case 'memory_percent': return { name: 'Memory usage', unit: '%' };
case 'memory_mb': return { name: 'Memory allocation', unit: ' MB' };
case 'net_rx': return { name: 'Inbound network traffic', unit: ' MB/s' };
case 'net_tx': return { name: 'Outbound network traffic', unit: ' MB/s' };
case 'net_rx': return { name: 'Inbound network traffic', unit: ' MB' };
case 'net_tx': return { name: 'Outbound network traffic', unit: ' MB' };
case 'restart_count': return { name: 'Restart count', unit: ' restarts' };
default: return { name: metric, unit: '' };
}
@@ -26,6 +30,25 @@ const getOperatorPhrase = (operator: string): string => {
return `triggered the operator ${operator}`;
};
/** Shape of the JSON returned by Docker container stats (stream: false). */
interface DockerContainerStats {
cpu_stats?: {
cpu_usage?: { total_usage: number; percpu_usage?: number[] };
system_cpu_usage?: number;
online_cpus?: number;
};
precpu_stats?: {
cpu_usage?: { total_usage: number };
system_cpu_usage?: number;
};
memory_stats?: {
usage?: number;
limit?: number;
stats?: { cache?: number };
};
networks?: Record<string, { rx_bytes: number; tx_bytes: number }>;
}
interface AlertState {
breachStartedAt: number; // timestamp when the rule first breached
}
@@ -51,6 +74,10 @@ export class MonitorService {
private alertedCrashes = new Map<string, number>();
private static readonly CRASH_ALERT_TTL_MS = 60 * 60 * 1000; // 1 hour
// Sencho version check cooldown (6 hours between external API calls)
private lastVersionCheckAt = 0;
private static readonly VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000;
private constructor() { }
public static getInstance(): MonitorService {
@@ -62,6 +89,7 @@ export class MonitorService {
public start() {
if (this.intervalId) return;
if (isDebugEnabled()) console.log('[Monitor:diag] Starting evaluation loop (30s interval)');
// Run every 30 seconds
this.intervalId = setInterval(() => {
@@ -76,6 +104,7 @@ export class MonitorService {
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
if (isDebugEnabled()) console.log('[Monitor:diag] Evaluation loop stopped');
}
}
@@ -232,12 +261,43 @@ export class MonitorService {
} catch (e) {
console.error('Error checking docker janitor limits', e);
}
// 4. Sencho version update check (runs once per VERSION_CHECK_INTERVAL_MS)
if (Date.now() - this.lastVersionCheckAt > MonitorService.VERSION_CHECK_INTERVAL_MS) {
this.lastVersionCheckAt = Date.now();
try {
const currentVersion = process.env.npm_package_version || '0.0.0';
const latest = await fetchLatestSenchoVersion();
if (isValidVersion(latest) && isValidVersion(currentVersion) && semver.gt(latest, currentVersion)) {
const db = DatabaseService.getInstance();
const stateKey = 'last_sencho_update_notified_version';
const lastNotified = db.getSystemState(stateKey) || '';
if (lastNotified !== latest) {
const notifier = NotificationService.getInstance();
await notifier.dispatchAlert('info',
`Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`);
db.setSystemState(stateKey, latest);
}
}
} catch (e) {
// Network errors are expected; do not spam logs
if (isDebugEnabled()) console.debug('[Monitor:diag] Sencho version check failed:', e);
}
}
}
private async evaluateStackAlerts(db: DatabaseService) {
const alerts = db.getStackAlerts();
const nodes = db.getNodes();
// Pre-group alerts by stack name to avoid O(containers * alerts) scanning
const alertsByStack = new Map<string, typeof alerts>();
for (const a of alerts) {
const list = alertsByStack.get(a.stack_name);
if (list) list.push(a);
else alertsByStack.set(a.stack_name, [a]);
}
for (const node of nodes) {
if (!node.id) continue;
// Remote nodes are self-monitoring - skip direct Docker access
@@ -250,16 +310,24 @@ export class MonitorService {
try {
const rawStats = await docker.getContainerStatsStream(container.Id);
const stats = JSON.parse(rawStats);
const stats: DockerContainerStats = JSON.parse(rawStats);
const usedMemory = (stats.memory_stats?.usage || 0) - (stats.memory_stats?.stats?.cache || 0);
// Only fetch restart count when at least one rule for this stack uses it
const stackAlerts = alertsByStack.get(stackName) || [];
const needsRestartCount = stackAlerts.some(a => a.metric === 'restart_count');
const restartCount = needsRestartCount
? await docker.getContainerRestartCount(container.Id)
: 0;
const metrics = {
cpu_percent: this.calculateCpuPercent(stats),
memory_percent: this.calculateMemoryPercent(stats),
memory_mb: Math.max(0, usedMemory) / (1024 * 1024),
net_rx: this.calculateNetwork(stats, 'rx'),
net_tx: this.calculateNetwork(stats, 'tx'),
restart_count: 0 // Simplification since ContainerInfo doesn't have it natively
restart_count: restartCount,
};
db.addContainerMetric({
@@ -272,7 +340,6 @@ export class MonitorService {
timestamp: Date.now()
});
const stackAlerts = alerts.filter(a => a.stack_name === stackName);
for (const rule of stackAlerts) {
const ruleId = rule.id!;
const currentValue = metrics[rule.metric as keyof typeof metrics];
@@ -284,6 +351,7 @@ export class MonitorService {
if (isBreaching) {
if (!this.activeBreaches.has(ruleId)) {
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`);
}
const breachState = this.activeBreaches.get(ruleId)!;
@@ -305,6 +373,7 @@ export class MonitorService {
const message = `[Node: ${node.name}] The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
if (isDebugEnabled()) console.log(`[Monitor:diag] Duration met for rule ${ruleId}, dispatching alert`);
await NotificationService.getInstance().dispatchAlert(
'warning',
message,
@@ -313,11 +382,14 @@ export class MonitorService {
// Update last fired
db.updateStackAlertLastFired(ruleId, Date.now());
} else if (isDebugEnabled()) {
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
}
}
} else {
// Rule isn't breaching anymore, reset tracker
if (this.activeBreaches.has(ruleId)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`);
this.activeBreaches.delete(ruleId);
}
}
@@ -347,6 +419,7 @@ export class MonitorService {
db.cleanupOldNotifications(isNaN(retentionDays) ? 30 : retentionDays);
const auditRetentionDays = parseInt(settings['audit_retention_days'] || '90', 10);
db.cleanupOldAuditLogs(isNaN(auditRetentionDays) ? 90 : auditRetentionDays);
if (isDebugEnabled()) console.log(`[Monitor:diag] Cleanup: metrics ${isNaN(retentionHours) ? 24 : retentionHours}h, notifications ${isNaN(retentionDays) ? 30 : retentionDays}d, audit ${isNaN(auditRetentionDays) ? 90 : auditRetentionDays}d`);
} catch (e) {
console.error('MonitorService: failed to cleanup old data', e);
}
@@ -376,7 +449,7 @@ export class MonitorService {
}
}
private calculateCpuPercent(stats: any): number {
private calculateCpuPercent(stats: DockerContainerStats): number {
let cpuPercent = 0.0;
if (!stats?.cpu_stats?.cpu_usage || !stats?.precpu_stats?.cpu_usage) return 0.0;
@@ -390,7 +463,7 @@ export class MonitorService {
return cpuPercent;
}
private calculateMemoryPercent(stats: any): number {
private calculateMemoryPercent(stats: DockerContainerStats): number {
if (!stats?.memory_stats?.usage || !stats?.memory_stats?.limit) return 0.0;
const used_memory = stats.memory_stats.usage - (stats.memory_stats.stats?.cache || 0);
@@ -401,11 +474,12 @@ export class MonitorService {
return 0.0;
}
private calculateNetwork(stats: any, direction: 'rx' | 'tx'): number {
private calculateNetwork(stats: DockerContainerStats, direction: 'rx' | 'tx'): number {
let bytes = 0;
if (stats.networks) {
const key = direction === 'rx' ? 'rx_bytes' : 'tx_bytes';
for (const iface in stats.networks) {
bytes += stats.networks[iface][`${direction}_bytes`];
bytes += stats.networks[iface][key];
}
}
return bytes / (1024 * 1024); // Return in MB
+60 -6
View File
@@ -1,4 +1,12 @@
import { DatabaseService, NotificationHistory } from './DatabaseService';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
/** Webhook timeout: 10 seconds per external dispatch call. */
const WEBHOOK_TIMEOUT_MS = 10_000;
/** Valid notification channel types for defense-in-depth validation. */
const ALLOWED_CHANNEL_TYPES = new Set(['discord', 'slack', 'webhook']);
export class NotificationService {
private static instance: NotificationService;
@@ -21,6 +29,16 @@ export class NotificationService {
this.broadcaster = fn;
}
/**
* Dispatch an alert: log to history, push via WebSocket, and route to
* external channels.
*
* Routing uses two tiers that coexist intentionally:
* - notification_routes (Admiral tier): per-stack pattern-based routing
* with priority ordering. If any route matches, global agents are skipped.
* - agents table (all tiers): global fallback channels used when no
* notification_routes match or when no stackName is provided.
*/
public async dispatchAlert(level: 'info' | 'warning' | 'error', message: string, stackName?: string) {
// 1. Log to history and get the full inserted record (with id)
const notification = this.dbService.addNotificationHistory({
@@ -35,16 +53,26 @@ export class NotificationService {
}
// 3. Check notification routing rules if a stack context is available
const errors: string[] = [];
if (stackName) {
const routes = this.dbService.getEnabledNotificationRoutes();
const matched = routes.filter(r => r.stack_patterns.includes(stackName));
if (matched.length > 0) {
if (isDebugEnabled()) console.log(`[Notify:diag] Matched ${matched.length} route(s) for stack "${stackName}"`);
await Promise.allSettled(
matched.map(route =>
this.sendToChannel(route.channel_type, route.channel_url, level, message)
.catch(error => console.error(`Failed to dispatch notification via route "${route.name}":`, error))
.then(() => {
if (isDebugEnabled()) console.log(`[Notify:diag] Dispatched ${level} via route "${route.name}" (${route.channel_type})`);
})
.catch(error => {
console.error(`Failed to dispatch notification via route "${route.name}":`, error);
errors.push(`Route "${route.name}": ${getErrorMessage(error, String(error))}`);
})
)
);
this.recordDispatchErrors(notification.id!, errors);
return;
}
}
@@ -52,16 +80,35 @@ export class NotificationService {
// 4. Fall back to global agents
const agents = this.dbService.getEnabledAgents();
if (agents.length === 0) {
console.log('No active notification agents found. Skipping external dispatch.');
if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch');
return;
}
if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`);
await Promise.allSettled(
agents.map(agent =>
this.sendToChannel(agent.type, agent.url, level, message)
.catch(error => console.error(`Failed to dispatch notification to ${agent.type}:`, error))
.then(() => {
if (isDebugEnabled()) console.log(`[Notify:diag] Dispatched ${level} via global agent (${agent.type})`);
})
.catch(error => {
console.error(`Failed to dispatch notification to ${agent.type}:`, error);
errors.push(`${agent.type}: ${getErrorMessage(error, String(error))}`);
})
)
);
this.recordDispatchErrors(notification.id!, errors);
}
/** Persist dispatch errors to the notification record for user visibility. */
private recordDispatchErrors(notificationId: number, errors: string[]) {
if (errors.length > 0) {
try {
this.dbService.updateNotificationDispatchError(notificationId, errors.join('; '));
} catch (e) {
console.error('[Notify] Failed to record dispatch error:', e);
}
}
}
private async sendToChannel(type: string, url: string, level: 'info' | 'warning' | 'error', message: string): Promise<void> {
@@ -71,10 +118,14 @@ export class NotificationService {
await this.sendSlackWebhook(url, level, message);
} else if (type === 'webhook') {
await this.sendCustomWebhook(url, level, message);
} else {
throw new Error(`Unsupported channel type: ${type}`);
}
}
public async testDispatch(type: 'discord' | 'slack' | 'webhook', url: string) {
if (!ALLOWED_CHANNEL_TYPES.has(type)) throw new Error(`Invalid notification type: ${type}`);
if (!url || !url.startsWith('https://')) throw new Error('URL must use HTTPS');
await this.sendToChannel(type, url, 'info', '🔌 Test Notification from Sencho!');
}
@@ -97,7 +148,8 @@ export class NotificationService {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
body: JSON.stringify(payload),
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
});
if (!response.ok) {
@@ -119,7 +171,8 @@ export class NotificationService {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
body: JSON.stringify(payload),
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
});
if (!response.ok) {
@@ -138,7 +191,8 @@ export class NotificationService {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
body: JSON.stringify(payload),
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
});
if (!response.ok) {
+52
View File
@@ -0,0 +1,52 @@
import semver from 'semver';
/**
* Fetches the latest Sencho release version from GitHub or Docker Hub.
* Extracted from index.ts so both the fleet endpoint and MonitorService
* can share the same lookup logic.
*/
async function fetchFromGitHub(): Promise<string | null> {
const res = await fetch('https://api.github.com/repos/AnsoCode/Sencho/releases/latest', {
headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'Sencho' },
signal: AbortSignal.timeout(10000),
});
if (!res.ok) return null;
const data = await res.json() as { tag_name?: string };
const tag = data.tag_name?.replace(/^v/, '') ?? null;
return tag && semver.valid(tag) ? tag : null;
}
async function fetchFromDockerHub(): Promise<string | null> {
const res = await fetch(
'https://hub.docker.com/v2/repositories/saelix/sencho/tags/?page_size=50&ordering=last_updated',
{ headers: { 'User-Agent': 'Sencho' }, signal: AbortSignal.timeout(10000) },
);
if (!res.ok) return null;
const data = await res.json() as { results?: { name: string }[] };
const tags = (data.results ?? [])
.map(t => t.name)
.filter(n => semver.valid(n));
if (tags.length === 0) return null;
tags.sort(semver.rcompare);
return tags[0];
}
export async function fetchLatestSenchoVersion(): Promise<string> {
try {
const gh = await fetchFromGitHub();
if (gh) return gh;
} catch (err) {
// GitHub API fails for private repos or rate limits; try Docker Hub
console.warn('[VersionCheck] GitHub fetch failed:', (err as Error).message);
}
try {
const hub = await fetchFromDockerHub();
if (hub) return hub;
} catch (err) {
console.warn('[VersionCheck] Docker Hub fetch failed:', (err as Error).message);
}
// Throw so CacheService falls back to a stale value if one exists,
// and so we do not poison the cache with null.
throw new Error('Both GitHub and Docker Hub version lookups failed');
}