fix(alerts): harden with security fixes, design compliance, and test coverage (#570)

* fix(alerts): harden with security fixes, design compliance, and test coverage

Add authMiddleware to all alert endpoints, validate notification test
dispatch inputs, fix restart_count metric via Docker inspect, correct
network metric units, replace any types with DockerContainerStats
interface, add webhook timeouts and dispatch error tracking.

Frontend: migrate Select to Combobox, add ScrollArea and delete
confirmation AlertDialog, fix icon strokeWidth to 1.5.

Add update availability notifications for both Sencho version updates
(6-hour check in MonitorService) and stack image updates (state
transition detection in ImageUpdateService). Extract shared version
fetch logic into utils/version-check.ts.

Add diagnostic logging gated behind developer_mode for MonitorService
breach state machine and NotificationService dispatch routing.

Tests: 24 new alert API integration tests, restart_count and version
check unit tests (688 total passing). Docs updated with HTTPS
requirement, update notifications section, and troubleshooting guide.

* fix(alerts): remove unused TEST_USERNAME import in alerts-api tests
This commit is contained in:
Anso
2026-04-13 21:40:53 -04:00
committed by GitHub
parent 4cf6949e9f
commit 0a94df318a
11 changed files with 963 additions and 282 deletions
+282
View File
@@ -0,0 +1,282 @@
/**
* Integration tests for Alert CRUD endpoints, notification test dispatch
* validation, and auth enforcement on all alert/notification routes.
*/
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import request from 'supertest';
import bcrypt from 'bcrypt';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let authCookie: string;
let viewerCookie: string;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
// Mock LicenseService so Admiral-gated routes are accessible
const { LicenseService } = await import('../services/LicenseService');
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
vi.spyOn(LicenseService.getInstance(), 'getVariant').mockReturnValue('admiral');
vi.spyOn(LicenseService.getInstance(), 'getSeatLimits').mockReturnValue({ maxAdmins: null, maxViewers: null });
({ app } = await import('../index'));
authCookie = await loginAsTestAdmin(app);
// Create a viewer user for non-admin tests
const viewerHash = await bcrypt.hash('viewerpass', 1);
DatabaseService.getInstance().addUser({ username: 'viewer', password_hash: viewerHash, role: 'viewer' });
const viewerRes = await request(app)
.post('/api/auth/login')
.send({ username: 'viewer', password: 'viewerpass' });
const cookies = viewerRes.headers['set-cookie'] as string | string[];
viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies;
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
// --- GET /api/alerts ---
describe('GET /api/alerts', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app).get('/api/alerts');
expect(res.status).toBe(401);
});
it('returns empty array when no alerts exist', async () => {
const res = await request(app)
.get('/api/alerts')
.set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body).toEqual([]);
});
it('filters alerts by stackName query param', async () => {
// Seed two alerts for different stacks
const db = DatabaseService.getInstance();
db.addStackAlert({ stack_name: 'web', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
db.addStackAlert({ stack_name: 'api', metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 });
const res = await request(app)
.get('/api/alerts?stackName=web')
.set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body.length).toBe(1);
expect(res.body[0].stack_name).toBe('web');
});
});
// --- POST /api/alerts ---
describe('POST /api/alerts', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app)
.post('/api/alerts')
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(401);
});
it('rejects non-admin users with 403', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', viewerCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(403);
});
it('creates alert and returns 201 with created resource', async () => {
const payload = {
stack_name: 'new-stack',
metric: 'memory_percent',
operator: '>=',
threshold: 85,
duration_mins: 10,
cooldown_mins: 30,
};
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send(payload);
expect(res.status).toBe(201);
expect(res.body.id).toBeDefined();
expect(res.body.stack_name).toBe('new-stack');
expect(res.body.metric).toBe('memory_percent');
expect(res.body.threshold).toBe(85);
});
it('validates required fields and returns 400 for missing data', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test' }); // missing metric, operator, threshold, etc.
expect(res.status).toBe(400);
expect(res.body.error).toBe('Invalid alert data');
});
it('rejects invalid metric values', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'invalid_metric', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects negative threshold', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: -1, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects empty stack_name', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: '', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects stack_name exceeding 255 characters', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'a'.repeat(256), metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects duration_mins exceeding 1440', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 1441, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
it('rejects cooldown_mins exceeding 10080', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 10081 });
expect(res.status).toBe(400);
});
it('rejects invalid operator', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({ stack_name: 'test', metric: 'cpu_percent', operator: '!=', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
expect(res.status).toBe(400);
});
});
// --- DELETE /api/alerts/:id ---
describe('DELETE /api/alerts/:id', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app).delete('/api/alerts/1');
expect(res.status).toBe(401);
});
it('rejects non-admin users with 403', async () => {
const res = await request(app)
.delete('/api/alerts/1')
.set('Cookie', viewerCookie);
expect(res.status).toBe(403);
});
it('deletes existing alert rule', async () => {
// Create an alert to delete
const created = DatabaseService.getInstance().addStackAlert({
stack_name: 'delete-me',
metric: 'cpu_percent',
operator: '>',
threshold: 90,
duration_mins: 0,
cooldown_mins: 0,
});
const res = await request(app)
.delete(`/api/alerts/${created.id}`)
.set('Cookie', authCookie);
expect(res.status).toBe(200);
expect(res.body.success).toBe(true);
});
});
// --- POST /api/notifications/test ---
describe('POST /api/notifications/test', () => {
it('rejects unauthenticated requests with 401', async () => {
const res = await request(app)
.post('/api/notifications/test')
.send({ type: 'discord', url: 'https://discord.com/api/webhooks/123/abc' });
expect(res.status).toBe(401);
});
it('rejects non-admin users with 403', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', viewerCookie)
.send({ type: 'discord', url: 'https://discord.com/api/webhooks/123/abc' });
expect(res.status).toBe(403);
});
it('rejects invalid type with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ type: 'telegram', url: 'https://example.com' });
expect(res.status).toBe(400);
expect(res.body.error).toContain('discord, slack, or webhook');
});
it('rejects missing type with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ url: 'https://example.com' });
expect(res.status).toBe(400);
});
it('rejects non-HTTPS url with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ type: 'discord', url: 'http://example.com' });
expect(res.status).toBe(400);
expect(res.body.error).toContain('HTTPS');
});
it('rejects missing url with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ type: 'discord' });
expect(res.status).toBe(400);
});
it('rejects malformed url with 400', async () => {
const res = await request(app)
.post('/api/notifications/test')
.set('Cookie', authCookie)
.send({ type: 'discord', url: 'https://' });
expect(res.status).toBe(400);
});
});
@@ -10,9 +10,11 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs,
mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState,
mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream,
mockGetContainerRestartCount,
mockDispatchAlert,
mockCurrentLoad, mockMem, mockFsSize,
mockExecAsync,
mockFetchLatestSenchoVersion,
} = vi.hoisted(() => ({
mockGetGlobalSettings: vi.fn().mockReturnValue({}),
mockGetNodes: vi.fn().mockReturnValue([]),
@@ -27,11 +29,13 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
mockGetRunningContainers: vi.fn().mockResolvedValue([]),
mockGetAllContainers: vi.fn().mockResolvedValue([]),
mockGetContainerStatsStream: vi.fn().mockResolvedValue('{}'),
mockGetContainerRestartCount: vi.fn().mockResolvedValue(0),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }),
mockMem: vi.fn().mockResolvedValue({ used: 4e9, total: 16e9 }),
mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
mockExecAsync: vi.fn().mockResolvedValue({ stdout: '' }),
mockFetchLatestSenchoVersion: vi.fn().mockRejectedValue(new Error('not configured')),
}));
vi.mock('../services/DatabaseService', () => ({
@@ -57,10 +61,15 @@ vi.mock('../services/DockerController', () => ({
getRunningContainers: mockGetRunningContainers,
getAllContainers: mockGetAllContainers,
getContainerStatsStream: mockGetContainerStatsStream,
getContainerRestartCount: mockGetContainerRestartCount,
}),
},
}));
vi.mock('../utils/version-check', () => ({
fetchLatestSenchoVersion: (...args: unknown[]) => mockFetchLatestSenchoVersion(...args),
}));
vi.mock('../services/NotificationService', () => ({
NotificationService: {
getInstance: () => ({
@@ -522,3 +531,135 @@ describe('MonitorService - isProcessing guard', () => {
expect((svc as any).isProcessing).toBe(false);
});
});
// ── restart_count metric ──────────────────────────────────────────────
describe('MonitorService - restart_count metric', () => {
function setupRestartScenario(restartCount: number, hasRestartRule: boolean) {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetRunningContainers.mockResolvedValue([{
Id: 'c1',
Labels: { 'com.docker.compose.project': 'my-stack' },
}]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetContainerRestartCount.mockResolvedValue(restartCount);
const alerts = [];
if (hasRestartRule) {
alerts.push({
id: 100,
stack_name: 'my-stack',
metric: 'restart_count',
operator: '>',
threshold: 3,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: 0,
});
}
mockGetStackAlerts.mockReturnValue(alerts);
mockGetGlobalSettings.mockReturnValue({});
}
it('fetches restart count from Docker when a restart_count rule exists', async () => {
setupRestartScenario(5, true);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
// restart_count=5 > threshold=3, should fire
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', expect.stringContaining('Restart count'), 'my-stack');
});
it('skips Docker inspect when no restart_count rules exist', async () => {
setupRestartScenario(5, false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockGetContainerRestartCount).not.toHaveBeenCalled();
});
it('does not fire when restart count is below threshold', async () => {
setupRestartScenario(2, true);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('Restart count'), expect.anything());
});
});
// ── Sencho version update check ───────────────────────────────────────
describe('MonitorService - Sencho version check', () => {
it('dispatches notification when newer version available', async () => {
// Set current version
process.env.npm_package_version = '0.45.0';
mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
mockGetSystemState.mockReturnValue(null); // No previous notification
mockGetGlobalSettings.mockReturnValue({});
mockGetNodes.mockReturnValue([]);
mockGetStackAlerts.mockReturnValue([]);
const svc = MonitorService.getInstance();
// Reset the version check timer so it runs immediately
(svc as any).lastVersionCheckAt = 0;
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
expect(mockSetSystemState).toHaveBeenCalledWith('last_sencho_update_notified_version', '0.46.0');
});
it('does not re-notify for the same version', async () => {
process.env.npm_package_version = '0.45.0';
mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
mockGetSystemState.mockReturnValue('0.46.0'); // Already notified for this version
mockGetGlobalSettings.mockReturnValue({});
mockGetNodes.mockReturnValue([]);
mockGetStackAlerts.mockReturnValue([]);
const svc = MonitorService.getInstance();
(svc as any).lastVersionCheckAt = 0;
await (svc as any).evaluate();
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('0.46.0'));
});
it('handles version check failure gracefully', async () => {
process.env.npm_package_version = '0.45.0';
mockFetchLatestSenchoVersion.mockRejectedValue(new Error('Network down'));
mockGetGlobalSettings.mockReturnValue({});
mockGetNodes.mockReturnValue([]);
mockGetStackAlerts.mockReturnValue([]);
const svc = MonitorService.getInstance();
(svc as any).lastVersionCheckAt = 0;
// Should not throw
await expect((svc as any).evaluate()).resolves.toBeUndefined();
expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', expect.stringContaining('available'));
});
it('respects the 6-hour cooldown interval', async () => {
process.env.npm_package_version = '0.45.0';
mockFetchLatestSenchoVersion.mockResolvedValue('0.46.0');
mockGetSystemState.mockReturnValue(null);
mockGetGlobalSettings.mockReturnValue({});
mockGetNodes.mockReturnValue([]);
mockGetStackAlerts.mockReturnValue([]);
const svc = MonitorService.getInstance();
// Simulate the check ran 1 hour ago (within 6-hour window)
(svc as any).lastVersionCheckAt = Date.now() - 1 * 60 * 60 * 1000;
await (svc as any).evaluate();
// fetchLatestSenchoVersion should not have been called since we're within cooldown
expect(mockFetchLatestSenchoVersion).not.toHaveBeenCalled();
});
});
+22 -53
View File
@@ -59,6 +59,7 @@ function invalidateNodeCaches(nodeId: number): void {
}
import { isDebugEnabled } from './utils/debug';
import { fetchLatestSenchoVersion } from './utils/version-check';
import { getErrorMessage } from './utils/errors';
import { captureLocalNodeFiles, captureRemoteNodeFiles, SnapshotNodeData } from './utils/snapshot-capture';
import { GlobalLogEntry, normalizeContainerName, parseLogTimestamp, detectLogLevel, demuxDockerLog } from './utils/log-parsing';
@@ -1285,50 +1286,8 @@ const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull
const LATEST_VERSION_CACHE_KEY = 'latest-version';
const LATEST_VERSION_CACHE_TTL = 30 * 60 * 1000; // 30 minutes
async function fetchFromGitHub(): Promise<string | null> {
const res = await fetch('https://api.github.com/repos/AnsoCode/Sencho/releases/latest', {
headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'Sencho' },
signal: AbortSignal.timeout(10000),
});
if (!res.ok) return null;
const data = await res.json() as { tag_name?: string };
const tag = data.tag_name?.replace(/^v/, '') ?? null;
return tag && semver.valid(tag) ? tag : null;
}
async function fetchFromDockerHub(): Promise<string | null> {
const res = await fetch(
'https://hub.docker.com/v2/repositories/saelix/sencho/tags/?page_size=50&ordering=last_updated',
{ headers: { 'User-Agent': 'Sencho' }, signal: AbortSignal.timeout(10000) },
);
if (!res.ok) return null;
const data = await res.json() as { results?: { name: string }[] };
const tags = (data.results ?? [])
.map(t => t.name)
.filter(n => semver.valid(n));
if (tags.length === 0) return null;
tags.sort(semver.rcompare);
return tags[0];
}
async function fetchLatestSenchoVersion(): Promise<string> {
try {
const gh = await fetchFromGitHub();
if (gh) return gh;
} catch (err) {
// GitHub API fails for private repos or rate limits; try Docker Hub
console.warn('[VersionCheck] GitHub fetch failed:', (err as Error).message);
}
try {
const hub = await fetchFromDockerHub();
if (hub) return hub;
} catch (err) {
console.warn('[VersionCheck] Docker Hub fetch failed:', (err as Error).message);
}
// Throw so CacheService falls back to a stale value if one exists,
// and so we do not poison the cache with null.
throw new Error('Both GitHub and Docker Hub version lookups failed');
}
// Version fetch logic lives in utils/version-check.ts; imported below for getLatestVersion().
// fetchFromGitHub + fetchFromDockerHub + fetchLatestSenchoVersion are shared with MonitorService.
async function getLatestVersion(forceRefresh = false): Promise<string | null> {
if (forceRefresh) {
@@ -4354,7 +4313,7 @@ app.patch('/api/settings', async (req: Request, res: Response) => {
}
});
app.get('/api/alerts', async (req: Request, res: Response) => {
app.get('/api/alerts', authMiddleware, async (req: Request, res: Response) => {
try {
let stackName = req.query.stackName as string | undefined;
if (Array.isArray(stackName)) stackName = stackName[0] as string;
@@ -4375,7 +4334,7 @@ const AlertCreateSchema = z.object({
cooldown_mins: z.coerce.number().int().min(0).max(10080),
});
app.post('/api/alerts', async (req: Request, res: Response) => {
app.post('/api/alerts', authMiddleware, async (req: Request, res: Response) => {
if (!requireAdmin(req, res)) return;
const parsed = AlertCreateSchema.safeParse(req.body);
if (!parsed.success) {
@@ -4383,15 +4342,15 @@ app.post('/api/alerts', async (req: Request, res: Response) => {
return;
}
try {
DatabaseService.getInstance().addStackAlert(parsed.data);
res.json({ success: true });
const created = DatabaseService.getInstance().addStackAlert(parsed.data);
res.status(201).json(created);
} catch (error) {
console.error('Failed to add alert:', error);
res.status(500).json({ error: 'Failed to add alert' });
}
});
app.delete('/api/alerts/:id', async (req: Request, res: Response) => {
app.delete('/api/alerts/:id', authMiddleware, async (req: Request, res: Response) => {
if (!requireAdmin(req, res)) return;
try {
const id = parseInt(req.params.id as string, 10);
@@ -4439,21 +4398,31 @@ app.delete('/api/notifications', authMiddleware, async (req: Request, res: Respo
}
});
const NOTIFICATION_CHANNEL_TYPES = ['discord', 'slack', 'webhook'] as const;
app.post('/api/notifications/test', authMiddleware, async (req: Request, res: Response) => {
if (!requireAdmin(req, res)) return;
try {
const { type, url } = req.body;
if (!type || !NOTIFICATION_CHANNEL_TYPES.includes(type)) {
res.status(400).json({ error: 'type must be discord, slack, or webhook' });
return;
}
if (!url || typeof url !== 'string' || !url.startsWith('https://')) {
res.status(400).json({ error: 'url must be a valid HTTPS URL' });
return;
}
try { new URL(url); } catch { res.status(400).json({ error: 'url is not a valid URL' }); return; }
await NotificationService.getInstance().testDispatch(type, url);
res.json({ success: true });
} catch (error: any) {
res.status(500).json({ error: 'Test failed', details: error.message });
} catch (error: unknown) {
const msg = error instanceof Error ? error.message : String(error);
res.status(500).json({ error: 'Test failed', details: msg });
}
});
// --- Notification Routes (Admiral) ---
const NOTIFICATION_CHANNEL_TYPES = ['discord', 'slack', 'webhook'] as const;
app.get('/api/notification-routes', authMiddleware, (req: Request, res: Response): void => {
if (!requireAdmin(req, res)) return;
if (!requireAdmiral(req, res)) return;
+9 -2
View File
@@ -655,6 +655,8 @@ export class DatabaseService {
);
CREATE INDEX IF NOT EXISTS idx_notification_routes_priority ON notification_routes(priority);
`);
// Track external dispatch errors on notification records
try { this.db.prepare('ALTER TABLE notification_history ADD COLUMN dispatch_error TEXT').run(); } catch { /* already exists */ }
}
// --- Agents ---
@@ -796,11 +798,11 @@ export class DatabaseService {
}
}
public addStackAlert(alert: StackAlert): void {
public addStackAlert(alert: StackAlert): StackAlert {
const stmt = this.db.prepare(
'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
);
stmt.run(
const result = stmt.run(
alert.stack_name,
alert.metric,
alert.operator,
@@ -809,6 +811,7 @@ export class DatabaseService {
alert.cooldown_mins,
alert.last_fired_at || 0
);
return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert;
}
public deleteStackAlert(id: number): void {
@@ -866,6 +869,10 @@ export class DatabaseService {
stmt.run();
}
public updateNotificationDispatchError(id: number, error: string): void {
this.db.prepare('UPDATE notification_history SET dispatch_error = ? WHERE id = ?').run(error, id);
}
// --- Container Metrics ---
public addContainerMetric(metric: Omit<any, 'id'>): void {
+7
View File
@@ -1028,6 +1028,13 @@ class DockerController {
return typeof stats === 'string' ? stats : JSON.stringify(stats);
}
/** Return the cumulative restart count for a container via inspect(). */
public async getContainerRestartCount(containerId: string): Promise<number> {
const container = this.docker.getContainer(containerId);
const info = await container.inspect();
return info.RestartCount ?? 0;
}
/**
* Exec into a container with full session isolation.
* All state (exec instance, stream) lives in this closure - no singleton traps.
+32 -6
View File
@@ -7,6 +7,7 @@ import { DatabaseService } from './DatabaseService';
import { FileSystemService } from './FileSystemService';
import { RegistryService } from './RegistryService';
import { NodeRegistry } from './NodeRegistry';
import { NotificationService } from './NotificationService';
import { isDebugEnabled } from '../utils/debug';
// ─── Image ref parsing ────────────────────────────────────────────────────────
@@ -284,7 +285,7 @@ export class ImageUpdateService {
for (const node of db.getNodes()) {
if (node.type !== 'local' || !node.id) continue;
try {
await this.checkNode(node.id, db);
await this.checkNode(node.id, node.name, db);
} catch (e) {
console.error(`[ImageUpdateService] Error on node ${node.name}:`, e);
}
@@ -297,7 +298,7 @@ export class ImageUpdateService {
}
}
private async checkNode(nodeId: number, db: DatabaseService) {
private async checkNode(nodeId: number, nodeName: string, db: DatabaseService) {
const docker = DockerController.getInstance(nodeId);
const fs = FileSystemService.getInstance(nodeId);
const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
@@ -373,20 +374,45 @@ export class ImageUpdateService {
await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS);
}
// Read previous state to detect new updates for notifications
const previousState = db.getStackUpdateStatus(nodeId);
// Write status for ALL stacks (including those with no pullable images)
const now = Date.now();
let updatesFound = 0;
const newlyUpdated: string[] = [];
for (const [stackName, images] of stackImages) {
const hasUpdate = Array.from(images).some(img => imageUpdateMap.get(img)?.hasUpdate === true);
if (hasUpdate) updatesFound++;
if (hasUpdate) {
updatesFound++;
// Notify only on state transition: was false/absent, now true
if (!previousState[stackName]) {
newlyUpdated.push(stackName);
}
}
db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now);
}
// Dispatch notifications for stacks that newly have updates
if (newlyUpdated.length > 0) {
const notifier = NotificationService.getInstance();
for (const stackName of newlyUpdated) {
try {
await notifier.dispatchAlert(
'info',
`[Node: ${nodeName}] Stack "${stackName}" has image updates available.`,
stackName,
);
} catch (e) {
console.error(`[ImageUpdateService] Failed to dispatch update notification for "${stackName}":`, e);
}
}
}
console.log(`[ImageUpdateService] Node ${nodeId}: checked ${allImages.size} image(s), ${updatesFound} stack(s) with updates`);
// Prune stale entries for stacks no longer on disk
const existing = db.getStackUpdateStatus(nodeId);
for (const staleStack of Object.keys(existing)) {
// Prune stale entries for stacks no longer on disk (reuse previousState to avoid extra DB read)
for (const staleStack of Object.keys(previousState)) {
if (!stackImages.has(staleStack)) {
db.clearStackUpdateStatus(nodeId, staleStack);
}
+83 -9
View File
@@ -1,9 +1,13 @@
import si from 'systeminformation';
import { exec } from 'child_process';
import { promisify } from 'util';
import semver from 'semver';
import DockerController from './DockerController';
import { DatabaseService } from './DatabaseService';
import { NotificationService } from './NotificationService';
import { isValidVersion } from './CapabilityRegistry';
import { fetchLatestSenchoVersion } from '../utils/version-check';
import { isDebugEnabled } from '../utils/debug';
const execAsync = promisify(exec);
@@ -12,8 +16,8 @@ const getMetricDetails = (metric: string): { name: string, unit: string } => {
case 'cpu_percent': return { name: 'CPU usage', unit: '%' };
case 'memory_percent': return { name: 'Memory usage', unit: '%' };
case 'memory_mb': return { name: 'Memory allocation', unit: ' MB' };
case 'net_rx': return { name: 'Inbound network traffic', unit: ' MB/s' };
case 'net_tx': return { name: 'Outbound network traffic', unit: ' MB/s' };
case 'net_rx': return { name: 'Inbound network traffic', unit: ' MB' };
case 'net_tx': return { name: 'Outbound network traffic', unit: ' MB' };
case 'restart_count': return { name: 'Restart count', unit: ' restarts' };
default: return { name: metric, unit: '' };
}
@@ -26,6 +30,25 @@ const getOperatorPhrase = (operator: string): string => {
return `triggered the operator ${operator}`;
};
/** Shape of the JSON returned by Docker container stats (stream: false). */
interface DockerContainerStats {
cpu_stats?: {
cpu_usage?: { total_usage: number; percpu_usage?: number[] };
system_cpu_usage?: number;
online_cpus?: number;
};
precpu_stats?: {
cpu_usage?: { total_usage: number };
system_cpu_usage?: number;
};
memory_stats?: {
usage?: number;
limit?: number;
stats?: { cache?: number };
};
networks?: Record<string, { rx_bytes: number; tx_bytes: number }>;
}
interface AlertState {
breachStartedAt: number; // timestamp when the rule first breached
}
@@ -51,6 +74,10 @@ export class MonitorService {
private alertedCrashes = new Map<string, number>();
private static readonly CRASH_ALERT_TTL_MS = 60 * 60 * 1000; // 1 hour
// Sencho version check cooldown (6 hours between external API calls)
private lastVersionCheckAt = 0;
private static readonly VERSION_CHECK_INTERVAL_MS = 6 * 60 * 60 * 1000;
private constructor() { }
public static getInstance(): MonitorService {
@@ -62,6 +89,7 @@ export class MonitorService {
public start() {
if (this.intervalId) return;
if (isDebugEnabled()) console.log('[Monitor:diag] Starting evaluation loop (30s interval)');
// Run every 30 seconds
this.intervalId = setInterval(() => {
@@ -76,6 +104,7 @@ export class MonitorService {
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
if (isDebugEnabled()) console.log('[Monitor:diag] Evaluation loop stopped');
}
}
@@ -232,12 +261,43 @@ export class MonitorService {
} catch (e) {
console.error('Error checking docker janitor limits', e);
}
// 4. Sencho version update check (runs once per VERSION_CHECK_INTERVAL_MS)
if (Date.now() - this.lastVersionCheckAt > MonitorService.VERSION_CHECK_INTERVAL_MS) {
this.lastVersionCheckAt = Date.now();
try {
const currentVersion = process.env.npm_package_version || '0.0.0';
const latest = await fetchLatestSenchoVersion();
if (isValidVersion(latest) && isValidVersion(currentVersion) && semver.gt(latest, currentVersion)) {
const db = DatabaseService.getInstance();
const stateKey = 'last_sencho_update_notified_version';
const lastNotified = db.getSystemState(stateKey) || '';
if (lastNotified !== latest) {
const notifier = NotificationService.getInstance();
await notifier.dispatchAlert('info',
`Sencho ${latest} is available (currently running ${currentVersion}). Visit the Fleet dashboard to update.`);
db.setSystemState(stateKey, latest);
}
}
} catch (e) {
// Network errors are expected; do not spam logs
if (isDebugEnabled()) console.debug('[Monitor:diag] Sencho version check failed:', e);
}
}
}
private async evaluateStackAlerts(db: DatabaseService) {
const alerts = db.getStackAlerts();
const nodes = db.getNodes();
// Pre-group alerts by stack name to avoid O(containers * alerts) scanning
const alertsByStack = new Map<string, typeof alerts>();
for (const a of alerts) {
const list = alertsByStack.get(a.stack_name);
if (list) list.push(a);
else alertsByStack.set(a.stack_name, [a]);
}
for (const node of nodes) {
if (!node.id) continue;
// Remote nodes are self-monitoring - skip direct Docker access
@@ -250,16 +310,24 @@ export class MonitorService {
try {
const rawStats = await docker.getContainerStatsStream(container.Id);
const stats = JSON.parse(rawStats);
const stats: DockerContainerStats = JSON.parse(rawStats);
const usedMemory = (stats.memory_stats?.usage || 0) - (stats.memory_stats?.stats?.cache || 0);
// Only fetch restart count when at least one rule for this stack uses it
const stackAlerts = alertsByStack.get(stackName) || [];
const needsRestartCount = stackAlerts.some(a => a.metric === 'restart_count');
const restartCount = needsRestartCount
? await docker.getContainerRestartCount(container.Id)
: 0;
const metrics = {
cpu_percent: this.calculateCpuPercent(stats),
memory_percent: this.calculateMemoryPercent(stats),
memory_mb: Math.max(0, usedMemory) / (1024 * 1024),
net_rx: this.calculateNetwork(stats, 'rx'),
net_tx: this.calculateNetwork(stats, 'tx'),
restart_count: 0 // Simplification since ContainerInfo doesn't have it natively
restart_count: restartCount,
};
db.addContainerMetric({
@@ -272,7 +340,6 @@ export class MonitorService {
timestamp: Date.now()
});
const stackAlerts = alerts.filter(a => a.stack_name === stackName);
for (const rule of stackAlerts) {
const ruleId = rule.id!;
const currentValue = metrics[rule.metric as keyof typeof metrics];
@@ -284,6 +351,7 @@ export class MonitorService {
if (isBreaching) {
if (!this.activeBreaches.has(ruleId)) {
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`);
}
const breachState = this.activeBreaches.get(ruleId)!;
@@ -305,6 +373,7 @@ export class MonitorService {
const message = `[Node: ${node.name}] The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
if (isDebugEnabled()) console.log(`[Monitor:diag] Duration met for rule ${ruleId}, dispatching alert`);
await NotificationService.getInstance().dispatchAlert(
'warning',
message,
@@ -313,11 +382,14 @@ export class MonitorService {
// Update last fired
db.updateStackAlertLastFired(ruleId, Date.now());
} else if (isDebugEnabled()) {
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
}
}
} else {
// Rule isn't breaching anymore, reset tracker
if (this.activeBreaches.has(ruleId)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`);
this.activeBreaches.delete(ruleId);
}
}
@@ -347,6 +419,7 @@ export class MonitorService {
db.cleanupOldNotifications(isNaN(retentionDays) ? 30 : retentionDays);
const auditRetentionDays = parseInt(settings['audit_retention_days'] || '90', 10);
db.cleanupOldAuditLogs(isNaN(auditRetentionDays) ? 90 : auditRetentionDays);
if (isDebugEnabled()) console.log(`[Monitor:diag] Cleanup: metrics ${isNaN(retentionHours) ? 24 : retentionHours}h, notifications ${isNaN(retentionDays) ? 30 : retentionDays}d, audit ${isNaN(auditRetentionDays) ? 90 : auditRetentionDays}d`);
} catch (e) {
console.error('MonitorService: failed to cleanup old data', e);
}
@@ -376,7 +449,7 @@ export class MonitorService {
}
}
private calculateCpuPercent(stats: any): number {
private calculateCpuPercent(stats: DockerContainerStats): number {
let cpuPercent = 0.0;
if (!stats?.cpu_stats?.cpu_usage || !stats?.precpu_stats?.cpu_usage) return 0.0;
@@ -390,7 +463,7 @@ export class MonitorService {
return cpuPercent;
}
private calculateMemoryPercent(stats: any): number {
private calculateMemoryPercent(stats: DockerContainerStats): number {
if (!stats?.memory_stats?.usage || !stats?.memory_stats?.limit) return 0.0;
const used_memory = stats.memory_stats.usage - (stats.memory_stats.stats?.cache || 0);
@@ -401,11 +474,12 @@ export class MonitorService {
return 0.0;
}
private calculateNetwork(stats: any, direction: 'rx' | 'tx'): number {
private calculateNetwork(stats: DockerContainerStats, direction: 'rx' | 'tx'): number {
let bytes = 0;
if (stats.networks) {
const key = direction === 'rx' ? 'rx_bytes' : 'tx_bytes';
for (const iface in stats.networks) {
bytes += stats.networks[iface][`${direction}_bytes`];
bytes += stats.networks[iface][key];
}
}
return bytes / (1024 * 1024); // Return in MB
+60 -6
View File
@@ -1,4 +1,12 @@
import { DatabaseService, NotificationHistory } from './DatabaseService';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
/** Webhook timeout: 10 seconds per external dispatch call. */
const WEBHOOK_TIMEOUT_MS = 10_000;
/** Valid notification channel types for defense-in-depth validation. */
const ALLOWED_CHANNEL_TYPES = new Set(['discord', 'slack', 'webhook']);
export class NotificationService {
private static instance: NotificationService;
@@ -21,6 +29,16 @@ export class NotificationService {
this.broadcaster = fn;
}
/**
* Dispatch an alert: log to history, push via WebSocket, and route to
* external channels.
*
* Routing uses two tiers that coexist intentionally:
* - notification_routes (Admiral tier): per-stack pattern-based routing
* with priority ordering. If any route matches, global agents are skipped.
* - agents table (all tiers): global fallback channels used when no
* notification_routes match or when no stackName is provided.
*/
public async dispatchAlert(level: 'info' | 'warning' | 'error', message: string, stackName?: string) {
// 1. Log to history and get the full inserted record (with id)
const notification = this.dbService.addNotificationHistory({
@@ -35,16 +53,26 @@ export class NotificationService {
}
// 3. Check notification routing rules if a stack context is available
const errors: string[] = [];
if (stackName) {
const routes = this.dbService.getEnabledNotificationRoutes();
const matched = routes.filter(r => r.stack_patterns.includes(stackName));
if (matched.length > 0) {
if (isDebugEnabled()) console.log(`[Notify:diag] Matched ${matched.length} route(s) for stack "${stackName}"`);
await Promise.allSettled(
matched.map(route =>
this.sendToChannel(route.channel_type, route.channel_url, level, message)
.catch(error => console.error(`Failed to dispatch notification via route "${route.name}":`, error))
.then(() => {
if (isDebugEnabled()) console.log(`[Notify:diag] Dispatched ${level} via route "${route.name}" (${route.channel_type})`);
})
.catch(error => {
console.error(`Failed to dispatch notification via route "${route.name}":`, error);
errors.push(`Route "${route.name}": ${getErrorMessage(error, String(error))}`);
})
)
);
this.recordDispatchErrors(notification.id!, errors);
return;
}
}
@@ -52,16 +80,35 @@ export class NotificationService {
// 4. Fall back to global agents
const agents = this.dbService.getEnabledAgents();
if (agents.length === 0) {
console.log('No active notification agents found. Skipping external dispatch.');
if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch');
return;
}
if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`);
await Promise.allSettled(
agents.map(agent =>
this.sendToChannel(agent.type, agent.url, level, message)
.catch(error => console.error(`Failed to dispatch notification to ${agent.type}:`, error))
.then(() => {
if (isDebugEnabled()) console.log(`[Notify:diag] Dispatched ${level} via global agent (${agent.type})`);
})
.catch(error => {
console.error(`Failed to dispatch notification to ${agent.type}:`, error);
errors.push(`${agent.type}: ${getErrorMessage(error, String(error))}`);
})
)
);
this.recordDispatchErrors(notification.id!, errors);
}
/** Persist dispatch errors to the notification record for user visibility. */
private recordDispatchErrors(notificationId: number, errors: string[]) {
if (errors.length > 0) {
try {
this.dbService.updateNotificationDispatchError(notificationId, errors.join('; '));
} catch (e) {
console.error('[Notify] Failed to record dispatch error:', e);
}
}
}
private async sendToChannel(type: string, url: string, level: 'info' | 'warning' | 'error', message: string): Promise<void> {
@@ -71,10 +118,14 @@ export class NotificationService {
await this.sendSlackWebhook(url, level, message);
} else if (type === 'webhook') {
await this.sendCustomWebhook(url, level, message);
} else {
throw new Error(`Unsupported channel type: ${type}`);
}
}
public async testDispatch(type: 'discord' | 'slack' | 'webhook', url: string) {
if (!ALLOWED_CHANNEL_TYPES.has(type)) throw new Error(`Invalid notification type: ${type}`);
if (!url || !url.startsWith('https://')) throw new Error('URL must use HTTPS');
await this.sendToChannel(type, url, 'info', '🔌 Test Notification from Sencho!');
}
@@ -97,7 +148,8 @@ export class NotificationService {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
body: JSON.stringify(payload),
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
});
if (!response.ok) {
@@ -119,7 +171,8 @@ export class NotificationService {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
body: JSON.stringify(payload),
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
});
if (!response.ok) {
@@ -138,7 +191,8 @@ export class NotificationService {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload)
body: JSON.stringify(payload),
signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),
});
if (!response.ok) {
+52
View File
@@ -0,0 +1,52 @@
import semver from 'semver';
/**
* Fetches the latest Sencho release version from GitHub or Docker Hub.
* Extracted from index.ts so both the fleet endpoint and MonitorService
* can share the same lookup logic.
*/
async function fetchFromGitHub(): Promise<string | null> {
const res = await fetch('https://api.github.com/repos/AnsoCode/Sencho/releases/latest', {
headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'Sencho' },
signal: AbortSignal.timeout(10000),
});
if (!res.ok) return null;
const data = await res.json() as { tag_name?: string };
const tag = data.tag_name?.replace(/^v/, '') ?? null;
return tag && semver.valid(tag) ? tag : null;
}
async function fetchFromDockerHub(): Promise<string | null> {
const res = await fetch(
'https://hub.docker.com/v2/repositories/saelix/sencho/tags/?page_size=50&ordering=last_updated',
{ headers: { 'User-Agent': 'Sencho' }, signal: AbortSignal.timeout(10000) },
);
if (!res.ok) return null;
const data = await res.json() as { results?: { name: string }[] };
const tags = (data.results ?? [])
.map(t => t.name)
.filter(n => semver.valid(n));
if (tags.length === 0) return null;
tags.sort(semver.rcompare);
return tags[0];
}
export async function fetchLatestSenchoVersion(): Promise<string> {
try {
const gh = await fetchFromGitHub();
if (gh) return gh;
} catch (err) {
// GitHub API fails for private repos or rate limits; try Docker Hub
console.warn('[VersionCheck] GitHub fetch failed:', (err as Error).message);
}
try {
const hub = await fetchFromDockerHub();
if (hub) return hub;
} catch (err) {
console.warn('[VersionCheck] Docker Hub fetch failed:', (err as Error).message);
}
// Throw so CacheService falls back to a stale value if one exists,
// and so we do not poison the cache with null.
throw new Error('Both GitHub and Docker Hub version lookups failed');
}
+41 -3
View File
@@ -30,7 +30,11 @@ Three channel types are available, each configured with a webhook URL and an ena
### Generic Webhook
Any HTTP endpoint that accepts a POST with a JSON body can receive Sencho alerts. Go to **Settings > Notifications > Webhook**, enter the URL, enable the toggle, and click **Save**.
Any HTTPS endpoint that accepts a POST with a JSON body can receive Sencho alerts. Go to **Settings > Notifications > Webhook**, enter the URL, enable the toggle, and click **Save**.
<Note>
All webhook URLs (Discord, Slack, and generic) must use HTTPS. HTTP URLs are rejected.
</Note>
The payload format is:
@@ -73,8 +77,8 @@ The panel includes:
| CPU Usage (%) | CPU usage relative to total host cores |
| Memory Usage (%) | Memory used as a fraction of the host total |
| Memory Usage (MB) | RSS memory used by the container |
| Network In (MB) | Inbound network throughput |
| Network Out (MB) | Outbound network throughput |
| Network In (MB) | Cumulative inbound network bytes (in MB) |
| Network Out (MB) | Cumulative outbound network bytes (in MB) |
| Restart Count | Number of times the container has restarted |
### Example: alert on high CPU
@@ -135,3 +139,37 @@ The alert panel shows a blue info banner when you are configuring alerts on a re
2. From your **primary** instance, switch to the remote node
3. Right-click a stack and select **Alerts** to create rules
4. The remote instance handles monitoring and notification delivery independently
## Update availability notifications
Sencho can notify you when software updates are available, both for Sencho itself and for your stack images.
### Sencho version updates
When a newer version of Sencho is published, an informational notification is dispatched through your configured channels. This check runs periodically and only notifies once per new version. After you update, the cycle resets and you will be notified when the next release is available.
### Stack image updates
When the periodic image check (every 6 hours) detects that a stack has new upstream images available, a notification is dispatched for each affected stack. Notifications are sent only on state transitions: you will be notified once when a new update appears, not on every check cycle. After you update the stack, the status resets.
Both notification types use the same channel routing as alerts: if notification routes are configured for a stack, those channels receive the message; otherwise, global notification channels are used as a fallback.
## Troubleshooting
### Notifications not being delivered
- Verify at least one notification channel is enabled in **Settings > Notifications**
- Click **Test** on the channel to confirm the webhook URL is reachable
- Check that the webhook URL uses HTTPS
- If using notification routing (Admiral tier), verify the route pattern matches the stack name
### Alert not firing
- Confirm the alert rule exists by opening the stack's alert panel
- Check that the **duration** has elapsed; the condition must hold continuously for the configured duration before an alert fires
- Check the **cooldown** period; after an alert fires, it will not fire again until the cooldown expires
- Verify the metric is being collected; container must be running for stats to be gathered
### Delete confirmation dialog
Deleting an alert rule now requires confirmation. Click the trash icon next to a rule, then confirm in the dialog that appears.
+234 -203
View File
@@ -6,10 +6,21 @@ import {
SheetHeader,
SheetTitle,
} from '@/components/ui/sheet';
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from '@/components/ui/alert-dialog';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { Label } from '@/components/ui/label';
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
import { Combobox } from '@/components/ui/combobox';
import { ScrollArea } from '@/components/ui/scroll-area';
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip';
import { Trash2, HelpCircle, AlertTriangle, Info, CheckCircle2, Loader2 } from 'lucide-react';
import { toast } from '@/components/ui/toast-store';
@@ -39,6 +50,37 @@ interface AgentStatus {
enabledTypes: string[];
}
const metricOptions = [
{ value: 'cpu_percent', label: 'CPU Usage (%)' },
{ value: 'memory_percent', label: 'Memory Usage (%)' },
{ value: 'memory_mb', label: 'Memory Usage (MB)' },
{ value: 'net_rx', label: 'Network In (MB)' },
{ value: 'net_tx', label: 'Network Out (MB)' },
{ value: 'restart_count', label: 'Restart Count' },
];
const operatorOptions = [
{ value: '>', label: 'Greater than' },
{ value: '>=', label: 'Greater or eq' },
{ value: '<', label: 'Less than' },
{ value: '<=', label: 'Less or eq' },
{ value: '==', label: 'Equals' },
];
const metricLabels: Record<string, string> = Object.fromEntries(metricOptions.map(o => [o.value, o.label]));
const agentTypeLabels: Record<string, string> = {
discord: 'Discord',
slack: 'Slack',
webhook: 'Webhook',
};
const clampNonNegative = (setter: (v: string) => void) => (e: React.ChangeEvent<HTMLInputElement>) => {
let val = e.target.value;
if (val !== '' && Number(val) < 0) val = '0';
setter(val);
};
export function StackAlertSheet({ isOpen, onClose, stackName }: StackAlertSheetProps) {
const { isAdmin } = useAuth();
const { activeNode } = useNodes();
@@ -46,6 +88,7 @@ export function StackAlertSheet({ isOpen, onClose, stackName }: StackAlertSheetP
const [alerts, setAlerts] = useState<StackAlert[]>([]);
const [isLoading, setIsLoading] = useState(false);
const [confirmDeleteId, setConfirmDeleteId] = useState<number | null>(null);
const [agentStatus, setAgentStatus] = useState<AgentStatus>({
loading: false,
hasEnabled: false,
@@ -156,27 +199,12 @@ export function StackAlertSheet({ isOpen, onClose, stackName }: StackAlertSheetP
}
};
const metricLabels: Record<string, string> = {
cpu_percent: 'CPU Usage (%)',
memory_percent: 'Memory Usage (%)',
memory_mb: 'Memory Usage (MB)',
net_rx: 'Network In (MB)',
net_tx: 'Network Out (MB)',
restart_count: 'Restart Count',
};
const agentTypeLabels: Record<string, string> = {
discord: 'Discord',
slack: 'Slack',
webhook: 'Webhook',
};
const renderAgentStatusBanner = () => {
if (agentStatus.loading) {
return (
<div className="flex items-center gap-2 p-3 rounded-lg bg-muted/50 border text-sm text-muted-foreground">
<Loader2 className="h-4 w-4 animate-spin shrink-0" />
<span>Checking notification channels</span>
<span>Checking notification channels...</span>
</div>
);
}
@@ -184,7 +212,7 @@ export function StackAlertSheet({ isOpen, onClose, stackName }: StackAlertSheetP
if (isRemote) {
return (
<div className="flex items-start gap-2 p-3 rounded-lg bg-info-muted border border-info/20 text-sm">
<Info className="h-4 w-4 text-info shrink-0 mt-0.5" />
<Info className="h-4 w-4 text-info shrink-0 mt-0.5" strokeWidth={1.5} />
<div className="space-y-0.5">
<p className="font-medium text-info">
Remote node: <span className="font-medium">{activeNode?.name}</span>
@@ -194,7 +222,7 @@ export function StackAlertSheet({ isOpen, onClose, stackName }: StackAlertSheetP
</p>
{!agentStatus.hasEnabled && (
<p className="text-warning font-medium mt-1">
No notification channels are configured on this remote node. Open Settings Notifications to configure them.
No notification channels are configured on this remote node. Open Settings &rarr; Notifications to configure them.
</p>
)}
{agentStatus.hasEnabled && (
@@ -210,12 +238,12 @@ export function StackAlertSheet({ isOpen, onClose, stackName }: StackAlertSheetP
if (!agentStatus.hasEnabled) {
return (
<div className="flex items-start gap-2 p-3 rounded-lg bg-warning-muted border border-warning/20 text-sm">
<AlertTriangle className="h-4 w-4 text-warning shrink-0 mt-0.5" />
<AlertTriangle className="h-4 w-4 text-warning shrink-0 mt-0.5" strokeWidth={1.5} />
<div>
<p className="font-medium text-warning">No notification channels configured</p>
<p className="text-muted-foreground mt-0.5">
Alert rules will be saved and evaluated, but no notifications will be dispatched. Configure Discord, Slack, or a webhook in{' '}
<span className="font-medium">Settings Notifications</span>.
<span className="font-medium">Settings &rarr; Notifications</span>.
</p>
</div>
</div>
@@ -224,7 +252,7 @@ export function StackAlertSheet({ isOpen, onClose, stackName }: StackAlertSheetP
return (
<div className="flex items-center gap-2 p-3 rounded-lg bg-success-muted border border-success/20 text-sm">
<CheckCircle2 className="h-4 w-4 text-success shrink-0" />
<CheckCircle2 className="h-4 w-4 text-success shrink-0" strokeWidth={1.5} />
<div>
<p className="font-medium text-success">
Notifications active via {agentStatus.enabledTypes.map(t => agentTypeLabels[t] ?? t).join(', ')}
@@ -235,196 +263,199 @@ export function StackAlertSheet({ isOpen, onClose, stackName }: StackAlertSheetP
};
return (
<Sheet open={isOpen} onOpenChange={(open) => !open && onClose()}>
<SheetContent className="overflow-y-auto sm:max-w-[420px]">
<SheetHeader>
<SheetTitle>Stack Alerts: {stackName}</SheetTitle>
<SheetDescription>
Configure metric thresholds to trigger notifications for this stack.
</SheetDescription>
</SheetHeader>
<>
<Sheet open={isOpen} onOpenChange={(open) => !open && onClose()}>
<SheetContent className="sm:max-w-[420px] flex flex-col">
<SheetHeader>
<SheetTitle>Stack Alerts: {stackName}</SheetTitle>
<SheetDescription>
Configure metric thresholds to trigger notifications for this stack.
</SheetDescription>
</SheetHeader>
<TooltipProvider>
<div className="mt-4 space-y-5">
{/* Notification agent status banner */}
{renderAgentStatusBanner()}
<ScrollArea className="flex-1">
<TooltipProvider>
<div className="mt-4 space-y-5 pr-2">
{/* Notification agent status banner */}
{renderAgentStatusBanner()}
{/* List Existing Alerts */}
<div className="space-y-3">
<h4 className="text-sm font-medium">Existing Rules</h4>
{alerts.length === 0 ? (
<div className="text-sm text-muted-foreground p-4 bg-muted/50 rounded-lg text-center">
No active alert rules for this stack.
</div>
) : (
alerts.map(alert => (
<div key={alert.id} className="flex flex-col gap-2 p-3 bg-muted/50 rounded-lg border text-sm">
<div className="flex justify-between items-start">
<div>
<span className="font-medium text-foreground">
{metricLabels[alert.metric] || alert.metric} {alert.operator} {alert.threshold}
</span>
<div className="text-muted-foreground mt-1">
Trigger after {alert.duration_mins}m Cooldown: {alert.cooldown_mins}m
{/* List Existing Alerts */}
<div className="space-y-3">
<h4 className="text-sm font-medium">Existing Rules</h4>
{alerts.length === 0 ? (
<div className="text-sm text-muted-foreground p-4 bg-muted/50 rounded-lg text-center">
No active alert rules for this stack.
</div>
) : (
alerts.map(alert => (
<div key={alert.id} className="flex flex-col gap-2 p-3 bg-muted/50 rounded-lg border text-sm">
<div className="flex justify-between items-start">
<div>
<span className="font-medium text-foreground">
{metricLabels[alert.metric] || alert.metric} {alert.operator} {alert.threshold}
</span>
<div className="text-muted-foreground mt-1">
Trigger after {alert.duration_mins}m &bull; Cooldown: {alert.cooldown_mins}m
</div>
</div>
{isAdmin && <Button
variant="ghost"
size="icon"
className="h-8 w-8 text-muted-foreground hover:text-destructive shrink-0"
onClick={() => alert.id && setConfirmDeleteId(alert.id)}
disabled={isLoading}
>
<Trash2 className="h-4 w-4" strokeWidth={1.5} />
</Button>}
</div>
</div>
{isAdmin && <Button
variant="ghost"
size="icon"
className="h-8 w-8 text-muted-foreground hover:text-destructive shrink-0"
onClick={() => alert.id && deleteAlert(alert.id)}
disabled={isLoading}
>
<Trash2 className="h-4 w-4" />
</Button>}
))
)}
</div>
<hr />
{/* Add New Alert Form */}
{isAdmin && <div className="space-y-4">
<h4 className="text-sm font-medium">Add New Rule</h4>
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Metric</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" strokeWidth={1.5} />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">The system resource or metric to monitor. Select from CPU, Memory, Network I/O, or Restarts.</p>
</TooltipContent>
</Tooltip>
</div>
<Combobox
options={metricOptions}
value={metric}
onValueChange={setMetric}
placeholder="Select metric..."
searchPlaceholder="Search metrics..."
/>
</div>
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Operator</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" strokeWidth={1.5} />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">The comparison condition to trigger the alert against the threshold.</p>
</TooltipContent>
</Tooltip>
</div>
<Combobox
options={operatorOptions}
value={operator}
onValueChange={setOperator}
placeholder="Select operator..."
/>
</div>
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Threshold</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" strokeWidth={1.5} />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">The numerical value the metric needs to breach to trigger the conditions.</p>
</TooltipContent>
</Tooltip>
</div>
<Input
type="number"
min={0}
value={threshold}
onChange={clampNonNegative(setThreshold)}
placeholder="e.g. 90"
/>
</div>
</div>
))
)}
</div>
<hr />
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Duration (mins)</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" strokeWidth={1.5} />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">How long the metric must stay in breach of the threshold before sending an alert.</p>
</TooltipContent>
</Tooltip>
</div>
<Input
type="number"
min={0}
value={duration}
onChange={clampNonNegative(setDuration)}
/>
</div>
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Cooldown (mins)</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" strokeWidth={1.5} />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">How long to wait before sending another alert if the stack continues to breach.</p>
</TooltipContent>
</Tooltip>
</div>
<Input
type="number"
min={0}
value={cooldown}
onChange={clampNonNegative(setCooldown)}
/>
</div>
</div>
{/* Add New Alert Form */}
{isAdmin && <div className="space-y-4">
<h4 className="text-sm font-medium">Add New Rule</h4>
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Metric</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">The system resource or metric to monitor. Select from CPU, Memory, Network I/O, or Restarts.</p>
</TooltipContent>
</Tooltip>
</div>
<Select value={metric} onValueChange={setMetric}>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
{Object.entries(metricLabels).map(([val, label]) => (
<SelectItem key={val} value={val}>{label}</SelectItem>
))}
</SelectContent>
</Select>
<Button className="w-full mt-2" onClick={addAlert} disabled={isLoading}>
{isLoading ? (
<><Loader2 className="h-4 w-4 mr-2 animate-spin" />Saving...</>
) : (
'Add Rule'
)}
</Button>
</div>}
</div>
</TooltipProvider>
</ScrollArea>
</SheetContent>
</Sheet>
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Operator</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">The comparison condition to trigger the alert against the threshold.</p>
</TooltipContent>
</Tooltip>
</div>
<Select value={operator} onValueChange={setOperator}>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value=">">Greater than</SelectItem>
<SelectItem value=">=">Greater or eq</SelectItem>
<SelectItem value="<">Less than</SelectItem>
<SelectItem value="<=">Less or eq</SelectItem>
<SelectItem value="==">Equals</SelectItem>
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Threshold</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">The numerical value the metric needs to breach to trigger the conditions.</p>
</TooltipContent>
</Tooltip>
</div>
<Input
type="number"
min={0}
value={threshold}
onChange={e => {
let val = e.target.value;
if (val !== '' && Number(val) < 0) val = '0';
setThreshold(val);
}}
placeholder="e.g. 90"
/>
</div>
</div>
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Duration (mins)</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">How long the metric must stay in breach of the threshold before sending an alert.</p>
</TooltipContent>
</Tooltip>
</div>
<Input
type="number"
min={0}
value={duration}
onChange={e => {
let val = e.target.value;
if (val !== '' && Number(val) < 0) val = '0';
setDuration(val);
}}
/>
</div>
<div className="space-y-2">
<div className="flex items-center gap-2">
<Label>Cooldown (mins)</Label>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="h-4 w-4 text-muted-foreground cursor-help" />
</TooltipTrigger>
<TooltipContent>
<p className="max-w-[200px] text-sm">How long to wait before sending another alert if the stack continues to breach.</p>
</TooltipContent>
</Tooltip>
</div>
<Input
type="number"
min={0}
value={cooldown}
onChange={e => {
let val = e.target.value;
if (val !== '' && Number(val) < 0) val = '0';
setCooldown(val);
}}
/>
</div>
</div>
<Button className="w-full mt-2" onClick={addAlert} disabled={isLoading}>
{isLoading ? (
<><Loader2 className="h-4 w-4 mr-2 animate-spin" />Saving…</>
) : (
'Add Rule'
)}
</Button>
</div>}
</div>
</TooltipProvider>
</SheetContent>
</Sheet>
<AlertDialog open={!!confirmDeleteId} onOpenChange={(open) => !open && setConfirmDeleteId(null)}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Delete Alert Rule</AlertDialogTitle>
<AlertDialogDescription>
This will permanently remove this alert rule. Notifications for this condition will no longer be sent.
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction
className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
onClick={() => { if (confirmDeleteId) deleteAlert(confirmDeleteId); setConfirmDeleteId(null); }}
>
Delete
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</>
);
}