mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-21 07:36:40 +00:00
feat: implement centralized logging and historical metrics dashboard
This commit is contained in:
SaelixCode
@@ -762,6 +762,59 @@ app.get('/api/stats', async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/api/metrics/historical', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const metrics = DatabaseService.getInstance().getContainerMetrics(24);
|
||||
res.json(metrics);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: 'Failed to fetch metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/api/logs/global', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const dockerController = DockerController.getInstance();
|
||||
const containers = await dockerController.getRunningContainers();
|
||||
const allLogs: any[] = [];
|
||||
|
||||
await Promise.all(containers.map(async (c) => {
|
||||
const stackName = c.Labels?.['com.docker.compose.project'] || 'system';
|
||||
const containerName = c.Names?.[0]?.replace(/^\//, '') || c.Id.substring(0, 12);
|
||||
try {
|
||||
const container = dockerController.getDocker().getContainer(c.Id);
|
||||
// Fetch last 50 lines with timestamps
|
||||
const logsBuffer = await container.logs({ stdout: true, stderr: true, tail: 50, timestamps: true });
|
||||
|
||||
// Strip docker multiplex headers (non-tty)
|
||||
const logsString = logsBuffer.toString('utf-8').replace(/[\u0000-\u0008\u000B-\u001F\u007F-\u009F]/g, "");
|
||||
const lines = logsString.split('\n').filter(l => l.trim().length > 0);
|
||||
|
||||
lines.forEach(line => {
|
||||
let level = 'INFO';
|
||||
const lowerLine = line.toLowerCase();
|
||||
if (lowerLine.includes('error') || lowerLine.includes('err') || lowerLine.includes('fail') || lowerLine.includes('fatal')) level = 'ERROR';
|
||||
else if (lowerLine.includes('warn')) level = 'WARN';
|
||||
|
||||
// Extract Docker timestamp if present (usually ISO format at start of line)
|
||||
const timeMatch = line.match(/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)\s+(.*)/);
|
||||
const timestamp = timeMatch ? new Date(timeMatch[1]).getTime() : Date.now();
|
||||
const cleanMessage = timeMatch ? timeMatch[2] : line;
|
||||
|
||||
allLogs.push({ stackName, containerName, level, message: cleanMessage, timestamp });
|
||||
});
|
||||
} catch (err) {
|
||||
// Ignore individual container fetch errors
|
||||
}
|
||||
}));
|
||||
|
||||
// Sort globally by timestamp descending and limit to 1000 lines to prevent payload bloat
|
||||
allLogs.sort((a, b) => b.timestamp - a.timestamp);
|
||||
res.json(allLogs.slice(0, 1000));
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: 'Failed to fetch global logs' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get host system stats
|
||||
app.get('/api/system/stats', async (req: Request, res: Response) => {
|
||||
try {
|
||||
|
||||
@@ -96,6 +96,20 @@ export class DatabaseService {
|
||||
timestamp INTEGER NOT NULL,
|
||||
is_read INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS container_metrics (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
container_id TEXT NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
cpu_percent REAL NOT NULL,
|
||||
memory_mb REAL NOT NULL,
|
||||
net_rx_mb REAL NOT NULL,
|
||||
net_tx_mb REAL NOT NULL,
|
||||
timestamp INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_metrics_timestamp ON container_metrics(timestamp);
|
||||
CREATE INDEX IF NOT EXISTS idx_metrics_container ON container_metrics(container_id);
|
||||
`);
|
||||
|
||||
// Initialize default global settings if they don't exist
|
||||
@@ -251,4 +265,25 @@ export class DatabaseService {
|
||||
const stmt = this.db.prepare('DELETE FROM notification_history');
|
||||
stmt.run();
|
||||
}
|
||||
|
||||
// --- Container Metrics ---
|
||||
|
||||
public addContainerMetric(metric: Omit<any, 'id'>): void {
|
||||
const stmt = this.db.prepare(
|
||||
'INSERT INTO container_metrics (container_id, stack_name, cpu_percent, memory_mb, net_rx_mb, net_tx_mb, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?)'
|
||||
);
|
||||
stmt.run(metric.container_id, metric.stack_name, metric.cpu_percent, metric.memory_mb, metric.net_rx_mb, metric.net_tx_mb, metric.timestamp);
|
||||
}
|
||||
|
||||
public getContainerMetrics(hoursLookback = 24): any[] {
|
||||
const cutoff = Date.now() - (hoursLookback * 60 * 60 * 1000);
|
||||
const stmt = this.db.prepare('SELECT * FROM container_metrics WHERE timestamp >= ? ORDER BY timestamp ASC');
|
||||
return stmt.all(cutoff);
|
||||
}
|
||||
|
||||
public cleanupOldMetrics(hoursToKeep = 24): void {
|
||||
const cutoff = Date.now() - (hoursToKeep * 60 * 60 * 1000);
|
||||
const stmt = this.db.prepare('DELETE FROM container_metrics WHERE timestamp < ?');
|
||||
stmt.run(cutoff);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,92 +203,92 @@ export class MonitorService {
|
||||
|
||||
private async evaluateStackAlerts(db: DatabaseService) {
|
||||
const alerts = db.getStackAlerts();
|
||||
if (alerts.length === 0) return;
|
||||
|
||||
// Group alerts by stack so we only fetch stats for stacks we care about
|
||||
const stacksToMonitor = new Set(alerts.map(a => a.stack_name));
|
||||
|
||||
const docker = DockerController.getInstance();
|
||||
|
||||
for (const stackName of stacksToMonitor) {
|
||||
const stackAlerts = alerts.filter(a => a.stack_name === stackName);
|
||||
if (stackAlerts.length === 0) continue;
|
||||
try {
|
||||
const containers = await docker.getRunningContainers();
|
||||
for (const container of containers) {
|
||||
const stackName = container.Labels?.['com.docker.compose.project'] || 'system';
|
||||
|
||||
try {
|
||||
const containers = await docker.getContainersByStack(stackName);
|
||||
if (containers.length === 0) continue;
|
||||
try {
|
||||
const rawStats = await docker.getContainerStatsStream(container.Id);
|
||||
const stats = JSON.parse(rawStats);
|
||||
|
||||
// Fetch stats for all containers in this stack
|
||||
for (const container of containers) {
|
||||
if (container.State !== 'running') continue;
|
||||
const metrics = {
|
||||
cpu_percent: this.calculateCpuPercent(stats),
|
||||
memory_percent: this.calculateMemoryPercent(stats),
|
||||
memory_mb: (stats.memory_stats?.usage || 0) / (1024 * 1024),
|
||||
net_rx: this.calculateNetwork(stats, 'rx'),
|
||||
net_tx: this.calculateNetwork(stats, 'tx'),
|
||||
restart_count: 0 // Simplification since ContainerInfo doesn't have it natively
|
||||
};
|
||||
|
||||
try {
|
||||
const rawStats = await docker.getContainerStatsStream(container.Id);
|
||||
const stats = JSON.parse(rawStats);
|
||||
db.addContainerMetric({
|
||||
container_id: container.Id,
|
||||
stack_name: stackName,
|
||||
cpu_percent: metrics.cpu_percent || 0,
|
||||
memory_mb: metrics.memory_mb || 0,
|
||||
net_rx_mb: metrics.net_rx || 0,
|
||||
net_tx_mb: metrics.net_tx || 0,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
const metrics = {
|
||||
cpu_percent: this.calculateCpuPercent(stats),
|
||||
memory_percent: this.calculateMemoryPercent(stats),
|
||||
memory_mb: (stats.memory_stats?.usage || 0) / (1024 * 1024),
|
||||
net_rx: this.calculateNetwork(stats, 'rx'), // KB/s (naive sum for now, proper net_rx requires time delta but we can do total MB for simple alert or just use what fits)
|
||||
net_tx: this.calculateNetwork(stats, 'tx'),
|
||||
restart_count: container.RestartCount || 0
|
||||
};
|
||||
const stackAlerts = alerts.filter(a => a.stack_name === stackName);
|
||||
for (const rule of stackAlerts) {
|
||||
const ruleId = rule.id!;
|
||||
const currentValue = metrics[rule.metric as keyof typeof metrics];
|
||||
|
||||
for (const rule of stackAlerts) {
|
||||
const ruleId = rule.id!;
|
||||
const currentValue = metrics[rule.metric as keyof typeof metrics];
|
||||
if (currentValue === undefined) continue;
|
||||
|
||||
if (currentValue === undefined) continue;
|
||||
const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold);
|
||||
|
||||
const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold);
|
||||
if (isBreaching) {
|
||||
if (!this.activeBreaches.has(ruleId)) {
|
||||
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
|
||||
}
|
||||
|
||||
if (isBreaching) {
|
||||
if (!this.activeBreaches.has(ruleId)) {
|
||||
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
|
||||
}
|
||||
const breachState = this.activeBreaches.get(ruleId)!;
|
||||
const durationMs = Date.now() - breachState.breachStartedAt;
|
||||
const requiredDurationMs = rule.duration_mins * 60 * 1000;
|
||||
|
||||
const breachState = this.activeBreaches.get(ruleId)!;
|
||||
const durationMs = Date.now() - breachState.breachStartedAt;
|
||||
const requiredDurationMs = rule.duration_mins * 60 * 1000;
|
||||
if (durationMs >= requiredDurationMs) {
|
||||
// Duration met! Check cooldown
|
||||
const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0);
|
||||
const requiredCooldownMs = rule.cooldown_mins * 60 * 1000;
|
||||
|
||||
if (durationMs >= requiredDurationMs) {
|
||||
// Duration met! Check cooldown
|
||||
const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0);
|
||||
const requiredCooldownMs = rule.cooldown_mins * 60 * 1000;
|
||||
if (timeSinceLastFired >= requiredCooldownMs) {
|
||||
// Formatted Alert Message
|
||||
const { name: metricName, unit } = getMetricDetails(rule.metric);
|
||||
const operatorPhrase = getOperatorPhrase(rule.operator);
|
||||
|
||||
if (timeSinceLastFired >= requiredCooldownMs) {
|
||||
// Formatted Alert Message
|
||||
const { name: metricName, unit } = getMetricDetails(rule.metric);
|
||||
const operatorPhrase = getOperatorPhrase(rule.operator);
|
||||
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
|
||||
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
|
||||
|
||||
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
|
||||
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
|
||||
const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
|
||||
|
||||
const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
|
||||
await NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
message
|
||||
);
|
||||
|
||||
await NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
message
|
||||
);
|
||||
|
||||
// Update last fired
|
||||
db.updateStackAlertLastFired(ruleId, Date.now());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Rule isn't breaching anymore, reset tracker
|
||||
if (this.activeBreaches.has(ruleId)) {
|
||||
this.activeBreaches.delete(ruleId);
|
||||
// Update last fired
|
||||
db.updateStackAlertLastFired(ruleId, Date.now());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Rule isn't breaching anymore, reset tracker
|
||||
if (this.activeBreaches.has(ruleId)) {
|
||||
this.activeBreaches.delete(ruleId);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Error parsing stats for container ${container.Id}`, e);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Error parsing stats for container ${container.Id}`, e);
|
||||
}
|
||||
} catch (e) { }
|
||||
}
|
||||
}
|
||||
|
||||
db.cleanupOldMetrics(24);
|
||||
} catch (e) { }
|
||||
}
|
||||
|
||||
private evaluateCondition(actual: number, operator: string, threshold: number): boolean {
|
||||
|
||||
Reference in New Issue
Block a user