gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-08-25 01:37:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement centralized logging and historical metrics dashboard

Browse Source
This commit is contained in:
SaelixCode
2026-03-06 21:05:23 -05:00
parent 49cef7acfa
commit a4a5365da1
9 changed files with 922 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/DatabaseService.ts
+35
View File
@@ -96,6 +96,20 @@ export class DatabaseService {
timestamp INTEGER NOT NULL,
is_read INTEGER DEFAULT 0
);
CREATE TABLE IF NOT EXISTS container_metrics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
container_id TEXT NOT NULL,
stack_name TEXT NOT NULL,
cpu_percent REAL NOT NULL,
memory_mb REAL NOT NULL,
net_rx_mb REAL NOT NULL,
net_tx_mb REAL NOT NULL,
timestamp INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_metrics_timestamp ON container_metrics(timestamp);
CREATE INDEX IF NOT EXISTS idx_metrics_container ON container_metrics(container_id);
`);
// Initialize default global settings if they don't exist
@@ -251,4 +265,25 @@ export class DatabaseService {
const stmt = this.db.prepare('DELETE FROM notification_history');
stmt.run();
}
// --- Container Metrics ---
public addContainerMetric(metric: Omit<any, 'id'>): void {
const stmt = this.db.prepare(
'INSERT INTO container_metrics (container_id, stack_name, cpu_percent, memory_mb, net_rx_mb, net_tx_mb, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?)'
);
stmt.run(metric.container_id, metric.stack_name, metric.cpu_percent, metric.memory_mb, metric.net_rx_mb, metric.net_tx_mb, metric.timestamp);
}
public getContainerMetrics(hoursLookback = 24): any[] {
const cutoff = Date.now() - (hoursLookback * 60 * 60 * 1000);
const stmt = this.db.prepare('SELECT * FROM container_metrics WHERE timestamp >= ? ORDER BY timestamp ASC');
return stmt.all(cutoff);
}
public cleanupOldMetrics(hoursToKeep = 24): void {
const cutoff = Date.now() - (hoursToKeep * 60 * 60 * 1000);
const stmt = this.db.prepare('DELETE FROM container_metrics WHERE timestamp < ?');
stmt.run(cutoff);
}
}
backend/src/services/MonitorService.ts
+65 -65
View File
@@ -203,92 +203,92 @@ export class MonitorService {
private async evaluateStackAlerts(db: DatabaseService) {
const alerts = db.getStackAlerts();
if (alerts.length === 0) return;
// Group alerts by stack so we only fetch stats for stacks we care about
const stacksToMonitor = new Set(alerts.map(a => a.stack_name));
const docker = DockerController.getInstance();
for (const stackName of stacksToMonitor) {
const stackAlerts = alerts.filter(a => a.stack_name === stackName);
if (stackAlerts.length === 0) continue;
try {
const containers = await docker.getRunningContainers();
for (const container of containers) {
const stackName = container.Labels?.['com.docker.compose.project'] || 'system';
try {
const containers = await docker.getContainersByStack(stackName);
if (containers.length === 0) continue;
try {
const rawStats = await docker.getContainerStatsStream(container.Id);
const stats = JSON.parse(rawStats);
// Fetch stats for all containers in this stack
for (const container of containers) {
if (container.State !== 'running') continue;
const metrics = {
cpu_percent: this.calculateCpuPercent(stats),
memory_percent: this.calculateMemoryPercent(stats),
memory_mb: (stats.memory_stats?.usage || 0) / (1024 * 1024),
net_rx: this.calculateNetwork(stats, 'rx'),
net_tx: this.calculateNetwork(stats, 'tx'),
restart_count: 0 // Simplification since ContainerInfo doesn't have it natively
};
try {
const rawStats = await docker.getContainerStatsStream(container.Id);
const stats = JSON.parse(rawStats);
db.addContainerMetric({
container_id: container.Id,
stack_name: stackName,
cpu_percent: metrics.cpu_percent || 0,
memory_mb: metrics.memory_mb || 0,
net_rx_mb: metrics.net_rx || 0,
net_tx_mb: metrics.net_tx || 0,
timestamp: Date.now()
});
const metrics = {
cpu_percent: this.calculateCpuPercent(stats),
memory_percent: this.calculateMemoryPercent(stats),
memory_mb: (stats.memory_stats?.usage || 0) / (1024 * 1024),
net_rx: this.calculateNetwork(stats, 'rx'), // KB/s (naive sum for now, proper net_rx requires time delta but we can do total MB for simple alert or just use what fits)
net_tx: this.calculateNetwork(stats, 'tx'),
restart_count: container.RestartCount || 0
};
const stackAlerts = alerts.filter(a => a.stack_name === stackName);
for (const rule of stackAlerts) {
const ruleId = rule.id!;
const currentValue = metrics[rule.metric as keyof typeof metrics];
for (const rule of stackAlerts) {
const ruleId = rule.id!;
const currentValue = metrics[rule.metric as keyof typeof metrics];
if (currentValue === undefined) continue;
if (currentValue === undefined) continue;
const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold);
const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold);
if (isBreaching) {
if (!this.activeBreaches.has(ruleId)) {
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
}
if (isBreaching) {
if (!this.activeBreaches.has(ruleId)) {
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
}
const breachState = this.activeBreaches.get(ruleId)!;
const durationMs = Date.now() - breachState.breachStartedAt;
const requiredDurationMs = rule.duration_mins * 60 * 1000;
const breachState = this.activeBreaches.get(ruleId)!;
const durationMs = Date.now() - breachState.breachStartedAt;
const requiredDurationMs = rule.duration_mins * 60 * 1000;
if (durationMs >= requiredDurationMs) {
// Duration met! Check cooldown
const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0);
const requiredCooldownMs = rule.cooldown_mins * 60 * 1000;
if (durationMs >= requiredDurationMs) {
// Duration met! Check cooldown
const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0);
const requiredCooldownMs = rule.cooldown_mins * 60 * 1000;
if (timeSinceLastFired >= requiredCooldownMs) {
// Formatted Alert Message
const { name: metricName, unit } = getMetricDetails(rule.metric);
const operatorPhrase = getOperatorPhrase(rule.operator);
if (timeSinceLastFired >= requiredCooldownMs) {
// Formatted Alert Message
const { name: metricName, unit } = getMetricDetails(rule.metric);
const operatorPhrase = getOperatorPhrase(rule.operator);
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
await NotificationService.getInstance().dispatchAlert(
'warning',
message
);
await NotificationService.getInstance().dispatchAlert(
'warning',
message
);
// Update last fired
db.updateStackAlertLastFired(ruleId, Date.now());
}
}
} else {
// Rule isn't breaching anymore, reset tracker
if (this.activeBreaches.has(ruleId)) {
this.activeBreaches.delete(ruleId);
// Update last fired
db.updateStackAlertLastFired(ruleId, Date.now());
}
}
} else {
// Rule isn't breaching anymore, reset tracker
if (this.activeBreaches.has(ruleId)) {
this.activeBreaches.delete(ruleId);
}
}
} catch (e) {
console.error(`Error parsing stats for container ${container.Id}`, e);
}
} catch (e) {
console.error(`Error parsing stats for container ${container.Id}`, e);
}
} catch (e) { }
}
}
db.cleanupOldMetrics(24);
} catch (e) { }
}
private evaluateCondition(actual: number, operator: string, threshold: number): boolean {