gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-08-28 03:06:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add service-scoped stack alert rules (#1681)

Browse Source 
* feat: add service-scoped stack alert rules

Stack alerts can target one Compose service or all services. Breach timers
are per container and cooldowns are per service so a healthy sibling no
longer clears another container's timer or silences a different service.

* fix: gate remote scoped alert creates without losing the body

Remote hops skip JSON parsing so the proxy stream stays pipeable, which
left service_name invisible to the capability gate. Buffer POST /alerts
bodies for inspection, fail closed when the remote lacks the capability,
and rewrite the buffered bytes on forward. Restore alert-panel alt text
to match the unchanged screenshot.

* fix: bound remote alert body buffer and reject encoded JSON

Cap proxied POST /alerts buffering at the local 100KB JSON limit with
structured 413 cleanup, reject non-identity Content-Encoding with 415 so
compressed scoped bodies cannot bypass the mixed-version gate, and cover
oversized, chunked, and gzip regressions.

* fix: harden service-scoped alert delete, cooldown, and proxy gates

Reject non-digit alert ids, dual-write last_fired_at for rollback safety,
gate cooldown on persisted notification history, fail-fast oversized proxy
bodies with 413, and clarify Not in compose UI semantics.

* test: expect dispatchAlert persisted result in crash-safety cases

Update notification-routing assertions for the new { persisted } return
shape so CI matches the cooldown-gating contract.
This commit is contained in:
Anso
2026-07-23 17:57:04 -04:00
committed by GitHub
parent dd54a2e483
commit 85842cc547
32 changed files with 1736 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/DatabaseService.ts
+72 -9
View File
@@ -101,6 +101,7 @@ function stringifyServicesJson(services: StackServiceStatus[], generation: numbe
export interface StackAlert {
id?: number;
stack_name: string;
service_name: string | null;
metric: string;
operator: string;
threshold: number;
@@ -1062,6 +1063,7 @@ export class DatabaseService {
this.migrateStackDossierHashes();
this.migrateGitSourceMultiFile();
this.migrateNodeUpdateSkips();
this.migrateStackAlertServiceScope();
// Reset the cache once at end of constructor in case any migration
// populated it via getGlobalSettings() and a subsequent migration
@@ -1098,6 +1100,7 @@ export class DatabaseService {
CREATE TABLE IF NOT EXISTS stack_alerts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
stack_name TEXT NOT NULL,
service_name TEXT,
metric TEXT NOT NULL,
operator TEXT NOT NULL,
threshold REAL NOT NULL,
@@ -1106,6 +1109,17 @@ export class DatabaseService {
last_fired_at INTEGER DEFAULT 0
);
-- FK cascade is declarative only: PRAGMA foreign_keys is never enabled
-- on this connection, so parent deletes must remove children explicitly
-- (see deleteStackAlert).
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
alert_id INTEGER NOT NULL,
service_name TEXT NOT NULL,
last_fired_at INTEGER NOT NULL,
PRIMARY KEY (alert_id, service_name),
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS notification_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id INTEGER NOT NULL DEFAULT 0,
@@ -2280,6 +2294,25 @@ export class DatabaseService {
}
}
private migrateStackAlertServiceScope(): void {
this.tryAddColumn('stack_alerts', 'service_name', 'TEXT');
try {
// FK is not enforced (foreign_keys pragma off); deleteStackAlert removes children.
this.db.prepare(`
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
alert_id INTEGER NOT NULL,
service_name TEXT NOT NULL,
last_fired_at INTEGER NOT NULL,
PRIMARY KEY (alert_id, service_name),
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
)
`).run();
} catch (e) {
console.error('[DatabaseService] stack_alert_service_cooldowns migration failed:', (e as Error).message);
throw e;
}
}
private migrateScanPolicyFleetColumns(): void {
this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''");
this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0');
@@ -3075,22 +3108,19 @@ export class DatabaseService {
// --- Stack Alerts ---
public getStackAlerts(stackName?: string): StackAlert[] {
let stmt;
if (stackName) {
stmt = this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?');
return stmt.all(stackName) as StackAlert[];
} else {
stmt = this.db.prepare('SELECT * FROM stack_alerts');
return stmt.all() as StackAlert[];
return this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?').all(stackName) as StackAlert[];
}
return this.db.prepare('SELECT * FROM stack_alerts').all() as StackAlert[];
}
public addStackAlert(alert: StackAlert): StackAlert {
const stmt = this.db.prepare(
'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
'INSERT INTO stack_alerts (stack_name, service_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
);
const result = stmt.run(
alert.stack_name,
alert.service_name ?? null,
alert.metric,
alert.operator,
alert.threshold,
@@ -3101,9 +3131,16 @@ export class DatabaseService {
return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert;
}
/**
* Delete an alert and its per-service cooldown rows.
* SQLite foreign_keys is not enabled here, so child rows are removed
* explicitly rather than relying on ON DELETE CASCADE.
*/
public deleteStackAlert(id: number): void {
const stmt = this.db.prepare('DELETE FROM stack_alerts WHERE id = ?');
stmt.run(id);
this.transaction(() => {
this.deleteStackAlertServiceCooldowns(id);
this.db.prepare('DELETE FROM stack_alerts WHERE id = ?').run(id);
});
}
public updateStackAlertLastFired(id: number, timestamp: number): void {
@@ -3111,6 +3148,32 @@ export class DatabaseService {
stmt.run(timestamp, id);
}
public getStackAlertServiceCooldown(alertId: number, serviceName: string): number | null {
const row = this.db.prepare(
'SELECT last_fired_at FROM stack_alert_service_cooldowns WHERE alert_id = ? AND service_name = ?'
).get(alertId, serviceName) as { last_fired_at: number } | undefined;
return row?.last_fired_at ?? null;
}
public hasAnyStackAlertServiceCooldown(alertId: number): boolean {
const row = this.db.prepare(
'SELECT 1 AS present FROM stack_alert_service_cooldowns WHERE alert_id = ? LIMIT 1'
).get(alertId) as { present: number } | undefined;
return !!row;
}
public upsertStackAlertServiceCooldown(alertId: number, serviceName: string, timestamp: number): void {
this.db.prepare(`
INSERT INTO stack_alert_service_cooldowns (alert_id, service_name, last_fired_at)
VALUES (?, ?, ?)
ON CONFLICT(alert_id, service_name) DO UPDATE SET last_fired_at = excluded.last_fired_at
`).run(alertId, serviceName, timestamp);
}
public deleteStackAlertServiceCooldowns(alertId: number): void {
this.db.prepare('DELETE FROM stack_alert_service_cooldowns WHERE alert_id = ?').run(alertId);
}
// --- Auto-Heal Policies ---
public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] {