mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-26 02:06:49 +00:00
feat: add service-scoped stack alert rules (#1681)
* feat: add service-scoped stack alert rules
Stack alerts can target one Compose service or all services. Breach timers
are per container and cooldowns are per service so a healthy sibling no
longer clears another container's timer or silences a different service.
* fix: gate remote scoped alert creates without losing the body
Remote hops skip JSON parsing so the proxy stream stays pipeable, which
left service_name invisible to the capability gate. Buffer POST /alerts
bodies for inspection, fail closed when the remote lacks the capability,
and rewrite the buffered bytes on forward. Restore alert-panel alt text
to match the unchanged screenshot.
* fix: bound remote alert body buffer and reject encoded JSON
Cap proxied POST /alerts buffering at the local 100KB JSON limit with
structured 413 cleanup, reject non-identity Content-Encoding with 415 so
compressed scoped bodies cannot bypass the mixed-version gate, and cover
oversized, chunked, and gzip regressions.
* fix: harden service-scoped alert delete, cooldown, and proxy gates
Reject non-digit alert ids, dual-write last_fired_at for rollback safety,
gate cooldown on persisted notification history, fail-fast oversized proxy
bodies with 413, and clarify Not in compose UI semantics.
* test: expect dispatchAlert persisted result in crash-safety cases
Update notification-routing assertions for the new { persisted } return
shape so CI matches the cooldown-gating contract.
This commit is contained in:
Anso
@@ -59,6 +59,7 @@ export const CAPABILITIES = [
|
||||
'stack-down-remove-volumes',
|
||||
'guided-external-network-preflight',
|
||||
'service-scoped-update',
|
||||
'service-scoped-stack-alert',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
@@ -88,6 +89,10 @@ export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes'
|
||||
/** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */
|
||||
export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability;
|
||||
|
||||
/** Capability for nullable `service_name` on stack alert rules and per-service cooldown evaluation. */
|
||||
export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY =
|
||||
'service-scoped-stack-alert' as const satisfies Capability;
|
||||
|
||||
/** Returns true when the string is a usable semver version. */
|
||||
export function isValidVersion(v: string | null | undefined): v is string {
|
||||
return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v);
|
||||
|
||||
@@ -101,6 +101,7 @@ function stringifyServicesJson(services: StackServiceStatus[], generation: numbe
|
||||
export interface StackAlert {
|
||||
id?: number;
|
||||
stack_name: string;
|
||||
service_name: string | null;
|
||||
metric: string;
|
||||
operator: string;
|
||||
threshold: number;
|
||||
@@ -1062,6 +1063,7 @@ export class DatabaseService {
|
||||
this.migrateStackDossierHashes();
|
||||
this.migrateGitSourceMultiFile();
|
||||
this.migrateNodeUpdateSkips();
|
||||
this.migrateStackAlertServiceScope();
|
||||
|
||||
// Reset the cache once at end of constructor in case any migration
|
||||
// populated it via getGlobalSettings() and a subsequent migration
|
||||
@@ -1098,6 +1100,7 @@ export class DatabaseService {
|
||||
CREATE TABLE IF NOT EXISTS stack_alerts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
stack_name TEXT NOT NULL,
|
||||
service_name TEXT,
|
||||
metric TEXT NOT NULL,
|
||||
operator TEXT NOT NULL,
|
||||
threshold REAL NOT NULL,
|
||||
@@ -1106,6 +1109,17 @@ export class DatabaseService {
|
||||
last_fired_at INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
-- FK cascade is declarative only: PRAGMA foreign_keys is never enabled
|
||||
-- on this connection, so parent deletes must remove children explicitly
|
||||
-- (see deleteStackAlert).
|
||||
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
|
||||
alert_id INTEGER NOT NULL,
|
||||
service_name TEXT NOT NULL,
|
||||
last_fired_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (alert_id, service_name),
|
||||
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS notification_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
node_id INTEGER NOT NULL DEFAULT 0,
|
||||
@@ -2280,6 +2294,25 @@ export class DatabaseService {
|
||||
}
|
||||
}
|
||||
|
||||
private migrateStackAlertServiceScope(): void {
|
||||
this.tryAddColumn('stack_alerts', 'service_name', 'TEXT');
|
||||
try {
|
||||
// FK is not enforced (foreign_keys pragma off); deleteStackAlert removes children.
|
||||
this.db.prepare(`
|
||||
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
|
||||
alert_id INTEGER NOT NULL,
|
||||
service_name TEXT NOT NULL,
|
||||
last_fired_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (alert_id, service_name),
|
||||
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
|
||||
)
|
||||
`).run();
|
||||
} catch (e) {
|
||||
console.error('[DatabaseService] stack_alert_service_cooldowns migration failed:', (e as Error).message);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private migrateScanPolicyFleetColumns(): void {
|
||||
this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''");
|
||||
this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0');
|
||||
@@ -3075,22 +3108,19 @@ export class DatabaseService {
|
||||
// --- Stack Alerts ---
|
||||
|
||||
public getStackAlerts(stackName?: string): StackAlert[] {
|
||||
let stmt;
|
||||
if (stackName) {
|
||||
stmt = this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?');
|
||||
return stmt.all(stackName) as StackAlert[];
|
||||
} else {
|
||||
stmt = this.db.prepare('SELECT * FROM stack_alerts');
|
||||
return stmt.all() as StackAlert[];
|
||||
return this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?').all(stackName) as StackAlert[];
|
||||
}
|
||||
return this.db.prepare('SELECT * FROM stack_alerts').all() as StackAlert[];
|
||||
}
|
||||
|
||||
public addStackAlert(alert: StackAlert): StackAlert {
|
||||
const stmt = this.db.prepare(
|
||||
'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
|
||||
'INSERT INTO stack_alerts (stack_name, service_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
);
|
||||
const result = stmt.run(
|
||||
alert.stack_name,
|
||||
alert.service_name ?? null,
|
||||
alert.metric,
|
||||
alert.operator,
|
||||
alert.threshold,
|
||||
@@ -3101,9 +3131,16 @@ export class DatabaseService {
|
||||
return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an alert and its per-service cooldown rows.
|
||||
* SQLite foreign_keys is not enabled here, so child rows are removed
|
||||
* explicitly rather than relying on ON DELETE CASCADE.
|
||||
*/
|
||||
public deleteStackAlert(id: number): void {
|
||||
const stmt = this.db.prepare('DELETE FROM stack_alerts WHERE id = ?');
|
||||
stmt.run(id);
|
||||
this.transaction(() => {
|
||||
this.deleteStackAlertServiceCooldowns(id);
|
||||
this.db.prepare('DELETE FROM stack_alerts WHERE id = ?').run(id);
|
||||
});
|
||||
}
|
||||
|
||||
public updateStackAlertLastFired(id: number, timestamp: number): void {
|
||||
@@ -3111,6 +3148,32 @@ export class DatabaseService {
|
||||
stmt.run(timestamp, id);
|
||||
}
|
||||
|
||||
public getStackAlertServiceCooldown(alertId: number, serviceName: string): number | null {
|
||||
const row = this.db.prepare(
|
||||
'SELECT last_fired_at FROM stack_alert_service_cooldowns WHERE alert_id = ? AND service_name = ?'
|
||||
).get(alertId, serviceName) as { last_fired_at: number } | undefined;
|
||||
return row?.last_fired_at ?? null;
|
||||
}
|
||||
|
||||
public hasAnyStackAlertServiceCooldown(alertId: number): boolean {
|
||||
const row = this.db.prepare(
|
||||
'SELECT 1 AS present FROM stack_alert_service_cooldowns WHERE alert_id = ? LIMIT 1'
|
||||
).get(alertId) as { present: number } | undefined;
|
||||
return !!row;
|
||||
}
|
||||
|
||||
public upsertStackAlertServiceCooldown(alertId: number, serviceName: string, timestamp: number): void {
|
||||
this.db.prepare(`
|
||||
INSERT INTO stack_alert_service_cooldowns (alert_id, service_name, last_fired_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(alert_id, service_name) DO UPDATE SET last_fired_at = excluded.last_fired_at
|
||||
`).run(alertId, serviceName, timestamp);
|
||||
}
|
||||
|
||||
public deleteStackAlertServiceCooldowns(alertId: number): void {
|
||||
this.db.prepare('DELETE FROM stack_alert_service_cooldowns WHERE alert_id = ?').run(alertId);
|
||||
}
|
||||
|
||||
// --- Auto-Heal Policies ---
|
||||
|
||||
public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] {
|
||||
|
||||
@@ -735,15 +735,15 @@ export class DockerEventService {
|
||||
}
|
||||
|
||||
private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise<void> {
|
||||
return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly));
|
||||
await this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly));
|
||||
}
|
||||
|
||||
private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
await this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
}
|
||||
|
||||
private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
|
||||
return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
await this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName));
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
|
||||
@@ -32,6 +32,39 @@ const getOperatorPhrase = (operator: string): string => {
|
||||
return `triggered the operator ${operator}`;
|
||||
};
|
||||
|
||||
/** Sentinel for containers without a Compose service label. Not a valid API target. */
|
||||
const UNLABELED_SERVICE_KEY = '_unlabeled';
|
||||
|
||||
function displayServiceName(serviceName: string): string {
|
||||
return serviceName === UNLABELED_SERVICE_KEY ? 'unknown service' : serviceName;
|
||||
}
|
||||
|
||||
function parseAlertBreachKey(key: string): { ruleId: number; containerId: string } | null {
|
||||
const sep = key.indexOf(':');
|
||||
if (sep < 0) return null;
|
||||
const ruleId = Number(key.slice(0, sep));
|
||||
if (!Number.isFinite(ruleId)) return null;
|
||||
return { ruleId, containerId: key.slice(sep + 1) };
|
||||
}
|
||||
|
||||
/** Per-service cooldown timestamp, with pre-migration fallback to rule.last_fired_at. */
|
||||
function resolveStackAlertLastFired(rule: StackAlert, serviceName: string, db: DatabaseService): number {
|
||||
const ruleId = rule.id!;
|
||||
let lastFired = db.getStackAlertServiceCooldown(ruleId, serviceName);
|
||||
if (lastFired == null && !db.hasAnyStackAlertServiceCooldown(ruleId) && (rule.last_fired_at || 0) > 0) {
|
||||
lastFired = rule.last_fired_at || 0;
|
||||
}
|
||||
return lastFired || 0;
|
||||
}
|
||||
|
||||
function normalizeContainerName(container: { Id: string; Names?: string[] }): string {
|
||||
const raw = container.Names?.[0];
|
||||
if (raw && raw.length > 0) {
|
||||
return raw.startsWith('/') ? raw.slice(1) : raw;
|
||||
}
|
||||
return container.Id.slice(0, 12);
|
||||
}
|
||||
|
||||
/** Shape of the JSON returned by Docker container stats (stream: false). */
|
||||
interface DockerContainerStats {
|
||||
cpu_stats?: {
|
||||
@@ -153,21 +186,18 @@ export class MonitorService {
|
||||
private janitorConsecutiveTimeouts = 0;
|
||||
private janitorBreakerUntil = 0;
|
||||
|
||||
// Track the duration a specific stack alert rule has been in breach state
|
||||
// key: rule_id, value: AlertState
|
||||
private activeBreaches = new Map<number, AlertState>();
|
||||
// Track the duration a specific stack alert rule+container has been in breach
|
||||
// key: `${ruleId}:${containerId}`, value: AlertState
|
||||
private activeBreaches = new Map<string, AlertState>();
|
||||
|
||||
// Track previous network counters per container for rate calculation.
|
||||
// key: container_id, value: { rx bytes, tx bytes, sample timestamp }
|
||||
private previousNetworkStats = new Map<string, { rx: number; tx: number; ts: number }>();
|
||||
|
||||
// Per-cycle dispatch dedup. Stack rules are shared across every container
|
||||
// in the stack, so parallel processContainer calls can race past the
|
||||
// cooldown check (DB write happens after the awaited dispatch) and fire
|
||||
// the same alert N times on first breach. The synchronous check-and-add
|
||||
// below is atomic in JS between awaits, so only the first worker wins.
|
||||
// Per-cycle dispatch dedup keyed by rule+service so replicas of the same
|
||||
// Compose service do not fire N times, while different services can.
|
||||
// Reset at the start of each evaluateStackAlerts call.
|
||||
private firedThisCycle = new Set<number>();
|
||||
private firedThisCycle = new Set<string>();
|
||||
|
||||
// Crash and healthcheck detection live in DockerEventService (event-driven,
|
||||
// causal classification). MonitorService no longer polls for container
|
||||
@@ -604,6 +634,10 @@ export class MonitorService {
|
||||
else alertsByStack.set(a.stack_name, [a]);
|
||||
}
|
||||
|
||||
const activeRuleIds = new Set(alerts.map(a => a.id!));
|
||||
const allCurrentIds = new Set<string>();
|
||||
let enumerationFailed = false;
|
||||
|
||||
for (const node of nodes) {
|
||||
if (!node.id) continue;
|
||||
// Remote nodes are self-monitoring - skip direct Docker access
|
||||
@@ -623,22 +657,39 @@ export class MonitorService {
|
||||
(container) => this.processContainer(node, container, alertsByStack, docker, db),
|
||||
);
|
||||
|
||||
for (const c of containers) allCurrentIds.add(c.Id);
|
||||
|
||||
// Clean up stale network stats for containers on this node that no longer run
|
||||
if (this.previousNetworkStats.size > containers.length * 2) {
|
||||
const currentIds = new Set(containers.map((c: { Id: string }) => c.Id));
|
||||
const nodeIds = new Set(containers.map((c: { Id: string }) => c.Id));
|
||||
for (const key of this.previousNetworkStats.keys()) {
|
||||
if (!currentIds.has(key)) this.previousNetworkStats.delete(key);
|
||||
if (!nodeIds.has(key)) this.previousNetworkStats.delete(key);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// Enumeration failed: preserve existing breach timers.
|
||||
enumerationFailed = true;
|
||||
console.error(`Error fetching containers for node ${node.name}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up stale breach trackers for rules that have been deleted
|
||||
const activeRuleIds = new Set(alerts.map(a => a.id!));
|
||||
for (const key of this.activeBreaches.keys()) {
|
||||
if (!activeRuleIds.has(key)) {
|
||||
// After successful enumeration(s), drop breach timers for containers
|
||||
// that are no longer running. Skip when any local enumeration failed
|
||||
// so a transient Docker error cannot reset duration timers.
|
||||
if (!enumerationFailed) {
|
||||
for (const key of [...this.activeBreaches.keys()]) {
|
||||
const parsed = parseAlertBreachKey(key);
|
||||
if (parsed && activeRuleIds.has(parsed.ruleId) && !allCurrentIds.has(parsed.containerId)) {
|
||||
this.activeBreaches.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up in-memory breach trackers for rules that have been deleted.
|
||||
// Persisted cooldowns are removed only via transactional deleteStackAlert.
|
||||
for (const key of [...this.activeBreaches.keys()]) {
|
||||
const parsed = parseAlertBreachKey(key);
|
||||
if (parsed && !activeRuleIds.has(parsed.ruleId)) {
|
||||
this.activeBreaches.delete(key);
|
||||
}
|
||||
}
|
||||
@@ -666,12 +717,14 @@ export class MonitorService {
|
||||
*/
|
||||
private async processContainer(
|
||||
node: Node,
|
||||
container: { Id: string; Labels?: Record<string, string> },
|
||||
container: { Id: string; Names?: string[]; Labels?: Record<string, string> },
|
||||
alertsByStack: Map<string, StackAlert[]>,
|
||||
docker: DockerController,
|
||||
db: DatabaseService,
|
||||
): Promise<void> {
|
||||
const stackName = container.Labels?.['com.docker.compose.project'] || 'system';
|
||||
const serviceName = container.Labels?.['com.docker.compose.service'] || UNLABELED_SERVICE_KEY;
|
||||
const containerName = normalizeContainerName(container);
|
||||
|
||||
try {
|
||||
const rawStats = await withTimeout(
|
||||
@@ -727,7 +780,11 @@ export class MonitorService {
|
||||
});
|
||||
|
||||
for (const rule of stackAlerts) {
|
||||
if (rule.service_name && rule.service_name !== serviceName) continue;
|
||||
|
||||
const ruleId = rule.id!;
|
||||
const breachKey = `${ruleId}:${container.Id}`;
|
||||
const cooldownKey = `${ruleId}:${serviceName}`;
|
||||
const currentValue = metrics[rule.metric as keyof typeof metrics];
|
||||
|
||||
if (currentValue === undefined) continue;
|
||||
@@ -735,57 +792,75 @@ export class MonitorService {
|
||||
const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold);
|
||||
|
||||
if (isBreaching) {
|
||||
if (!this.activeBreaches.has(ruleId)) {
|
||||
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`);
|
||||
if (!this.activeBreaches.has(breachKey)) {
|
||||
this.activeBreaches.set(breachKey, { breachStartedAt: Date.now() });
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}" service "${serviceName}" container "${containerName}"`);
|
||||
}
|
||||
|
||||
const breachState = this.activeBreaches.get(ruleId)!;
|
||||
const breachState = this.activeBreaches.get(breachKey)!;
|
||||
const durationMs = Date.now() - breachState.breachStartedAt;
|
||||
const requiredDurationMs = rule.duration_mins * 60 * 1000;
|
||||
|
||||
if (durationMs >= requiredDurationMs) {
|
||||
const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0);
|
||||
const timeSinceLastFired = Date.now() - resolveStackAlertLastFired(rule, serviceName, db);
|
||||
const requiredCooldownMs = rule.cooldown_mins * 60 * 1000;
|
||||
|
||||
if (timeSinceLastFired >= requiredCooldownMs) {
|
||||
// Claim this rule for the cycle before awaiting
|
||||
// dispatch. The check-and-add is synchronous, so
|
||||
// sibling workers evaluating the same shared rule
|
||||
// see the claim and skip — preventing N-fire when
|
||||
// multiple containers in one stack all breach.
|
||||
if (this.firedThisCycle.has(ruleId)) {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} (already fired this cycle by sibling container)`);
|
||||
} else {
|
||||
this.firedThisCycle.add(ruleId);
|
||||
// Claim this rule+service for the cycle before awaiting
|
||||
// dispatch so replicas of the same service skip.
|
||||
if (this.firedThisCycle.has(cooldownKey)) {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} service "${serviceName}" (already fired this cycle by sibling replica)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const { name: metricName, unit } = getMetricDetails(rule.metric);
|
||||
const operatorPhrase = getOperatorPhrase(rule.operator);
|
||||
this.firedThisCycle.add(cooldownKey);
|
||||
|
||||
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
|
||||
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
|
||||
const { name: metricName, unit } = getMetricDetails(rule.metric);
|
||||
const operatorPhrase = getOperatorPhrase(rule.operator);
|
||||
|
||||
// Node-neutral body: the hub bell badge attributes remotes.
|
||||
const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
|
||||
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
|
||||
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
|
||||
const serviceLabel = displayServiceName(serviceName);
|
||||
|
||||
console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`);
|
||||
await NotificationService.getInstance().dispatchAlert(
|
||||
// Node-neutral body: the hub bell badge attributes remotes.
|
||||
const message = `The **${metricName}** for **${serviceLabel}** in **${rule.stack_name}** (container **${containerName}**) ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
|
||||
|
||||
try {
|
||||
const { persisted } = await NotificationService.getInstance().dispatchAlert(
|
||||
'warning',
|
||||
'monitor_alert',
|
||||
message,
|
||||
{ stackName: rule.stack_name, actor: 'system:monitor' },
|
||||
{ stackName: rule.stack_name, containerName, actor: 'system:monitor' },
|
||||
);
|
||||
if (!persisted) {
|
||||
// History was not written; do not advance cooldown or we silence retries.
|
||||
this.firedThisCycle.delete(cooldownKey);
|
||||
console.error(
|
||||
`[MonitorService] Alert history not persisted for rule ${ruleId} service "${serviceName}"; cooldown not advanced`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const firedAt = Date.now();
|
||||
// Dual-write: per-service row for current code, parent last_fired_at
|
||||
// so a downgrade that only reads the parent column still has a floor.
|
||||
db.upsertStackAlertServiceCooldown(ruleId, serviceName, firedAt);
|
||||
db.updateStackAlertLastFired(ruleId, firedAt);
|
||||
console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}" service "${serviceName}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`);
|
||||
} catch (fireErr) {
|
||||
this.firedThisCycle.delete(cooldownKey);
|
||||
console.error(
|
||||
`[MonitorService] Failed to fire alert rule ${ruleId} service "${serviceName}" container "${containerName}":`,
|
||||
fireErr,
|
||||
);
|
||||
|
||||
db.updateStackAlertLastFired(ruleId, Date.now());
|
||||
}
|
||||
} else if (isDebugEnabled()) {
|
||||
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
|
||||
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId} service "${serviceName}": ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (this.activeBreaches.has(ruleId)) {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`);
|
||||
this.activeBreaches.delete(ruleId);
|
||||
if (this.activeBreaches.has(breachKey)) {
|
||||
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}" container "${containerName}"`);
|
||||
this.activeBreaches.delete(breachKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ export class NotificationService {
|
||||
category: NotificationCategory,
|
||||
message: string,
|
||||
options?: { stackName?: string; containerName?: string; actor?: string },
|
||||
) {
|
||||
): Promise<{ persisted: boolean }> {
|
||||
const t0 = Date.now();
|
||||
const { stackName, containerName, actor } = options ?? {};
|
||||
|
||||
@@ -191,6 +191,8 @@ export class NotificationService {
|
||||
// WebSocket broadcast can all throw on an unhealthy DB, which would
|
||||
// otherwise surface as an unhandledRejection and take the process down.
|
||||
// The whole body is wrapped so the worst case is a dropped notification.
|
||||
// Callers that gate cooldowns on history use `persisted` (true only after the row write).
|
||||
let wroteHistory = false;
|
||||
try {
|
||||
// Internal writes use the middleware default so they share a row key
|
||||
// with user-initiated requests; otherwise the UI and monitors split
|
||||
@@ -216,11 +218,12 @@ export class NotificationService {
|
||||
container_name: containerName,
|
||||
actor_username: actor ?? null,
|
||||
});
|
||||
wroteHistory = true;
|
||||
StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, true);
|
||||
} catch (err) {
|
||||
StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, false);
|
||||
console.error('[Notify] Failed to persist notification:', err);
|
||||
return;
|
||||
return { persisted: false };
|
||||
}
|
||||
// Separate [StackActivity:diag] namespace from the [Notify:diag] lines
|
||||
// below so a single grep can pull every per-stack timeline write across
|
||||
@@ -273,7 +276,7 @@ export class NotificationService {
|
||||
}
|
||||
|
||||
if (suppressExternal) {
|
||||
return;
|
||||
return { persisted: wroteHistory };
|
||||
}
|
||||
|
||||
// Resolve retry extras once for this dispatch (shared by all destinations).
|
||||
@@ -298,14 +301,14 @@ export class NotificationService {
|
||||
)
|
||||
);
|
||||
this.recordDispatchErrors(notification.id!, errors);
|
||||
return;
|
||||
return { persisted: wroteHistory };
|
||||
}
|
||||
|
||||
// 4. Fall back to this instance's agents (keyed by this instance's default node id).
|
||||
const agents = this.dbService.getEnabledAgents(localNodeId);
|
||||
if (agents.length === 0) {
|
||||
if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch');
|
||||
return;
|
||||
return { persisted: wroteHistory };
|
||||
}
|
||||
|
||||
if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`);
|
||||
@@ -322,8 +325,11 @@ export class NotificationService {
|
||||
)
|
||||
);
|
||||
this.recordDispatchErrors(notification.id!, errors);
|
||||
return { persisted: wroteHistory };
|
||||
} catch (err) {
|
||||
console.error('[Notify] dispatchAlert failed:', err);
|
||||
// History may already be written; callers must not treat that as a miss.
|
||||
return { persisted: wroteHistory };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user