gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-08-20 07:13:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(mesh): persist PilotMetrics counters across central restart (#1078)

Browse Source 
The 12 PilotMetrics counters (proxy_dials_failed, proxy_bridges_total,
proxy_idle_closes, the pilot-tunnel counters, and the peer-to-central
callback counters) live in a singleton holding scalar fields, so process
restart wipes them. Operators investigating rotation- or dial-failure
trends across a central restart lost the aggregate metric.

Persist the snapshot under a pilot_metrics_counters JSON row on the
existing system_state KV table. Hydrate via PilotMetrics.load() in
bootstrap/startup.ts before any service that increments runs; final-flush
in bootstrap/shutdown.ts before the SQLite handle closes. Between those
boundaries a buffered flush mirrors the audit-log pattern: every 1 s or
every 100 increments, whichever fires first. Hot increment path stays a
single property write plus a pending-counter check.

Defensive parse on the row: object check + numeric filter, so an operator
hand-edit or schema drift across releases degrades to zero state with a
warn rather than crashing boot. Schema additions back-fill missing keys
with zero so a release that adds a counter does not need a migration.

Closes F-R-4 from the mesh tracker.
This commit is contained in:
Anso
2026-05-17 01:55:17 -04:00
committed by GitHub
parent 9aaa8573c0
commit 6b728599c4
5 changed files with 369 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/DatabaseService.ts
+35
View File
@@ -605,6 +605,8 @@ export interface ScanSummary {
const AUDIT_LOG_FLUSH_INTERVAL_MS = 1_000;
const AUDIT_LOG_FLUSH_THRESHOLD = 100;
export const PILOT_METRICS_COUNTERS_KEY = 'pilot_metrics_counters';
export class DatabaseService {
private static instance: DatabaseService;
private db: Database.Database;
@@ -1775,6 +1777,39 @@ export class DatabaseService {
this.db.prepare('INSERT OR REPLACE INTO system_state (key, value) VALUES (?, ?)').run(key, value);
}
/**
* Persisted PilotMetrics counters. Returns the parsed JSON object (a
* numeric record keyed by counter name) or null when the row is missing
* or unparseable. Callers (PilotMetrics.load) handle per-field defaulting
* so a missing counter in the persisted blob does not break a new release
* that added the counter.
*
* This is the first JSON blob stored in `system_state`; mirror this
* parse/validate shape (object check + numeric filter) for any future
* JSON-shaped system_state row so an operator-edited row cannot crash
* boot.
*/
public getPilotMetricsCounters(): Record<string, number> | null {
const raw = this.getSystemState(PILOT_METRICS_COUNTERS_KEY);
if (raw === null) return null;
try {
const parsed = JSON.parse(raw) as unknown;
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return null;
const out: Record<string, number> = {};
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
if (typeof v === 'number' && Number.isFinite(v)) out[k] = v;
}
return out;
} catch (err) {
console.warn('[DatabaseService] pilot_metrics_counters JSON parse failed:', (err as Error).message);
return null;
}
}
public setPilotMetricsCounters(counters: Record<string, number>): void {
this.setSystemState(PILOT_METRICS_COUNTERS_KEY, JSON.stringify(counters));
}
/**
* Run `fn` inside a single SQLite transaction. better-sqlite3 promotes a
* nested call to a SAVEPOINT, so callers can compose this with methods
backend/src/services/PilotMetrics.ts
+133 -23
View File
@@ -1,16 +1,24 @@
/**
* PilotMetrics: in-memory counters for the pilot-agent reverse-tunnel
* subsystem. Strictly process-local — Sencho does not export metrics to any
* external sink (privacy posture, see CLAUDE.md). Counters reset on process
* restart by design; their purpose is operator support and debug, not
* long-term trend analysis.
* PilotMetrics: process-local counters for the pilot-agent reverse-tunnel
* subsystem and the proxy-mode mesh dialer. Strictly process-local. Sencho
* does not export metrics to any external sink (privacy posture, see
* CLAUDE.md). Their purpose is operator support and debug, surfaced via
* GET /api/system/pilot-tunnels (admin only).
*
* No general in-process metrics facility exists in the backend today, so this
* is a per-feature pattern. When a shared facility lands, this module should
* be replaced by an instance of it rather than grown.
* Counters are hydrated from SQLite on startup via load() and persisted via
* a buffered flush that mirrors the audit-log buffer pattern in
* DatabaseService: one flush per PILOT_METRICS_FLUSH_INTERVAL_MS or
* PILOT_METRICS_FLUSH_THRESHOLD increments, whichever comes first, plus an
* explicit flush() in the shutdown handler before the DB closes.
*
* No general in-process metrics facility exists in the backend today, so
* this is a per-feature pattern. When a shared facility lands, this module
* should be replaced by an instance of it rather than grown.
*/
interface Counters {
import type { DatabaseService } from './DatabaseService';
export interface Counters {
tunnels_total: number;
tunnels_replaced: number;
tunnels_rejected_capacity: number;
@@ -32,29 +40,131 @@ interface Counters {
mesh_callback_auth_failures_total: number;
}
const ZERO_COUNTERS: Counters = {
tunnels_total: 0,
tunnels_replaced: 0,
tunnels_rejected_capacity: 0,
enroll_acks: 0,
frame_decode_errors: 0,
proxy_bridges_total: 0,
proxy_dials_failed: 0,
proxy_idle_closes: 0,
proxy_bridges_peer_initiated_total: 0,
mesh_central_bootstraps_total: 0,
mesh_callback_dials_failed_total: 0,
mesh_callback_auth_failures_total: 0,
};
export const PILOT_METRICS_FLUSH_INTERVAL_MS = 1_000;
export const PILOT_METRICS_FLUSH_THRESHOLD = 100;
export interface PilotMetricsTestOverrides {
intervalMs?: number;
threshold?: number;
}
class PilotMetricsImpl {
private counters: Counters = {
tunnels_total: 0,
tunnels_replaced: 0,
tunnels_rejected_capacity: 0,
enroll_acks: 0,
frame_decode_errors: 0,
proxy_bridges_total: 0,
proxy_dials_failed: 0,
proxy_idle_closes: 0,
proxy_bridges_peer_initiated_total: 0,
mesh_central_bootstraps_total: 0,
mesh_callback_dials_failed_total: 0,
mesh_callback_auth_failures_total: 0,
};
private counters: Counters = { ...ZERO_COUNTERS };
private db: DatabaseService | null = null;
private pendingWrites = 0;
private flushTimer: ReturnType<typeof setTimeout> | null = null;
private intervalMs: number = PILOT_METRICS_FLUSH_INTERVAL_MS;
private threshold: number = PILOT_METRICS_FLUSH_THRESHOLD;
/**
* Hydrate counters from persisted JSON (if any) and wire periodic flush
* to disk. Idempotent: a second call rebinds the DB reference and
* re-hydrates from it but does not double-schedule the timer.
*/
public load(db: DatabaseService, overrides?: PilotMetricsTestOverrides): void {
this.db = db;
if (overrides?.intervalMs !== undefined) this.intervalMs = overrides.intervalMs;
if (overrides?.threshold !== undefined) this.threshold = overrides.threshold;
const persisted = db.getPilotMetricsCounters();
const next: Counters = { ...ZERO_COUNTERS };
if (persisted) {
for (const key of Object.keys(next) as Array<keyof Counters>) {
const value = persisted[key];
if (typeof value === 'number' && Number.isFinite(value)) {
next[key] = value;
}
}
}
this.counters = next;
this.pendingWrites = 0;
}
public increment<K extends keyof Counters>(name: K): void {
this.counters[name] += 1;
if (!this.db) return;
this.pendingWrites += 1;
if (this.pendingWrites >= this.threshold) {
this.flush();
return;
}
if (!this.flushTimer) {
this.flushTimer = setTimeout(() => {
this.flushTimer = null;
this.flush();
}, this.intervalMs);
// Allow process exit even when this timer is pending; shutdown
// calls flush() explicitly and a stuck DB write must not block
// SIGTERM.
if (typeof this.flushTimer.unref === 'function') {
this.flushTimer.unref();
}
}
}
public snapshot(): Counters {
return { ...this.counters };
}
/**
* Persist the current counter snapshot to SQLite. Safe to call from any
* path; the shutdown handler invokes this before closing the DB. A flush
* with no pending writes is a no-op (avoids writing on every shutdown
* even when nothing changed since load).
*/
public flush(): void {
if (this.flushTimer) {
clearTimeout(this.flushTimer);
this.flushTimer = null;
}
if (!this.db || this.pendingWrites === 0) return;
try {
this.db.setPilotMetricsCounters({ ...this.counters });
this.pendingWrites = 0;
} catch (err) {
// Keep counters in memory so the next flush retries with the
// accumulated value. Do NOT zero pendingWrites here.
console.error('[PilotMetrics] Failed to persist counters:', (err as Error).message);
}
}
/**
* Cancel any pending interval flush without writing. Intended for tests;
* the production shutdown path uses flush() instead so in-memory writes
* survive the restart.
*/
public stop(): void {
if (this.flushTimer) {
clearTimeout(this.flushTimer);
this.flushTimer = null;
}
}
/**
* Reset all in-memory state. Tests only.
*/
public resetForTests(): void {
this.stop();
this.counters = { ...ZERO_COUNTERS };
this.db = null;
this.pendingWrites = 0;
this.intervalMs = PILOT_METRICS_FLUSH_INTERVAL_MS;
this.threshold = PILOT_METRICS_FLUSH_THRESHOLD;
}
}
export const PilotMetrics = new PilotMetricsImpl();