mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-12 19:57:37 +00:00
fix(mesh): persist PilotMetrics counters across central restart (#1078)
The 12 PilotMetrics counters (proxy_dials_failed, proxy_bridges_total, proxy_idle_closes, the pilot-tunnel counters, and the peer-to-central callback counters) live in a singleton holding scalar fields, so process restart wipes them. Operators investigating rotation- or dial-failure trends across a central restart lost the aggregate metric. Persist the snapshot under a pilot_metrics_counters JSON row on the existing system_state KV table. Hydrate via PilotMetrics.load() in bootstrap/startup.ts before any service that increments runs; final-flush in bootstrap/shutdown.ts before the SQLite handle closes. Between those boundaries a buffered flush mirrors the audit-log pattern: every 1 s or every 100 increments, whichever fires first. Hot increment path stays a single property write plus a pending-counter check. Defensive parse on the row: object check + numeric filter, so an operator hand-edit or schema drift across releases degrades to zero state with a warn rather than crashing boot. Schema additions back-fill missing keys with zero so a release that adds a counter does not need a migration. Closes F-R-4 from the mesh tracker.
This commit is contained in:
@@ -10,6 +10,7 @@ import { SchedulerService } from '../services/SchedulerService';
|
||||
import { MfaService } from '../services/MfaService';
|
||||
import { MeshService } from '../services/MeshService';
|
||||
import { BlueprintReconciler } from '../services/BlueprintReconciler';
|
||||
import { PilotMetrics } from '../services/PilotMetrics';
|
||||
|
||||
/**
|
||||
* Wire graceful shutdown handlers. Docker sends SIGTERM when the container
|
||||
@@ -49,6 +50,9 @@ export function installShutdownHandlers(server: Server): void {
|
||||
try { BlueprintReconciler.getInstance().stop(); } catch (e) {
|
||||
console.warn('[Shutdown] BlueprintReconciler cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { PilotMetrics.flush(); } catch (e) {
|
||||
console.warn('[Shutdown] PilotMetrics flush failed:', (e as Error).message);
|
||||
}
|
||||
try { DatabaseService.getInstance().flushAuditLogBuffer(); } catch (e) {
|
||||
console.warn('[Shutdown] Audit log flush failed:', (e as Error).message);
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import { MeshService } from '../services/MeshService';
|
||||
import { BlueprintReconciler } from '../services/BlueprintReconciler';
|
||||
import { applyPilotModeCapabilityFilter } from '../services/CapabilityRegistry';
|
||||
import { PilotTunnelManager } from '../services/PilotTunnelManager';
|
||||
import { PilotMetrics } from '../services/PilotMetrics';
|
||||
import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
|
||||
import { sweepStaleTempDirs as sweepStaleGitTempDirs } from '../services/GitSourceService';
|
||||
import { PORT } from '../helpers/constants';
|
||||
@@ -70,6 +71,15 @@ export async function startServer(server: Server): Promise<void> {
|
||||
applyPilotModeCapabilityFilter();
|
||||
}
|
||||
|
||||
// Hydrate pilot/mesh counters from the persisted snapshot before any
|
||||
// service that increments them (MeshService, PilotTunnelManager) starts.
|
||||
// Failures fall through to zero-initialized counters; do not block boot.
|
||||
try {
|
||||
PilotMetrics.load(DatabaseService.getInstance());
|
||||
} catch (err) {
|
||||
console.warn('[Startup] PilotMetrics load failed:', (err as Error).message);
|
||||
}
|
||||
|
||||
// Initialize the license service before any tier-gated code can run.
|
||||
LicenseService.getInstance().initialize();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user