Files
sencho/backend/src/__tests__/pilot-metrics.test.ts
T
Anso f6e42535c8 fix(mesh): route peer→central traffic over the existing forward WS (#1094)
* fix(mesh): route peer→central traffic over the existing forward WS

The reverse mesh callback path (`/api/mesh/proxy-tunnel-from-peer`) needed
SENCHO_PRIMARY_URL on central plus a publicly reachable origin from the
peer's perspective. In a typical homelab where central sits behind NAT,
peer→central dispatch silently failed at the dialer's short-circuit and
the headline "call any service on any node by hostname" worked one way
only.

The forward WS at `/api/mesh/proxy-tunnel` is already bidirectional end
to end. Make the bridge a persistent control-plane primitive: dial every
mesh-enabled proxy peer at startup, reconcile every 60 s, never idle-close.
Peer→central traffic multiplexes over the same WS via `tcp_open_reverse`.

Removed:
- `meshProxyTunnelFromPeer.ts` WS handler and dispatch
- `MeshCentralRegistry`, `PeerToCentralMeshSessionDialer`
- `mesh_handshake` first-frame state machine in `meshProxyTunnel.ts`
- `maybeSendBootstrap`, `buildHandshakeFrame` in the dialer
- `mesh_proxy_callback_bootstrap` capability and `maybeWarnUnsetPrimaryUrl`
- `mesh_centrals` table (drop migration; greenfield, no users)
- `PilotTunnelManager.replaceOrRegisterProxyBridge` (dead after handler removal)
- twelve associated unit/integration tests plus the peer-recovery branch
  in `MeshService.openCrossNode`

Added:
- `MeshService.proactiveBridgeFanout` selects every mesh-enabled proxy
  peer (no longer gated on `mesh_stacks` rows)
- `startBridgeReconcileLoop` runs the fanout every 60 s (override via
  `SENCHO_MESH_RECONCILE_INTERVAL_MS`)
- `MeshProxyTunnelDialer` default idle TTL is now `0` and exposes
  `isDialing(nodeId)` for the status surface
- `MeshNodeStatus.reverseCallbackStatus` discriminator
  (`connected | connecting | unavailable | not_applicable`) surfaced via
  `/api/mesh/status` and rendered as a pill in the Routing tab
- `openCrossNode` error message distinguishes "no proxy target" from
  "waiting for central to dial the reverse bridge"
- New tests: `mesh-service-proxy-tunnel-reconcile`,
  `mesh-status-reverse-callback`, `mesh-proxy-tunnel-dialer-no-idle-close`

SENCHO_PRIMARY_URL is no longer required for any mesh function.

* fix(mesh): rewrite proxy-tunnel reconcile test contents

The previous commit renamed the file but the rewritten test bodies stayed
unstaged on top of the rename. This commit lands the actual rewrite: the
fanout assertion now requires every mesh-enabled proxy peer to be dialed,
not just those with `mesh_stacks` rows, and adds a reconcile-tick
repeated-call test.
2026-05-17 22:00:31 -04:00

183 lines
7.5 KiB
TypeScript

/**
* PilotMetrics persistence behavior. Verifies the F-R-4 contract:
*
* - cold start: zero counters, no DB row
* - cold start with a persisted blob: snapshot reflects persisted values
* - increments queue writes that drain on the threshold or interval
* - explicit flush() persists the current snapshot
* - malformed JSON in the DB row degrades gracefully (warn, zero state)
* - persisted blob missing a counter back-fills that field with zero
* (schema-drift scenario for releases that add a new counter)
* - stop() cancels a pending interval flush without writing
*/
import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, vi } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let PilotMetrics: typeof import('../services/PilotMetrics').PilotMetrics;
let PILOT_METRICS_COUNTERS_KEY: string;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService, PILOT_METRICS_COUNTERS_KEY } = await import('../services/DatabaseService'));
({ PilotMetrics } = await import('../services/PilotMetrics'));
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
beforeEach(() => {
// Clear any persisted blob from a prior test and reset the singleton.
const db = DatabaseService.getInstance();
db.getDb().prepare('DELETE FROM system_state WHERE key = ?').run(PILOT_METRICS_COUNTERS_KEY);
PilotMetrics.resetForTests();
});
afterEach(() => {
PilotMetrics.stop();
vi.useRealTimers();
});
describe('PilotMetrics persistence', () => {
it('cold start: load with no row returns zeros and writes nothing', () => {
const db = DatabaseService.getInstance();
const setSpy = vi.spyOn(db, 'setPilotMetricsCounters');
PilotMetrics.load(db);
const snap = PilotMetrics.snapshot();
expect(snap.proxy_dials_failed).toBe(0);
expect(snap.proxy_bridges_total).toBe(0);
expect(snap.proxy_idle_closes).toBe(0);
expect(snap.tunnels_total).toBe(0);
PilotMetrics.flush();
expect(setSpy).not.toHaveBeenCalled();
setSpy.mockRestore();
});
it('cold start with persisted blob: snapshot reflects persisted values', () => {
const db = DatabaseService.getInstance();
db.setPilotMetricsCounters({
tunnels_total: 11,
tunnels_replaced: 1,
tunnels_rejected_capacity: 0,
enroll_acks: 0,
frame_decode_errors: 0,
proxy_bridges_total: 7,
proxy_dials_failed: 5,
proxy_idle_closes: 2,
});
PilotMetrics.load(db);
const snap = PilotMetrics.snapshot();
expect(snap.tunnels_total).toBe(11);
expect(snap.proxy_dials_failed).toBe(5);
expect(snap.proxy_bridges_total).toBe(7);
});
it('threshold flush: increments past the threshold trigger a single persist', () => {
const db = DatabaseService.getInstance();
const setSpy = vi.spyOn(db, 'setPilotMetricsCounters');
PilotMetrics.load(db, { threshold: 3, intervalMs: 60_000 });
PilotMetrics.increment('proxy_dials_failed');
PilotMetrics.increment('proxy_dials_failed');
expect(setSpy).not.toHaveBeenCalled();
PilotMetrics.increment('proxy_dials_failed');
expect(setSpy).toHaveBeenCalledTimes(1);
const written = setSpy.mock.calls[0][0];
expect(written.proxy_dials_failed).toBe(3);
setSpy.mockRestore();
});
it('interval flush: a single increment persists after the interval fires', () => {
vi.useFakeTimers();
const db = DatabaseService.getInstance();
const setSpy = vi.spyOn(db, 'setPilotMetricsCounters');
PilotMetrics.load(db, { threshold: 1000, intervalMs: 1_000 });
PilotMetrics.increment('proxy_bridges_total');
expect(setSpy).not.toHaveBeenCalled();
vi.advanceTimersByTime(1_000);
expect(setSpy).toHaveBeenCalledTimes(1);
expect(setSpy.mock.calls[0][0].proxy_bridges_total).toBe(1);
setSpy.mockRestore();
});
it('explicit flush with no pending writes is a no-op', () => {
const db = DatabaseService.getInstance();
const setSpy = vi.spyOn(db, 'setPilotMetricsCounters');
PilotMetrics.load(db);
PilotMetrics.flush();
expect(setSpy).not.toHaveBeenCalled();
setSpy.mockRestore();
});
it('explicit flush after one increment persists immediately', () => {
const db = DatabaseService.getInstance();
const setSpy = vi.spyOn(db, 'setPilotMetricsCounters');
PilotMetrics.load(db, { threshold: 1000, intervalMs: 60_000 });
PilotMetrics.increment('proxy_idle_closes');
PilotMetrics.flush();
expect(setSpy).toHaveBeenCalledTimes(1);
expect(setSpy.mock.calls[0][0].proxy_idle_closes).toBe(1);
setSpy.mockRestore();
});
it('malformed JSON in the persisted row degrades to zero state', () => {
const db = DatabaseService.getInstance();
db.setSystemState(PILOT_METRICS_COUNTERS_KEY, '{not json');
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
PilotMetrics.load(db);
const snap = PilotMetrics.snapshot();
expect(snap.proxy_dials_failed).toBe(0);
expect(warnSpy).toHaveBeenCalled();
PilotMetrics.increment('proxy_dials_failed');
expect(PilotMetrics.snapshot().proxy_dials_failed).toBe(1);
warnSpy.mockRestore();
});
it('schema drift: persisted blob missing a counter back-fills with zero', () => {
const db = DatabaseService.getInstance();
db.setSystemState(
PILOT_METRICS_COUNTERS_KEY,
JSON.stringify({ proxy_dials_failed: 9 }),
);
PilotMetrics.load(db);
const snap = PilotMetrics.snapshot();
expect(snap.proxy_dials_failed).toBe(9);
expect(snap.proxy_bridges_total).toBe(0);
expect(snap.tunnels_total).toBe(0);
expect(snap.proxy_idle_closes).toBe(0);
});
it('stop() cancels a pending interval flush without persisting', () => {
vi.useFakeTimers();
const db = DatabaseService.getInstance();
const setSpy = vi.spyOn(db, 'setPilotMetricsCounters');
PilotMetrics.load(db, { threshold: 1000, intervalMs: 1_000 });
PilotMetrics.increment('proxy_dials_failed');
PilotMetrics.stop();
vi.advanceTimersByTime(5_000);
expect(setSpy).not.toHaveBeenCalled();
setSpy.mockRestore();
});
it('flush() catches DB write errors without losing pending count', () => {
const db = DatabaseService.getInstance();
const setSpy = vi
.spyOn(db, 'setPilotMetricsCounters')
.mockImplementationOnce(() => { throw new Error('disk full'); })
.mockImplementationOnce(() => undefined);
const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
PilotMetrics.load(db, { threshold: 1000, intervalMs: 60_000 });
PilotMetrics.increment('proxy_dials_failed');
PilotMetrics.flush();
expect(setSpy).toHaveBeenCalledTimes(1);
expect(errSpy).toHaveBeenCalled();
// Counter still in memory; next successful flush persists the value.
PilotMetrics.flush();
expect(setSpy).toHaveBeenCalledTimes(2);
expect(setSpy.mock.calls[1][0].proxy_dials_failed).toBe(1);
setSpy.mockRestore();
errSpy.mockRestore();
});
});