fix(fleet): show capabilities, version, metrics, and stacks for pilot-agent nodes (#1044)

When a pilot-agent node was the active node, the UI rendered "does not
advertise this capability" across most tabs, a perpetual "Update
available" badge, and a Fleet card body with blank CPU/RAM/Disk and "No
stacks found". The cause was central-side aggregators in /api/fleet/* and
/api/nodes/:id/meta only fanning out to proxy-mode remotes via
node.api_url + node.api_token, which are null for pilot-agent.

Route every affected aggregator through NodeRegistry.getProxyTarget so
the loopback URL backed by the active pilot tunnel is used uniformly:

- /api/nodes/:id/meta and /api/fleet/update-status fetch via the new
  NodeRegistry.fetchMetaForNode helper (resolves the target, delegates
  to fetchRemoteMeta, returns the shared OFFLINE_META on null).
- fetchRemoteNodeOverview, /api/fleet/configuration,
  /api/fleet/node/:nodeId/stacks, and the stack-containers drilldown
  fetch through target.apiUrl with conditional Authorization.
- fetchRemoteMeta omits the Authorization header when the token is empty
  (pilot-agent loopback) instead of sending a malformed Bearer string.
- Pilot-agent rows preserve pilot_last_seen and mirror it into
  last_successful_contact so the Fleet "last seen" cell renders the
  recent tunnel timestamp during a brief reconnect.

Pilot-mode capability filter excludes capabilities whose central-pilot
path is not yet wired (host-console, self-update). Without this, the
Console tab would surface for an Admiral pilot session and click
through to central's host because the WS upgrade handler still gates
on api_url + api_token. Filtered capabilities are removed at boot via
applyPilotModeCapabilityFilter when SENCHO_MODE=pilot.

Cache invalidation on tunnel-up: the meta cache for a reconnecting
pilot is dropped so the next request rebuilds capabilities and version
through the live bridge instead of waiting for the 3-minute TTL. The
namespace constant moves to helpers/cacheInvalidation.ts alongside the
new invalidateRemoteMetaCache helper.

Husky commit-msg hook: add the missing shebang and a .gitattributes
rule pinning .husky/* to LF line endings so commits do not fail with
"Exec format error" on Windows shells where autocrlf=true converts the
hook to CRLF.

Tests cover Authorization-header behavior, pilot-mode filter idempotency,
fetchMetaForNode dispatch (offline target, pilot-agent loopback,
proxy-mode), and the four affected fleet routes for pilot-agent both
when the tunnel is up and when it is down.
This commit is contained in:
Anso
2026-05-14 10:21:08 -04:00
committed by GitHub
parent e7a3b544c0
commit 8dd0fce621
11 changed files with 663 additions and 66 deletions
+5
View File
@@ -1,3 +1,8 @@
# Force LF line endings for shell scripts regardless of the developer's OS. # Force LF line endings for shell scripts regardless of the developer's OS.
# Shell scripts with CRLF endings will fail with "exec format error" in Linux containers. # Shell scripts with CRLF endings will fail with "exec format error" in Linux containers.
*.sh text eol=lf *.sh text eol=lf
# Husky git hooks are POSIX shell scripts without the .sh extension. They run
# via /usr/bin/env sh, so the same eol=lf rule must apply or autocrlf=true
# breaks commit/push on Windows shells that exec the file directly.
.husky/* text eol=lf
+1
View File
@@ -1 +1,2 @@
#!/usr/bin/env sh
npx --no -- commitlint --edit "$1" npx --no -- commitlint --edit "$1"
@@ -0,0 +1,93 @@
/**
* F9 regression guard for CapabilityRegistry:
*
* - fetchRemoteMeta omits the Authorization header when the apiToken is
* empty so the loopback bridge (used by pilot-agent proxy targets) is
* not handed a malformed `Bearer ` header.
* - applyPilotModeCapabilityFilter strips capabilities whose central->pilot
* path is not yet wired (host-console, self-update) so the frontend
* cannot offer them on a pilot-active session.
*/
import { afterEach, describe, expect, it, vi } from 'vitest';
import axios from 'axios';
import {
CAPABILITIES,
applyPilotModeCapabilityFilter,
enableCapability,
fetchRemoteMeta,
getActiveCapabilities,
} from '../services/CapabilityRegistry';
describe('fetchRemoteMeta Authorization header', () => {
afterEach(() => {
vi.restoreAllMocks();
});
it('sends Authorization: Bearer <token> when token is non-empty', async () => {
const getSpy = vi.spyOn(axios, 'get').mockResolvedValue({
data: { version: '0.76.7', capabilities: ['stacks'], startedAt: 1, updateError: null },
});
await fetchRemoteMeta('https://remote.example.com:1852', 'real-token');
expect(getSpy).toHaveBeenCalledTimes(1);
const init = getSpy.mock.calls[0][1] as { headers: Record<string, string> };
expect(init.headers).toEqual({ Authorization: 'Bearer real-token' });
});
it('omits Authorization entirely when token is empty (pilot-agent loopback)', async () => {
const getSpy = vi.spyOn(axios, 'get').mockResolvedValue({
data: { version: '0.76.7', capabilities: ['stacks'], startedAt: 1, updateError: null },
});
await fetchRemoteMeta('http://127.0.0.1:54321', '');
expect(getSpy).toHaveBeenCalledTimes(1);
const init = getSpy.mock.calls[0][1] as { headers: Record<string, string> };
expect(init.headers).toEqual({});
expect(init.headers).not.toHaveProperty('Authorization');
});
it('returns OFFLINE_META shape on transport failure', async () => {
vi.spyOn(axios, 'get').mockRejectedValue(new Error('connect ECONNREFUSED'));
const meta = await fetchRemoteMeta('http://127.0.0.1:54321', '');
expect(meta).toEqual({
version: null,
capabilities: [],
startedAt: null,
updateError: null,
online: false,
});
});
});
describe('applyPilotModeCapabilityFilter', () => {
afterEach(() => {
enableCapability('host-console');
enableCapability('self-update');
});
it('removes host-console and self-update from active capabilities', () => {
expect(CAPABILITIES).toContain('host-console');
expect(CAPABILITIES).toContain('self-update');
applyPilotModeCapabilityFilter();
const active = getActiveCapabilities();
expect(active).not.toContain('host-console');
expect(active).not.toContain('self-update');
expect(active).toContain('stacks');
});
it('is idempotent (safe to call multiple times)', () => {
applyPilotModeCapabilityFilter();
applyPilotModeCapabilityFilter();
const active = getActiveCapabilities();
expect(active).not.toContain('host-console');
expect(active.length).toBe(CAPABILITIES.length - 2);
});
});
@@ -0,0 +1,297 @@
/**
* F9 regression guard: fleet aggregator routes return real data for
* pilot-agent nodes (capabilities, version, metrics, stacks, drilldown,
* configuration) by dispatching through NodeRegistry.getProxyTarget instead
* of reading node.api_url/api_token directly.
*
* Pre-fix:
* - GET /api/fleet/overview returned stats=null/systemStats=null/stacks=null
* for pilot-agent rows.
* - GET /api/fleet/node/:id/stacks 503'd with "Remote node not configured".
* - GET /api/fleet/node/:id/stacks/:stack/containers 503'd the same way.
* - GET /api/fleet/update-status reported version=null and the Fleet card
* showed perpetual "Update available".
* - GET /api/fleet/configuration reported configuration=null.
*
* Post-fix: each surface fetches through the loopback URL when a pilot
* tunnel is active and degrades to a mode-aware offline shape when not.
*/
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let authHeader: string;
let pilotNodeId: number;
let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
const LOOPBACK = 'http://127.0.0.1:54321';
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
({ NodeRegistry } = await import('../services/NodeRegistry'));
({ DatabaseService } = await import('../services/DatabaseService'));
pilotNodeId = DatabaseService.getInstance().addNode({
name: 'pilot-parity-test',
type: 'remote',
mode: 'pilot_agent',
compose_dir: '/tmp',
is_default: false,
api_url: '',
api_token: '',
});
// Mark as recently seen so offline-status branches that key on
// pilot_last_seen render the expected shape.
DatabaseService.getInstance().updateNode(pilotNodeId, {
pilot_last_seen: Date.now(),
pilot_agent_version: '0.76.7',
});
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
authHeader = `Bearer ${token}`;
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
afterEach(() => {
vi.restoreAllMocks();
});
function mockTargetActive() {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => {
if (id === pilotNodeId) return { apiUrl: LOOPBACK, apiToken: '' };
return null;
});
}
function mockTargetOffline() {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue(null);
}
function mockFetch(handler: (url: string, init?: RequestInit) => Response | Promise<Response>) {
vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: Parameters<typeof fetch>[0], init?: RequestInit) => {
const url = typeof input === 'string'
? input
: input instanceof URL ? input.toString() : (input as Request).url;
return handler(url, init);
});
}
describe('GET /api/fleet/node/:nodeId/stacks (pilot-agent)', () => {
it('returns the pilot stacks via the loopback target', async () => {
mockTargetActive();
mockFetch((url) => {
expect(url).toBe(`${LOOPBACK}/api/stacks`);
return new Response(JSON.stringify(['audit-mesh-pilot', 'monitor']), {
status: 200, headers: { 'content-type': 'application/json' },
});
});
const res = await request(app)
.get(`/api/fleet/node/${pilotNodeId}/stacks`)
.set('Authorization', authHeader);
expect(res.status).toBe(200);
expect(res.body).toEqual(['audit-mesh-pilot', 'monitor']);
});
it('returns 503 with a pilot-tunnel-disconnected message when no target is available', async () => {
mockTargetOffline();
const fetchSpy = vi.spyOn(globalThis, 'fetch');
const res = await request(app)
.get(`/api/fleet/node/${pilotNodeId}/stacks`)
.set('Authorization', authHeader);
expect(res.status).toBe(503);
expect(res.body?.error).toMatch(/pilot tunnel/i);
expect(fetchSpy).not.toHaveBeenCalled();
});
it('omits Authorization when target.apiToken is empty', async () => {
mockTargetActive();
let observedHeaders: Record<string, string> | undefined;
mockFetch((_url, init) => {
observedHeaders = (init?.headers as Record<string, string> | undefined) ?? undefined;
return new Response('[]', { status: 200, headers: { 'content-type': 'application/json' } });
});
await request(app)
.get(`/api/fleet/node/${pilotNodeId}/stacks`)
.set('Authorization', authHeader);
expect(observedHeaders).toBeDefined();
expect(observedHeaders).not.toHaveProperty('Authorization');
});
});
describe('GET /api/fleet/node/:nodeId/stacks/:stackName/containers (pilot-agent)', () => {
it('returns containers via the loopback target', async () => {
mockTargetActive();
mockFetch((url) => {
expect(url).toBe(`${LOOPBACK}/api/stacks/audit-mesh-pilot/containers`);
return new Response(
JSON.stringify([{ id: 'c1', name: 'audit-mesh-pilot-echo-1', state: 'running' }]),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
});
const res = await request(app)
.get(`/api/fleet/node/${pilotNodeId}/stacks/audit-mesh-pilot/containers`)
.set('Authorization', authHeader);
expect(res.status).toBe(200);
expect(res.body).toHaveLength(1);
expect(res.body[0].name).toBe('audit-mesh-pilot-echo-1');
});
it('returns 503 with pilot-tunnel-disconnected when no target is available', async () => {
mockTargetOffline();
const res = await request(app)
.get(`/api/fleet/node/${pilotNodeId}/stacks/audit-mesh-pilot/containers`)
.set('Authorization', authHeader);
expect(res.status).toBe(503);
expect(res.body?.error).toMatch(/pilot tunnel/i);
});
});
describe('GET /api/fleet/overview (pilot-agent)', () => {
it('populates stats, systemStats, and stacks for pilot-agent rows when the tunnel is up', async () => {
mockTargetActive();
mockFetch((url) => {
if (url === `${LOOPBACK}/api/stats`) {
return new Response(
JSON.stringify({ active: 3, managed: 2, unmanaged: 1, exited: 0, total: 3 }),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
}
if (url === `${LOOPBACK}/api/system/stats`) {
return new Response(
JSON.stringify({
cpu: { usage: '12.3', cores: 4 },
memory: { total: 8000000000, used: 2000000000, free: 6000000000, usagePercent: '25.0' },
disk: { total: 10, used: 5, free: 5, usagePercent: '50.0' },
}),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
}
if (url === `${LOOPBACK}/api/stacks`) {
return new Response(JSON.stringify(['audit-mesh-pilot']), {
status: 200, headers: { 'content-type': 'application/json' },
});
}
return new Response('not found', { status: 404 });
});
const res = await request(app).get('/api/fleet/overview').set('Authorization', authHeader);
expect(res.status).toBe(200);
const pilotRow = (res.body as Array<Record<string, unknown>>).find(r => r.id === pilotNodeId);
expect(pilotRow).toBeDefined();
expect(pilotRow!.status).toBe('online');
expect(pilotRow!.stats).toEqual({ active: 3, managed: 2, unmanaged: 1, exited: 0, total: 3 });
expect(pilotRow!.systemStats).toMatchObject({
cpu: { usage: '12.3', cores: 4 },
memory: { usagePercent: '25.0' },
});
expect(pilotRow!.stacks).toEqual(['audit-mesh-pilot']);
expect(pilotRow!.pilot_last_seen).toBeTypeOf('number');
});
it('falls back to an offline shape when the tunnel is down, preserving pilot_last_seen', async () => {
mockTargetOffline();
const fetchSpy = vi.spyOn(globalThis, 'fetch');
const res = await request(app).get('/api/fleet/overview').set('Authorization', authHeader);
expect(res.status).toBe(200);
const pilotRow = (res.body as Array<Record<string, unknown>>).find(r => r.id === pilotNodeId);
expect(pilotRow).toBeDefined();
expect(pilotRow!.stats).toBeNull();
expect(pilotRow!.systemStats).toBeNull();
expect(pilotRow!.stacks).toBeNull();
expect(pilotRow!.pilot_last_seen).toBeTypeOf('number');
expect(fetchSpy).not.toHaveBeenCalled();
});
});
describe('GET /api/fleet/update-status (pilot-agent)', () => {
it('reports the pilot version via fetchMetaForNode', async () => {
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockImplementation(async (id: number) => {
if (id === pilotNodeId) {
return {
version: '0.76.7',
capabilities: ['stacks', 'containers'],
startedAt: 1700000000,
updateError: null,
online: true,
};
}
return { version: null, capabilities: [], startedAt: null, updateError: null, online: false };
});
const res = await request(app).get('/api/fleet/update-status').set('Authorization', authHeader);
expect(res.status).toBe(200);
const nodes = (res.body as { nodes: Array<Record<string, unknown>> }).nodes;
const pilotRow = nodes.find(n => n.nodeId === pilotNodeId);
expect(pilotRow).toBeDefined();
expect(pilotRow!.version).toBe('0.76.7');
});
it('reports null version when the pilot meta fetch returns offline', async () => {
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue({
version: null,
capabilities: [],
startedAt: null,
updateError: null,
online: false,
});
const res = await request(app).get('/api/fleet/update-status').set('Authorization', authHeader);
expect(res.status).toBe(200);
const nodes = (res.body as { nodes: Array<Record<string, unknown>> }).nodes;
const pilotRow = nodes.find(n => n.nodeId === pilotNodeId);
expect(pilotRow!.version).toBeNull();
});
});
describe('GET /api/fleet/configuration (pilot-agent)', () => {
it('fetches the dashboard configuration via the loopback target', async () => {
mockTargetActive();
mockFetch((url) => {
expect(url).toBe(`${LOOPBACK}/api/dashboard/configuration`);
return new Response(
JSON.stringify({ ssoConfigured: false, alertsConfigured: true }),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
});
const res = await request(app).get('/api/fleet/configuration').set('Authorization', authHeader);
expect(res.status).toBe(200);
const pilotRow = (res.body as Array<Record<string, unknown>>).find(r => r.id === pilotNodeId);
expect(pilotRow).toBeDefined();
expect(pilotRow!.status).toBe('online');
expect(pilotRow!.configuration).toMatchObject({ alertsConfigured: true });
});
it('returns offline configuration=null when the tunnel is down', async () => {
mockTargetOffline();
const res = await request(app).get('/api/fleet/configuration').set('Authorization', authHeader);
expect(res.status).toBe(200);
const pilotRow = (res.body as Array<Record<string, unknown>>).find(r => r.id === pilotNodeId);
expect(pilotRow!.status).toBe('offline');
expect(pilotRow!.configuration).toBeNull();
});
});
@@ -0,0 +1,131 @@
/**
* F9 regression guard for NodeRegistry.fetchMetaForNode:
*
* - Resolves getProxyTarget for the node and delegates to fetchRemoteMeta.
* - Pilot-agent with active tunnel resolves to a loopback URL with empty
* token; the request must reach fetchRemoteMeta with that exact shape.
* - Null target (proxy-mode missing api_url/api_token, or pilot-agent
* tunnel disconnected) returns OFFLINE_META without touching the network.
*/
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
import axios from 'axios';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
let tmpDir: string;
let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ NodeRegistry } = await import('../services/NodeRegistry'));
({ DatabaseService } = await import('../services/DatabaseService'));
});
afterAll(() => {
cleanupTestDb(tmpDir);
});
afterEach(() => {
vi.restoreAllMocks();
});
describe('NodeRegistry.fetchMetaForNode', () => {
it('returns OFFLINE_META when getProxyTarget is null', async () => {
const reg = NodeRegistry.getInstance();
const db = DatabaseService.getInstance();
const nodeId = db.addNode({
name: 'meta-pilot-down',
type: 'remote',
mode: 'pilot_agent',
compose_dir: '/tmp',
is_default: false,
api_url: '',
api_token: '',
});
vi.spyOn(reg, 'getProxyTarget').mockReturnValue(null);
const axiosSpy = vi.spyOn(axios, 'get');
const meta = await reg.fetchMetaForNode(nodeId);
expect(meta).toEqual({
version: null,
capabilities: [],
startedAt: null,
updateError: null,
online: false,
});
expect(axiosSpy).not.toHaveBeenCalled();
db.deleteNode(nodeId);
});
it('delegates to fetchRemoteMeta against the loopback URL for pilot-agent', async () => {
const reg = NodeRegistry.getInstance();
const db = DatabaseService.getInstance();
const nodeId = db.addNode({
name: 'meta-pilot-up',
type: 'remote',
mode: 'pilot_agent',
compose_dir: '/tmp',
is_default: false,
api_url: '',
api_token: '',
});
vi.spyOn(reg, 'getProxyTarget').mockReturnValue({
apiUrl: 'http://127.0.0.1:54321',
apiToken: '',
});
const axiosSpy = vi.spyOn(axios, 'get').mockResolvedValue({
data: {
version: '0.76.7',
capabilities: ['stacks', 'containers'],
startedAt: 1234,
updateError: null,
},
});
const meta = await reg.fetchMetaForNode(nodeId);
expect(meta.version).toBe('0.76.7');
expect(meta.capabilities).toEqual(['stacks', 'containers']);
expect(meta.online).toBe(true);
expect(axiosSpy).toHaveBeenCalledTimes(1);
const url = axiosSpy.mock.calls[0][0];
expect(url).toBe('http://127.0.0.1:54321/api/meta');
const init = axiosSpy.mock.calls[0][1] as { headers: Record<string, string> };
expect(init.headers).toEqual({});
db.deleteNode(nodeId);
});
it('forwards Authorization for proxy-mode targets with non-empty tokens', async () => {
const reg = NodeRegistry.getInstance();
const db = DatabaseService.getInstance();
const nodeId = db.addNode({
name: 'meta-proxy',
type: 'remote',
mode: 'proxy',
compose_dir: '/tmp',
is_default: false,
api_url: 'https://remote.example.com:1852',
api_token: 'real-token',
});
vi.spyOn(reg, 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'real-token',
});
const axiosSpy = vi.spyOn(axios, 'get').mockResolvedValue({
data: { version: '0.76.7', capabilities: [], startedAt: 1, updateError: null },
});
await reg.fetchMetaForNode(nodeId);
const init = axiosSpy.mock.calls[0][1] as { headers: Record<string, string> };
expect(init.headers).toEqual({ Authorization: 'Bearer real-token' });
db.deleteNode(nodeId);
});
});
+18 -2
View File
@@ -15,9 +15,16 @@ import { SchedulerService } from '../services/SchedulerService';
import { MfaService } from '../services/MfaService'; import { MfaService } from '../services/MfaService';
import { MeshService } from '../services/MeshService'; import { MeshService } from '../services/MeshService';
import { BlueprintReconciler } from '../services/BlueprintReconciler'; import { BlueprintReconciler } from '../services/BlueprintReconciler';
import { applyPilotModeCapabilityFilter } from '../services/CapabilityRegistry';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
import { sweepStaleTempDirs as sweepStaleGitTempDirs } from '../services/GitSourceService'; import { sweepStaleTempDirs as sweepStaleGitTempDirs } from '../services/GitSourceService';
import { PORT } from '../helpers/constants'; import { PORT } from '../helpers/constants';
function isPilotMode(): boolean {
return process.env.SENCHO_MODE === 'pilot';
}
/** /**
* Pilot-agent hosts never run the first-run setup wizard, so the wizard * Pilot-agent hosts never run the first-run setup wizard, so the wizard
* path that normally generates `auth_jwt_secret` (routes/auth.ts) never * path that normally generates `auth_jwt_secret` (routes/auth.ts) never
@@ -32,7 +39,7 @@ import { PORT } from '../helpers/constants';
* Returns true when a fresh secret was written, false otherwise. * Returns true when a fresh secret was written, false otherwise.
*/ */
export function ensurePilotJwtSecret(): boolean { export function ensurePilotJwtSecret(): boolean {
if (process.env.SENCHO_MODE !== 'pilot') return false; if (!isPilotMode()) return false;
const dbSvc = DatabaseService.getInstance(); const dbSvc = DatabaseService.getInstance();
if (dbSvc.getGlobalSettings().auth_jwt_secret) return false; if (dbSvc.getGlobalSettings().auth_jwt_secret) return false;
const generated = crypto.randomBytes(64).toString('hex'); const generated = crypto.randomBytes(64).toString('hex');
@@ -59,6 +66,10 @@ export async function startServer(server: Server): Promise<void> {
ensurePilotJwtSecret(); ensurePilotJwtSecret();
if (isPilotMode()) {
applyPilotModeCapabilityFilter();
}
// Initialize the license service before any tier-gated code can run. // Initialize the license service before any tier-gated code can run.
LicenseService.getInstance().initialize(); LicenseService.getInstance().initialize();
@@ -76,6 +87,11 @@ export async function startServer(server: Server): Promise<void> {
}); });
BlueprintReconciler.getInstance().start(); BlueprintReconciler.getInstance().start();
// Drop the cached /api/meta entry on tunnel reconnect so the next
// /api/nodes/:id/meta refetches fresh capabilities and version through
// the live loopback bridge instead of waiting for the 3-minute TTL.
PilotTunnelManager.getInstance().on('tunnel-up', invalidateRemoteMetaCache);
// Async initializers are independent of each other; run in parallel // Async initializers are independent of each other; run in parallel
// so total boot time is the slowest one rather than the sum. // so total boot time is the slowest one rather than the sum.
await Promise.all([ await Promise.all([
@@ -92,7 +108,7 @@ export async function startServer(server: Server): Promise<void> {
console.warn('[Trivy] Temp dir sweep failed:', (err as Error).message); console.warn('[Trivy] Temp dir sweep failed:', (err as Error).message);
}); });
const isPilotAgent = process.env.SENCHO_MODE === 'pilot'; const isPilotAgent = isPilotMode();
const listenHost = isPilotAgent ? '127.0.0.1' : undefined; const listenHost = isPilotAgent ? '127.0.0.1' : undefined;
server.listen(PORT, listenHost, () => { server.listen(PORT, listenHost, () => {
+11
View File
@@ -1,5 +1,7 @@
import { CacheService } from '../services/CacheService'; import { CacheService } from '../services/CacheService';
export const REMOTE_META_NAMESPACE = 'remote-meta';
/** /**
* Drop the per-node caches affected by a stack or container mutation so the * Drop the per-node caches affected by a stack or container mutation so the
* next dashboard poll shows fresh state instead of stale reads. * next dashboard poll shows fresh state instead of stale reads.
@@ -14,3 +16,12 @@ export function invalidateNodeCaches(nodeId: number): void {
cache.invalidate(`stack-statuses:${nodeId}`); cache.invalidate(`stack-statuses:${nodeId}`);
cache.invalidate('project-name-map'); cache.invalidate('project-name-map');
} }
/**
* Drop the cached `/api/meta` response for a remote node. Triggered on pilot
* tunnel reconnect so the next request rebuilds capabilities and version
* through the live loopback bridge instead of waiting for the TTL.
*/
export function invalidateRemoteMetaCache(nodeId: number): void {
CacheService.getInstance().invalidate(`${REMOTE_META_NAMESPACE}:${nodeId}`);
}
+53 -40
View File
@@ -222,39 +222,55 @@ async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
} }
} }
async function fetchRemoteNodeOverview(node: Node, db: DatabaseService): Promise<FleetNodeOverview> { function pilotLastSeenSeconds(node: Node): number | null {
// Pilot-agent nodes: use pilot_last_seen as the contact signal; no HTTP fetch. return node.mode === 'pilot_agent' && node.pilot_last_seen
if (node.mode === 'pilot_agent') { ? Math.floor(node.pilot_last_seen / 1000)
: null;
}
function noTargetMessage(node: Node): string {
return node.mode === 'pilot_agent'
? `Pilot tunnel to "${node.name}" is disconnected. Operations resume when the agent reconnects.`
: 'Remote node not configured';
}
function offlineRemoteOverview(node: Node, status: 'online' | 'offline'): FleetNodeOverview {
const pilotSeen = pilotLastSeenSeconds(node);
// For pilot-agent rows the tunnel heartbeat is the contact signal. Mirror
// it into last_successful_contact so the Fleet "last seen" cell renders
// the recent tunnel timestamp instead of a stale HTTP-success time.
const lastContact = pilotSeen ?? node.last_successful_contact ?? null;
return { return {
id: node.id, id: node.id,
name: node.name, name: node.name,
type: node.type, type: node.type,
mode: node.mode, mode: node.mode,
status: node.pilot_last_seen ? 'online' : 'offline', status,
stats: null, stats: null,
systemStats: null, systemStats: null,
stacks: null, stacks: null,
last_successful_contact: node.pilot_last_seen ? Math.floor(node.pilot_last_seen / 1000) : null, last_successful_contact: lastContact,
pilot_last_seen: node.pilot_last_seen ? Math.floor(node.pilot_last_seen / 1000) : null, pilot_last_seen: pilotSeen,
cordoned: node.cordoned, cordoned: node.cordoned,
cordoned_at: node.cordoned_at, cordoned_at: node.cordoned_at,
cordoned_reason: node.cordoned_reason, cordoned_reason: node.cordoned_reason,
}; };
}
async function fetchRemoteNodeOverview(node: Node, db: DatabaseService): Promise<FleetNodeOverview> {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
// Soft-online keeps the Fleet card from flapping during a brief pilot
// tunnel reconnect: a recent pilot_last_seen still counts as reachable.
const status: 'online' | 'offline' =
node.mode === 'pilot_agent' && node.pilot_last_seen ? 'online' : 'offline';
return offlineRemoteOverview(node, status);
} }
if (!node.api_url || !node.api_token) { const baseUrl = target.apiUrl.replace(/\/$/, '');
return { const headers: Record<string, string> = target.apiToken
id: node.id, name: node.name, type: node.type, status: 'offline', ? { Authorization: `Bearer ${target.apiToken}` }
stats: null, systemStats: null, stacks: null, : {};
last_successful_contact: node.last_successful_contact ?? null,
cordoned: node.cordoned,
cordoned_at: node.cordoned_at,
cordoned_reason: node.cordoned_reason,
};
}
const baseUrl = node.api_url.replace(/\/$/, '');
const headers = { Authorization: `Bearer ${node.api_token}` };
const t0 = Date.now(); const t0 = Date.now();
try { try {
@@ -309,20 +325,14 @@ async function fetchRemoteNodeOverview(node: Node, db: DatabaseService): Promise
last_successful_contact: isOnline last_successful_contact: isOnline
? Math.floor(completedAt / 1000) ? Math.floor(completedAt / 1000)
: node.last_successful_contact ?? null, : node.last_successful_contact ?? null,
pilot_last_seen: pilotLastSeenSeconds(node),
cordoned: node.cordoned, cordoned: node.cordoned,
cordoned_at: node.cordoned_at, cordoned_at: node.cordoned_at,
cordoned_reason: node.cordoned_reason, cordoned_reason: node.cordoned_reason,
}; };
} catch (error) { } catch (error) {
console.error(`[Fleet] Remote node ${node.name} error:`, error); console.error(`[Fleet] Remote node ${node.name} error:`, error);
return { return offlineRemoteOverview(node, 'offline');
id: node.id, name: node.name, type: node.type, mode: node.mode, status: 'offline',
stats: null, systemStats: null, stacks: null,
last_successful_contact: node.last_successful_contact ?? null,
cordoned: node.cordoned,
cordoned_at: node.cordoned_at,
cordoned_reason: node.cordoned_reason,
};
} }
} }
@@ -552,16 +562,17 @@ fleetRouter.get('/configuration', authMiddleware, async (req: Request, res: Resp
}; };
} }
if (!node.api_url || !node.api_token) { const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
return { id: node.id, name: node.name, type: 'remote', status: 'offline', configuration: null }; return { id: node.id, name: node.name, type: 'remote', status: 'offline', configuration: null };
} }
try { try {
const resp = await fetch( const resp = await fetch(
`${node.api_url.replace(/\/$/, '')}/api/dashboard/configuration`, `${target.apiUrl.replace(/\/$/, '')}/api/dashboard/configuration`,
{ {
headers: { headers: {
Authorization: `Bearer ${node.api_token}`, ...(target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {}),
[PROXY_TIER_HEADER]: localTier, [PROXY_TIER_HEADER]: localTier,
[PROXY_VARIANT_HEADER]: localVariant ?? '', [PROXY_VARIANT_HEADER]: localVariant ?? '',
}, },
@@ -606,12 +617,13 @@ fleetRouter.get('/node/:nodeId/stacks', authMiddleware, async (req: Request, res
} }
if (node.type === 'remote') { if (node.type === 'remote') {
if (!node.api_url || !node.api_token) { const target = NodeRegistry.getInstance().getProxyTarget(node.id);
res.status(503).json({ error: 'Remote node not configured' }); if (!target) {
res.status(503).json({ error: noTargetMessage(node) });
return; return;
} }
const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks`, { const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks`, {
headers: { Authorization: `Bearer ${node.api_token}` }, headers: target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {},
signal: AbortSignal.timeout(10000), signal: AbortSignal.timeout(10000),
}); });
if (!response.ok) { if (!response.ok) {
@@ -649,12 +661,13 @@ fleetRouter.get('/node/:nodeId/stacks/:stackName/containers', authMiddleware, as
} }
if (node.type === 'remote') { if (node.type === 'remote') {
if (!node.api_url || !node.api_token) { const target = NodeRegistry.getInstance().getProxyTarget(node.id);
res.status(503).json({ error: 'Remote node not configured' }); if (!target) {
res.status(503).json({ error: noTargetMessage(node) });
return; return;
} }
const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, { const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, {
headers: { Authorization: `Bearer ${node.api_token}` }, headers: target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {},
signal: AbortSignal.timeout(10000), signal: AbortSignal.timeout(10000),
}); });
if (!response.ok) { if (!response.ok) {
@@ -696,8 +709,8 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
let remoteOnline = false; let remoteOnline = false;
if (node.type === 'local') { if (node.type === 'local') {
version = gatewayVersion; version = gatewayVersion;
} else if (node.api_url && node.api_token) { } else {
const meta = await fetchRemoteMeta(node.api_url, node.api_token); const meta = await NodeRegistry.getInstance().fetchMetaForNode(node.id);
version = meta.version; version = meta.version;
remoteStartedAt = meta.startedAt; remoteStartedAt = meta.startedAt;
remoteUpdateError = meta.updateError; remoteUpdateError = meta.updateError;
+3 -9
View File
@@ -9,7 +9,8 @@ import { enrollmentLimiter } from '../middleware/rateLimiters';
import { DatabaseService } from '../services/DatabaseService'; import { DatabaseService } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry'; import { NodeRegistry } from '../services/NodeRegistry';
import { CacheService } from '../services/CacheService'; import { CacheService } from '../services/CacheService';
import { CAPABILITIES, getSenchoVersion, fetchRemoteMeta, type RemoteMeta } from '../services/CapabilityRegistry'; import { REMOTE_META_NAMESPACE } from '../helpers/cacheInvalidation';
import { CAPABILITIES, getSenchoVersion, type RemoteMeta } from '../services/CapabilityRegistry';
import { PilotTunnelManager } from '../services/PilotTunnelManager'; import { PilotTunnelManager } from '../services/PilotTunnelManager';
import { PilotCloseCode } from '../pilot/protocol'; import { PilotCloseCode } from '../pilot/protocol';
import { FleetUpdateTrackerService } from '../services/FleetUpdateTrackerService'; import { FleetUpdateTrackerService } from '../services/FleetUpdateTrackerService';
@@ -18,7 +19,6 @@ import { isValidRemoteUrl } from '../utils/validation';
import { getErrorMessage } from '../utils/errors'; import { getErrorMessage } from '../utils/errors';
const NODE_SCOPE_MESSAGE = 'API tokens cannot manage nodes.'; const NODE_SCOPE_MESSAGE = 'API tokens cannot manage nodes.';
const REMOTE_META_NAMESPACE = 'remote-meta';
const REMOTE_META_CACHE_TTL = 3 * 60 * 1000; const REMOTE_META_CACHE_TTL = 3 * 60 * 1000;
function mintPilotEnrollment(nodeId: number, req: Request): { token: string; expiresAt: number; dockerRun: string } { function mintPilotEnrollment(nodeId: number, req: Request): { token: string; expiresAt: number; dockerRun: string } {
@@ -354,18 +354,12 @@ nodesRouter.get('/:id/meta', authMiddleware, async (req: Request, res: Response)
return; return;
} }
const baseUrl = node.api_url?.replace(/\/$/, '');
if (!baseUrl || !node.api_token) {
res.json({ version: null, capabilities: [] });
return;
}
const cacheKey = `${REMOTE_META_NAMESPACE}:${id}`; const cacheKey = `${REMOTE_META_NAMESPACE}:${id}`;
const meta = await CacheService.getInstance().getOrFetch<RemoteMeta>( const meta = await CacheService.getInstance().getOrFetch<RemoteMeta>(
cacheKey, cacheKey,
REMOTE_META_CACHE_TTL, REMOTE_META_CACHE_TTL,
async () => { async () => {
const fetched = await fetchRemoteMeta(baseUrl, node.api_token!); const fetched = await NodeRegistry.getInstance().fetchMetaForNode(id);
if (fetched.version === null) { if (fetched.version === null) {
throw new Error('Remote meta fetch returned null version'); throw new Error('Remote meta fetch returned null version');
} }
+28 -3
View File
@@ -79,7 +79,7 @@ export interface RemoteMeta {
online: boolean; online: boolean;
} }
// Runtime capability overrides services call disableCapability() during init // Runtime capability overrides; services call disableCapability() during init.
const disabledCapabilities = new Set<Capability>(); const disabledCapabilities = new Set<Capability>();
export function disableCapability(c: Capability): void { export function disableCapability(c: Capability): void {
@@ -96,11 +96,36 @@ export function getActiveCapabilities(): readonly string[] {
return CAPABILITIES.filter(c => !disabledCapabilities.has(c)); return CAPABILITIES.filter(c => !disabledCapabilities.has(c));
} }
/**
* Capabilities a pilot-agent process should hide from its own /api/meta because
* the central->pilot path for them is not yet wired through the reverse tunnel.
* Surfacing them would let the frontend offer a tab whose click silently falls
* through to central's local handler.
*/
const PILOT_DISABLED_CAPABILITIES: readonly Capability[] = [
'host-console',
'self-update',
];
/** Disable capabilities that require a central->pilot path that is not yet wired. */
export function applyPilotModeCapabilityFilter(): void {
for (const cap of PILOT_DISABLED_CAPABILITIES) disableCapability(cap);
}
/** Shared offline shape returned when a remote node is unreachable. */
export const OFFLINE_META: RemoteMeta = {
version: null,
capabilities: [],
startedAt: null,
updateError: null,
online: false,
};
/** Fetch /api/meta from a remote Sencho instance. Returns empty data on failure. */ /** Fetch /api/meta from a remote Sencho instance. Returns empty data on failure. */
export async function fetchRemoteMeta(baseUrl: string, apiToken: string): Promise<RemoteMeta> { export async function fetchRemoteMeta(baseUrl: string, apiToken: string): Promise<RemoteMeta> {
try { try {
const res = await axios.get(`${baseUrl.replace(/\/$/, '')}/api/meta`, { const res = await axios.get(`${baseUrl.replace(/\/$/, '')}/api/meta`, {
headers: { Authorization: `Bearer ${apiToken}` }, headers: apiToken ? { Authorization: `Bearer ${apiToken}` } : {},
timeout: 5000, timeout: 5000,
}); });
const rawVersion: string | undefined = res.data.version; const rawVersion: string | undefined = res.data.version;
@@ -113,6 +138,6 @@ export async function fetchRemoteMeta(baseUrl: string, apiToken: string): Promis
}; };
} catch (err) { } catch (err) {
console.warn(`[CapabilityRegistry] Failed to fetch meta from ${baseUrl}:`, (err as Error).message); console.warn(`[CapabilityRegistry] Failed to fetch meta from ${baseUrl}:`, (err as Error).message);
return { version: null, capabilities: [], startedAt: null, updateError: null, online: false }; return { ...OFFLINE_META };
} }
} }
+12 -1
View File
@@ -2,7 +2,7 @@ import Docker from 'dockerode';
import axios from 'axios'; import axios from 'axios';
import { EventEmitter } from 'events'; import { EventEmitter } from 'events';
import { DatabaseService, Node } from './DatabaseService'; import { DatabaseService, Node } from './DatabaseService';
import { fetchRemoteMeta } from './CapabilityRegistry'; import { fetchRemoteMeta, OFFLINE_META, RemoteMeta } from './CapabilityRegistry';
import { PilotTunnelManager } from './PilotTunnelManager'; import { PilotTunnelManager } from './PilotTunnelManager';
/** /**
@@ -119,6 +119,17 @@ export class NodeRegistry extends EventEmitter {
return { apiUrl: node.api_url, apiToken: node.api_token }; return { apiUrl: node.api_url, apiToken: node.api_token };
} }
/**
* Fetch /api/meta from a remote node, dispatching through the proxy
* target. Returns OFFLINE_META when no target is reachable (proxy-mode
* missing api_url/api_token, or pilot-agent tunnel disconnected).
*/
public async fetchMetaForNode(nodeId: number): Promise<RemoteMeta> {
const target = this.getProxyTarget(nodeId);
if (!target) return { ...OFFLINE_META };
return fetchRemoteMeta(target.apiUrl, target.apiToken);
}
/** /**
* Test connectivity to a specific node. * Test connectivity to a specific node.
* - Local: pings the Docker daemon directly * - Local: pings the Docker daemon directly