fix(fleet): show capabilities, version, metrics, and stacks for pilot-agent nodes (#1044)

When a pilot-agent node was the active node, the UI rendered "does not
advertise this capability" across most tabs, a perpetual "Update
available" badge, and a Fleet card body with blank CPU/RAM/Disk and "No
stacks found". The cause was central-side aggregators in /api/fleet/* and
/api/nodes/:id/meta only fanning out to proxy-mode remotes via
node.api_url + node.api_token, which are null for pilot-agent.

Route every affected aggregator through NodeRegistry.getProxyTarget so
the loopback URL backed by the active pilot tunnel is used uniformly:

- /api/nodes/:id/meta and /api/fleet/update-status fetch via the new
  NodeRegistry.fetchMetaForNode helper (resolves the target, delegates
  to fetchRemoteMeta, returns the shared OFFLINE_META on null).
- fetchRemoteNodeOverview, /api/fleet/configuration,
  /api/fleet/node/:nodeId/stacks, and the stack-containers drilldown
  fetch through target.apiUrl with conditional Authorization.
- fetchRemoteMeta omits the Authorization header when the token is empty
  (pilot-agent loopback) instead of sending a malformed Bearer string.
- Pilot-agent rows preserve pilot_last_seen and mirror it into
  last_successful_contact so the Fleet "last seen" cell renders the
  recent tunnel timestamp during a brief reconnect.

Pilot-mode capability filter excludes capabilities whose central-pilot
path is not yet wired (host-console, self-update). Without this, the
Console tab would surface for an Admiral pilot session and click
through to central's host because the WS upgrade handler still gates
on api_url + api_token. Filtered capabilities are removed at boot via
applyPilotModeCapabilityFilter when SENCHO_MODE=pilot.

Cache invalidation on tunnel-up: the meta cache for a reconnecting
pilot is dropped so the next request rebuilds capabilities and version
through the live bridge instead of waiting for the 3-minute TTL. The
namespace constant moves to helpers/cacheInvalidation.ts alongside the
new invalidateRemoteMetaCache helper.

Husky commit-msg hook: add the missing shebang and a .gitattributes
rule pinning .husky/* to LF line endings so commits do not fail with
"Exec format error" on Windows shells where autocrlf=true converts the
hook to CRLF.

Tests cover Authorization-header behavior, pilot-mode filter idempotency,
fetchMetaForNode dispatch (offline target, pilot-agent loopback,
proxy-mode), and the four affected fleet routes for pilot-agent both
when the tunnel is up and when it is down.
This commit is contained in:
Anso
2026-05-14 10:21:08 -04:00
committed by GitHub
parent e7a3b544c0
commit 8dd0fce621
11 changed files with 663 additions and 66 deletions
+64 -51
View File
@@ -222,39 +222,55 @@ async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
}
}
function pilotLastSeenSeconds(node: Node): number | null {
return node.mode === 'pilot_agent' && node.pilot_last_seen
? Math.floor(node.pilot_last_seen / 1000)
: null;
}
function noTargetMessage(node: Node): string {
return node.mode === 'pilot_agent'
? `Pilot tunnel to "${node.name}" is disconnected. Operations resume when the agent reconnects.`
: 'Remote node not configured';
}
function offlineRemoteOverview(node: Node, status: 'online' | 'offline'): FleetNodeOverview {
const pilotSeen = pilotLastSeenSeconds(node);
// For pilot-agent rows the tunnel heartbeat is the contact signal. Mirror
// it into last_successful_contact so the Fleet "last seen" cell renders
// the recent tunnel timestamp instead of a stale HTTP-success time.
const lastContact = pilotSeen ?? node.last_successful_contact ?? null;
return {
id: node.id,
name: node.name,
type: node.type,
mode: node.mode,
status,
stats: null,
systemStats: null,
stacks: null,
last_successful_contact: lastContact,
pilot_last_seen: pilotSeen,
cordoned: node.cordoned,
cordoned_at: node.cordoned_at,
cordoned_reason: node.cordoned_reason,
};
}
async function fetchRemoteNodeOverview(node: Node, db: DatabaseService): Promise<FleetNodeOverview> {
// Pilot-agent nodes: use pilot_last_seen as the contact signal; no HTTP fetch.
if (node.mode === 'pilot_agent') {
return {
id: node.id,
name: node.name,
type: node.type,
mode: node.mode,
status: node.pilot_last_seen ? 'online' : 'offline',
stats: null,
systemStats: null,
stacks: null,
last_successful_contact: node.pilot_last_seen ? Math.floor(node.pilot_last_seen / 1000) : null,
pilot_last_seen: node.pilot_last_seen ? Math.floor(node.pilot_last_seen / 1000) : null,
cordoned: node.cordoned,
cordoned_at: node.cordoned_at,
cordoned_reason: node.cordoned_reason,
};
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
// Soft-online keeps the Fleet card from flapping during a brief pilot
// tunnel reconnect: a recent pilot_last_seen still counts as reachable.
const status: 'online' | 'offline' =
node.mode === 'pilot_agent' && node.pilot_last_seen ? 'online' : 'offline';
return offlineRemoteOverview(node, status);
}
if (!node.api_url || !node.api_token) {
return {
id: node.id, name: node.name, type: node.type, status: 'offline',
stats: null, systemStats: null, stacks: null,
last_successful_contact: node.last_successful_contact ?? null,
cordoned: node.cordoned,
cordoned_at: node.cordoned_at,
cordoned_reason: node.cordoned_reason,
};
}
const baseUrl = node.api_url.replace(/\/$/, '');
const headers = { Authorization: `Bearer ${node.api_token}` };
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers: Record<string, string> = target.apiToken
? { Authorization: `Bearer ${target.apiToken}` }
: {};
const t0 = Date.now();
try {
@@ -309,20 +325,14 @@ async function fetchRemoteNodeOverview(node: Node, db: DatabaseService): Promise
last_successful_contact: isOnline
? Math.floor(completedAt / 1000)
: node.last_successful_contact ?? null,
pilot_last_seen: pilotLastSeenSeconds(node),
cordoned: node.cordoned,
cordoned_at: node.cordoned_at,
cordoned_reason: node.cordoned_reason,
};
} catch (error) {
console.error(`[Fleet] Remote node ${node.name} error:`, error);
return {
id: node.id, name: node.name, type: node.type, mode: node.mode, status: 'offline',
stats: null, systemStats: null, stacks: null,
last_successful_contact: node.last_successful_contact ?? null,
cordoned: node.cordoned,
cordoned_at: node.cordoned_at,
cordoned_reason: node.cordoned_reason,
};
return offlineRemoteOverview(node, 'offline');
}
}
@@ -552,16 +562,17 @@ fleetRouter.get('/configuration', authMiddleware, async (req: Request, res: Resp
};
}
if (!node.api_url || !node.api_token) {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
return { id: node.id, name: node.name, type: 'remote', status: 'offline', configuration: null };
}
try {
const resp = await fetch(
`${node.api_url.replace(/\/$/, '')}/api/dashboard/configuration`,
`${target.apiUrl.replace(/\/$/, '')}/api/dashboard/configuration`,
{
headers: {
Authorization: `Bearer ${node.api_token}`,
...(target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {}),
[PROXY_TIER_HEADER]: localTier,
[PROXY_VARIANT_HEADER]: localVariant ?? '',
},
@@ -606,12 +617,13 @@ fleetRouter.get('/node/:nodeId/stacks', authMiddleware, async (req: Request, res
}
if (node.type === 'remote') {
if (!node.api_url || !node.api_token) {
res.status(503).json({ error: 'Remote node not configured' });
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
res.status(503).json({ error: noTargetMessage(node) });
return;
}
const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks`, {
headers: { Authorization: `Bearer ${node.api_token}` },
const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks`, {
headers: target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {},
signal: AbortSignal.timeout(10000),
});
if (!response.ok) {
@@ -649,12 +661,13 @@ fleetRouter.get('/node/:nodeId/stacks/:stackName/containers', authMiddleware, as
}
if (node.type === 'remote') {
if (!node.api_url || !node.api_token) {
res.status(503).json({ error: 'Remote node not configured' });
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
res.status(503).json({ error: noTargetMessage(node) });
return;
}
const response = await fetch(`${node.api_url.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, {
headers: { Authorization: `Bearer ${node.api_token}` },
const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, {
headers: target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {},
signal: AbortSignal.timeout(10000),
});
if (!response.ok) {
@@ -696,8 +709,8 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
let remoteOnline = false;
if (node.type === 'local') {
version = gatewayVersion;
} else if (node.api_url && node.api_token) {
const meta = await fetchRemoteMeta(node.api_url, node.api_token);
} else {
const meta = await NodeRegistry.getInstance().fetchMetaForNode(node.id);
version = meta.version;
remoteStartedAt = meta.startedAt;
remoteUpdateError = meta.updateError;
+3 -9
View File
@@ -9,7 +9,8 @@ import { enrollmentLimiter } from '../middleware/rateLimiters';
import { DatabaseService } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry';
import { CacheService } from '../services/CacheService';
import { CAPABILITIES, getSenchoVersion, fetchRemoteMeta, type RemoteMeta } from '../services/CapabilityRegistry';
import { REMOTE_META_NAMESPACE } from '../helpers/cacheInvalidation';
import { CAPABILITIES, getSenchoVersion, type RemoteMeta } from '../services/CapabilityRegistry';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import { PilotCloseCode } from '../pilot/protocol';
import { FleetUpdateTrackerService } from '../services/FleetUpdateTrackerService';
@@ -18,7 +19,6 @@ import { isValidRemoteUrl } from '../utils/validation';
import { getErrorMessage } from '../utils/errors';
const NODE_SCOPE_MESSAGE = 'API tokens cannot manage nodes.';
const REMOTE_META_NAMESPACE = 'remote-meta';
const REMOTE_META_CACHE_TTL = 3 * 60 * 1000;
function mintPilotEnrollment(nodeId: number, req: Request): { token: string; expiresAt: number; dockerRun: string } {
@@ -354,18 +354,12 @@ nodesRouter.get('/:id/meta', authMiddleware, async (req: Request, res: Response)
return;
}
const baseUrl = node.api_url?.replace(/\/$/, '');
if (!baseUrl || !node.api_token) {
res.json({ version: null, capabilities: [] });
return;
}
const cacheKey = `${REMOTE_META_NAMESPACE}:${id}`;
const meta = await CacheService.getInstance().getOrFetch<RemoteMeta>(
cacheKey,
REMOTE_META_CACHE_TTL,
async () => {
const fetched = await fetchRemoteMeta(baseUrl, node.api_token!);
const fetched = await NodeRegistry.getInstance().fetchMetaForNode(id);
if (fetched.version === null) {
throw new Error('Remote meta fetch returned null version');
}