mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-10 02:41:14 +00:00
fix(fleet): resolve stuck update states and improve detection (#405)
* fix(fleet): resolve stuck update states and improve update UX The fleet node update flow had several bugs: the in-memory update tracker never cleared terminal states (timeout, failed, completed), leaving nodes permanently stuck with no way to retry or dismiss. The Recheck button only re-fetched stale state without clearing it, and the POST trigger rejected retries with 409 even after timeout. Backend fixes: - Add DELETE endpoints (single node + batch) to clear tracker entries - Fix 409 race: detect expired timeouts and clear terminal states before re-triggering - Populate error messages in the tracker for timeouts and failures - Include error field in the update-status API response - Auto-expire completed entries after 60 seconds Frontend fixes: - Add retry (RotateCcw) and dismiss (X) buttons on failed/timed-out badges - Show error details via animated cursor hover (CursorFollow pattern) - Recheck button now batch-clears all terminal states before fetching - Recheck shows loading spinner and disables while checking - Extract NodeCardProps interface for readability * fix(fleet): detect update completion via process start time Remote nodes that cannot report their version (e.g. older builds) caused updates to always time out because completion detection relied solely on version comparison. The gateway now tracks the remote node's process start time from /api/meta and detects container restarts by comparing it across polls. Also extracts a createTracker() factory to eliminate repeated object construction across 5 call sites. * docs: add troubleshooting for first-update timeout on old nodes Adds a new troubleshooting entry explaining why the first remote update on nodes running pre-v0.40.0 always times out (neither version nor process start time can be detected). Documents the fix: dismiss, recheck, and confirm the node updated. Also adds a screenshot of the timed-out state with retry/dismiss buttons to the remote updates feature page. * fix(fleet): detect update completion via offline detection and error reporting The update completion detection relied on version change and process start time, both of which fail on nodes running older Sencho versions that report "unknown" and lack the startedAt field. This caused every update to time out after 5 minutes. Add three-signal detection: version change, process restart (startedAt), and offline/online detection (node went unreachable during update and came back). Also add a 90-second early failure heuristic for when the remote image pull fails silently, and surface pull errors from SelfUpdateService via /api/meta so the gateway can report them immediately. * fix(deps): bump vite to 8.0.5 to resolve high severity vulnerabilities Fixes GHSA-4w7w-66w2-5vf9, GHSA-v2wj-q39q-566r, GHSA-p9ff-h696-f583. * fix(deps): bump vite in backend lockfile to resolve audit failures Vitest pulls in vite as a transitive dependency. Bumps to 8.0.5.
This commit is contained in:
Anso
+135
-13
@@ -346,8 +346,16 @@ app.get('/api/health', (_req: Request, res: Response): void => {
|
||||
|
||||
// Public meta endpoint - returns this instance's version and supported capabilities.
|
||||
// No auth required (like /health). Used by remote nodes during connection tests.
|
||||
const processStartedAt = Date.now();
|
||||
|
||||
app.get('/api/meta', (_req: Request, res: Response): void => {
|
||||
res.json({ version: getSenchoVersion(), capabilities: getActiveCapabilities() });
|
||||
const updateError = SelfUpdateService.getInstance().getLastError();
|
||||
res.json({
|
||||
version: getSenchoVersion(),
|
||||
capabilities: getActiveCapabilities(),
|
||||
startedAt: processStartedAt,
|
||||
...(updateError ? { updateError } : {}),
|
||||
});
|
||||
});
|
||||
|
||||
// Auth Routes (no authentication required)
|
||||
@@ -1103,9 +1111,24 @@ interface UpdateTracker {
|
||||
startedAt: number;
|
||||
previousVersion: string | null;
|
||||
error?: string;
|
||||
/** Process start time of the remote node before the update was triggered. */
|
||||
previousProcessStart: number | null;
|
||||
/** True when the node became unreachable at least once during the update window. */
|
||||
wasOffline: boolean;
|
||||
}
|
||||
const updateTracker = new Map<number, UpdateTracker>();
|
||||
const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
const UPDATE_TIMEOUT_MSG = 'Node did not come back online within 5 minutes.';
|
||||
const EARLY_FAIL_MS = 90 * 1000; // 90 seconds before declaring a probable pull failure
|
||||
|
||||
function createTracker(
|
||||
status: UpdateTracker['status'],
|
||||
previousVersion: string | null,
|
||||
previousProcessStart: number | null,
|
||||
error?: string,
|
||||
): UpdateTracker {
|
||||
return { status, startedAt: Date.now(), previousVersion, previousProcessStart, wasOffline: false, error };
|
||||
}
|
||||
|
||||
interface FleetNodeOverview {
|
||||
id: number;
|
||||
@@ -1258,22 +1281,71 @@ app.get('/api/fleet/update-status', async (_req: Request, res: Response): Promis
|
||||
const tracker = updateTracker.get(node.id);
|
||||
|
||||
let version: string | null = null;
|
||||
let remoteStartedAt: number | null = null;
|
||||
let remoteUpdateError: string | null = null;
|
||||
let remoteOnline = false;
|
||||
if (node.type === 'local') {
|
||||
version = gatewayVersion;
|
||||
} else if (node.api_url && node.api_token) {
|
||||
const meta = await fetchRemoteMeta(node.api_url, node.api_token);
|
||||
version = meta.version;
|
||||
remoteStartedAt = meta.startedAt;
|
||||
remoteUpdateError = meta.updateError;
|
||||
remoteOnline = meta.online;
|
||||
}
|
||||
|
||||
// For nodes actively updating, check if they've come back with a new version
|
||||
// For nodes actively updating, check if they've come back
|
||||
if (tracker?.status === 'updating') {
|
||||
if (Date.now() - tracker.startedAt > UPDATE_TIMEOUT_MS) {
|
||||
updateTracker.set(node.id, { ...tracker, status: 'timeout' });
|
||||
} else if (node.type === 'remote' && version && version !== tracker.previousVersion) {
|
||||
updateTracker.set(node.id, { ...tracker, status: 'completed' });
|
||||
const elapsed = Date.now() - tracker.startedAt;
|
||||
|
||||
if (elapsed > UPDATE_TIMEOUT_MS) {
|
||||
// Final timeout (5 min)
|
||||
updateTracker.set(node.id, { ...tracker, status: 'timeout', error: UPDATE_TIMEOUT_MSG });
|
||||
} else if (node.type === 'remote') {
|
||||
if (remoteUpdateError) {
|
||||
// Remote reported a pull failure via /api/meta
|
||||
updateTracker.set(node.id, { ...tracker, status: 'failed', error: remoteUpdateError });
|
||||
} else if (!remoteOnline) {
|
||||
// Node is unreachable (restarting); record that it went offline
|
||||
if (!tracker.wasOffline) {
|
||||
updateTracker.set(node.id, { ...tracker, wasOffline: true });
|
||||
}
|
||||
} else if (version !== tracker.previousVersion) {
|
||||
// Signal 1: Version changed (or version now resolvable after being unknown)
|
||||
updateTracker.set(node.id, { ...tracker, status: 'completed' });
|
||||
} else if (
|
||||
remoteStartedAt !== null &&
|
||||
tracker.previousProcessStart !== null &&
|
||||
remoteStartedAt !== tracker.previousProcessStart
|
||||
) {
|
||||
// Signal 2: Process restarted (startedAt changed)
|
||||
updateTracker.set(node.id, { ...tracker, status: 'completed' });
|
||||
} else if (tracker.wasOffline && remoteOnline) {
|
||||
// Signal 3: Node went offline and is back online (container was recreated)
|
||||
updateTracker.set(node.id, { ...tracker, status: 'completed' });
|
||||
} else if (elapsed > EARLY_FAIL_MS) {
|
||||
// Heuristic: node never went offline and nothing changed after 90s
|
||||
updateTracker.set(node.id, {
|
||||
...tracker,
|
||||
status: 'failed',
|
||||
error: 'Update may have failed. The node is still running and its version has not changed.',
|
||||
});
|
||||
}
|
||||
} else if (node.type === 'local') {
|
||||
// Local node: check if SelfUpdateService reported a pull failure
|
||||
const localError = SelfUpdateService.getInstance().getLastError();
|
||||
if (localError) {
|
||||
updateTracker.set(node.id, { ...tracker, status: 'failed', error: localError });
|
||||
SelfUpdateService.getInstance().clearLastError();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-expire completed entries after 60 seconds so nodes return to "Up to date"
|
||||
if (tracker?.status === 'completed' && Date.now() - tracker.startedAt > 60_000) {
|
||||
updateTracker.delete(node.id);
|
||||
}
|
||||
|
||||
// Assume remote nodes are outdated when their version is unresolvable
|
||||
let updateAvailable = false;
|
||||
if (!isValidVersion(version)) {
|
||||
@@ -1291,6 +1363,7 @@ app.get('/api/fleet/update-status', async (_req: Request, res: Response): Promis
|
||||
latestVersion: gatewayVersion,
|
||||
updateAvailable,
|
||||
updateStatus: currentTracker?.status ?? null,
|
||||
error: currentTracker?.error ?? null,
|
||||
};
|
||||
})
|
||||
);
|
||||
@@ -1329,8 +1402,16 @@ app.post('/api/fleet/nodes/:nodeId/update', async (req: Request, res: Response):
|
||||
|
||||
const existing = updateTracker.get(nodeId);
|
||||
if (existing?.status === 'updating') {
|
||||
res.status(409).json({ error: 'Update already in progress for this node.' });
|
||||
return;
|
||||
if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) {
|
||||
updateTracker.set(nodeId, { ...existing, status: 'timeout', error: UPDATE_TIMEOUT_MSG });
|
||||
} else {
|
||||
res.status(409).json({ error: 'Update already in progress for this node.' });
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Clear terminal states to allow retry
|
||||
if (existing && (existing.status === 'timeout' || existing.status === 'failed' || existing.status === 'completed')) {
|
||||
updateTracker.delete(nodeId);
|
||||
}
|
||||
|
||||
if (node.type === 'local') {
|
||||
@@ -1338,7 +1419,7 @@ app.post('/api/fleet/nodes/:nodeId/update', async (req: Request, res: Response):
|
||||
res.status(503).json({ error: 'Self-update unavailable on the local node.' });
|
||||
return;
|
||||
}
|
||||
updateTracker.set(nodeId, { status: 'updating', startedAt: Date.now(), previousVersion: getSenchoVersion() });
|
||||
updateTracker.set(nodeId, createTracker('updating', getSenchoVersion(), null));
|
||||
scheduleLocalUpdate(res, 'Update initiated on local node. The server will restart shortly.');
|
||||
return;
|
||||
}
|
||||
@@ -1368,14 +1449,21 @@ app.post('/api/fleet/nodes/:nodeId/update', async (req: Request, res: Response):
|
||||
|
||||
if (!response.ok) {
|
||||
const err = await response.json().catch(() => ({}));
|
||||
res.status(502).json({ error: (err as Record<string, string>)?.error || 'Remote node rejected update request.' });
|
||||
const errorMsg = (err as Record<string, string>)?.error || 'Remote node rejected update request.';
|
||||
updateTracker.set(nodeId, createTracker('failed', meta.version, meta.startedAt, errorMsg));
|
||||
res.status(502).json({ error: errorMsg });
|
||||
return;
|
||||
}
|
||||
|
||||
updateTracker.set(nodeId, { status: 'updating', startedAt: Date.now(), previousVersion: meta.version });
|
||||
updateTracker.set(nodeId, createTracker('updating', meta.version, meta.startedAt));
|
||||
res.status(202).json({ message: `Update initiated on ${node.name}.` });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] Node update error:', error);
|
||||
const errorMsg = (error as Error)?.message || 'Failed to trigger node update.';
|
||||
const failedNodeId = parseInt(req.params.nodeId as string, 10);
|
||||
if (!isNaN(failedNodeId)) {
|
||||
updateTracker.set(failedNodeId, createTracker('failed', null, null, errorMsg));
|
||||
}
|
||||
res.status(500).json({ error: 'Failed to trigger node update.' });
|
||||
}
|
||||
});
|
||||
@@ -1391,8 +1479,13 @@ app.post('/api/fleet/update-all', async (req: Request, res: Response): Promise<v
|
||||
// Filter to eligible candidates, then trigger all in parallel
|
||||
const candidates = nodes.filter(node => {
|
||||
if (node.type === 'local') return false;
|
||||
if (updateTracker.get(node.id)?.status === 'updating') return false;
|
||||
const tracker = updateTracker.get(node.id);
|
||||
if (tracker?.status === 'updating') return false;
|
||||
if (!node.api_url || !node.api_token) return false;
|
||||
// Clear terminal states so they can be re-triggered
|
||||
if (tracker && (tracker.status === 'timeout' || tracker.status === 'failed' || tracker.status === 'completed')) {
|
||||
updateTracker.delete(node.id);
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
@@ -1410,7 +1503,7 @@ app.post('/api/fleet/update-all', async (req: Request, res: Response): Promise<v
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (response.ok) {
|
||||
updateTracker.set(node.id, { status: 'updating', startedAt: Date.now(), previousVersion: meta.version });
|
||||
updateTracker.set(node.id, createTracker('updating', meta.version, meta.startedAt));
|
||||
return { name: node.name, triggered: true };
|
||||
}
|
||||
return { name: node.name, triggered: false };
|
||||
@@ -1430,6 +1523,35 @@ app.post('/api/fleet/update-all', async (req: Request, res: Response): Promise<v
|
||||
}
|
||||
});
|
||||
|
||||
// Clear update tracker entry for a specific node (dismiss or before retry)
|
||||
app.delete('/api/fleet/nodes/:nodeId/update-status', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requirePaid(req, res)) return;
|
||||
try {
|
||||
const nodeId = parseInt(req.params.nodeId as string, 10);
|
||||
const node = DatabaseService.getInstance().getNode(nodeId);
|
||||
if (!node) {
|
||||
res.status(404).json({ error: 'Node not found' });
|
||||
return;
|
||||
}
|
||||
updateTracker.delete(nodeId);
|
||||
res.status(204).send();
|
||||
} catch (error) {
|
||||
console.error('[Fleet] Clear update status error:', error);
|
||||
res.status(500).json({ error: 'Failed to clear update status.' });
|
||||
}
|
||||
});
|
||||
|
||||
// Clear all terminal (timed-out, failed, completed) tracker entries at once
|
||||
app.delete('/api/fleet/update-status', async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requirePaid(req, res)) return;
|
||||
for (const [nodeId, tracker] of updateTracker) {
|
||||
if (tracker.status === 'timeout' || tracker.status === 'failed' || tracker.status === 'completed') {
|
||||
updateTracker.delete(nodeId);
|
||||
}
|
||||
}
|
||||
res.status(204).send();
|
||||
});
|
||||
|
||||
async function fetchLocalNodeOverview(node: Node): Promise<FleetNodeOverview> {
|
||||
try {
|
||||
const composeDir = path.resolve(NodeRegistry.getInstance().getComposeDir(node.id));
|
||||
|
||||
@@ -68,6 +68,11 @@ export function getSenchoVersion(): string | null {
|
||||
export interface RemoteMeta {
|
||||
version: string | null;
|
||||
capabilities: string[];
|
||||
startedAt: number | null;
|
||||
/** Error message from a failed self-update attempt on the remote node. */
|
||||
updateError: string | null;
|
||||
/** True when the /api/meta request succeeded (node is reachable). */
|
||||
online: boolean;
|
||||
}
|
||||
|
||||
// Runtime capability overrides — services call disableCapability() during init
|
||||
@@ -94,9 +99,12 @@ export async function fetchRemoteMeta(baseUrl: string, apiToken: string): Promis
|
||||
return {
|
||||
version: isValidVersion(rawVersion) ? rawVersion : null,
|
||||
capabilities: Array.isArray(res.data.capabilities) ? res.data.capabilities : [],
|
||||
startedAt: typeof res.data.startedAt === 'number' ? res.data.startedAt : null,
|
||||
updateError: typeof res.data.updateError === 'string' ? res.data.updateError : null,
|
||||
online: true,
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn(`[CapabilityRegistry] Failed to fetch meta from ${baseUrl}:`, (err as Error).message);
|
||||
return { version: null, capabilities: [] };
|
||||
return { version: null, capabilities: [], startedAt: null, updateError: null, online: false };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ class SelfUpdateService {
|
||||
private static instance: SelfUpdateService;
|
||||
private canSelfUpdate = false;
|
||||
private composeContext: ComposeContext | null = null;
|
||||
private lastUpdateError: string | null = null;
|
||||
|
||||
public static getInstance(): SelfUpdateService {
|
||||
if (!SelfUpdateService.instance) {
|
||||
@@ -57,10 +58,21 @@ class SelfUpdateService {
|
||||
return this.canSelfUpdate;
|
||||
}
|
||||
|
||||
/** Returns the error message from the last failed update attempt, or null. */
|
||||
getLastError(): string | null {
|
||||
return this.lastUpdateError;
|
||||
}
|
||||
|
||||
/** Clears the stored update error (call after reading it). */
|
||||
clearLastError(): void {
|
||||
this.lastUpdateError = null;
|
||||
}
|
||||
|
||||
triggerUpdate(): void {
|
||||
if (!this.composeContext) return;
|
||||
const { workingDir, configFiles, serviceName } = this.composeContext;
|
||||
const env = { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' };
|
||||
this.lastUpdateError = null;
|
||||
|
||||
console.log(`[SelfUpdate] Pulling latest image for ${serviceName}...`);
|
||||
try {
|
||||
@@ -71,7 +83,8 @@ class SelfUpdateService {
|
||||
timeout: 300_000, // 5 min max for pull
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[SelfUpdate] Pull failed:', (error as Error).message);
|
||||
this.lastUpdateError = (error as Error).message;
|
||||
console.error('[SelfUpdate] Pull failed:', this.lastUpdateError);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user