mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-30 12:09:15 +00:00
fix(fleet): gate node update actions to admins and harden update tracking (#1272)
* fix(fleet): gate node update actions to admins and harden update tracking Node update affordances now render only for admins, matching the admin-only routes behind them. Previously a non-admin could open the Fleet view and see the per-node Update button, Update all, retry, dismiss, and Recheck controls, then get a 403 on click. Those controls are now hidden for non-admins, who still see read-only update status. Both update-status clear routes (per-node and bulk) now require admin, and the bulk recheck throttles its forced "latest published version" lookup so a caller cannot loop it to hammer the upstream registries; the response reports whether the refresh actually ran so the UI can surface a "checked recently" note. Completion detection no longer reports a node as Updated when it merely blips offline and returns on the same version with an unchanged process start time. That case stays in progress and is decided by the existing early-fail and timeout heuristics, so a momentary network glitch is not mistaken for a successful update. Failed and timed-out updates now emit an operator-visible warning, and a periodic safety-net sweep bounds in-flight trackers when no client is polling for status. * fix(fleet): harden update completion and recheck failure handling Refinements from review of the node self-update hardening: - Completion signal 1 now requires a valid version, not merely a different one. A node whose /api/meta momentarily omits or mangles its version (online, same process) reported version=null, which compared unequal to the previous version and falsely marked the update completed. It now stays in progress and is decided by the early-fail/timeout heuristics. - Terminal resolution is atomic: it re-reads the live tracker and transitions only if it is still in flight with the same start time, so two concurrent status polls cannot both warn or clobber each other's transition. - The operator warning for a failed or timed-out update now redacts secret-shaped text (bearer/basic/token/password, credentialed URLs) from the underlying error before logging, in addition to stripping control characters. - The Recheck button now surfaces an error toast when the request throws (network or auth failure), matching the existing non-ok-response path instead of only logging to the console.
This commit is contained in:
Anso
+76
-18
@@ -6,7 +6,7 @@ import type Dockerode from 'dockerode';
|
||||
import { DatabaseService, type Node } from '../services/DatabaseService';
|
||||
import { ControlIdentityMismatchError, FleetSyncService, StaleSyncPushError } from '../services/FleetSyncService';
|
||||
import { MAX_SYNC_ROWS, SYNC_ERROR_CODES } from '../services/fleetSyncConstants';
|
||||
import { FleetUpdateTrackerService } from '../services/FleetUpdateTrackerService';
|
||||
import { FleetUpdateTrackerService, type UpdateTracker, type TerminalStatus, UPDATE_TIMEOUT_MS, UPDATE_TIMEOUT_MSG, TERMINAL_TTL_MS } from '../services/FleetUpdateTrackerService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import DockerController from '../services/DockerController';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
@@ -29,7 +29,7 @@ import { withTimeout, TimeoutError } from '../utils/withTimeout';
|
||||
// paths cap the slow `docker system df` call at the same 8s budget (F-6).
|
||||
const FLEET_DF_TIMEOUT_MS = 8_000;
|
||||
import { POLICY_SEVERITIES } from '../utils/severity';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
|
||||
import { formatNoTargetError } from '../utils/remoteTarget';
|
||||
import { CloudBackupService } from '../services/CloudBackupService';
|
||||
import { NotificationService } from '../services/NotificationService';
|
||||
@@ -41,9 +41,44 @@ import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from '../services/license-hea
|
||||
import { LicenseService } from '../services/LicenseService';
|
||||
|
||||
const updateTracker = FleetUpdateTrackerService.getInstance();
|
||||
const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
const UPDATE_TIMEOUT_MSG = 'Node did not come back online within 5 minutes.';
|
||||
const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure
|
||||
// Throttle the forced latest-version refresh so a caller cannot loop the recheck
|
||||
// endpoint to hammer GitHub / Docker Hub. The 30-minute cache still serves reads
|
||||
// between forced refreshes; this only bounds how often we bypass it.
|
||||
const FORCED_RECHECK_COOLDOWN_MS = 2 * 60 * 1000; // 2 minutes
|
||||
let lastForcedRecheckAt = 0;
|
||||
|
||||
/** Test-only: reset the forced-recheck throttle clock so suites do not depend
|
||||
* on cross-test ordering of the module-scope timestamp. */
|
||||
export function _resetForcedRecheckThrottleForTests(): void {
|
||||
lastForcedRecheckAt = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically resolve an in-flight update tracker to a terminal state and store
|
||||
* it. Re-reads the live entry and transitions only if it is still 'updating'
|
||||
* with the same startedAt; because there is no await between the read and the
|
||||
* set, a concurrent /update-status poll that already resolved this node cannot
|
||||
* be clobbered or cause a duplicate WARN. For failure-class outcomes (failed /
|
||||
* timeout) it emits one operator-visible WARN so a failed fleet update is
|
||||
* observable without enabling developer mode. The error text is secret-redacted
|
||||
* and control-stripped before logging; no tokens or meta dumps.
|
||||
*/
|
||||
function resolveTerminal(
|
||||
node: { id: number; name: string },
|
||||
tracker: UpdateTracker,
|
||||
status: TerminalStatus,
|
||||
error?: string,
|
||||
): void {
|
||||
const live = updateTracker.get(node.id);
|
||||
if (!live || live.status !== 'updating' || live.startedAt !== tracker.startedAt) return;
|
||||
if (status !== 'completed') {
|
||||
const elapsedSec = Math.round((Date.now() - live.startedAt) / 1000);
|
||||
const detail = error ? `: ${sanitizeForLog(redactSensitiveText(error))}` : '';
|
||||
console.warn(`[Fleet] Node update ${status} for "${sanitizeForLog(node.name)}" (id ${node.id}) after ${elapsedSec}s${detail}`);
|
||||
}
|
||||
updateTracker.set(node.id, updateTracker.resolve(live, status, error));
|
||||
}
|
||||
|
||||
const CVE_ID_RE = /^(CVE-\d{4}-\d{4,}|GHSA-[\w-]{14,})$/;
|
||||
|
||||
@@ -727,17 +762,20 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
|
||||
|
||||
if (elapsed > UPDATE_TIMEOUT_MS) {
|
||||
if (debug) console.debug('[Fleet:debug] Node', node.id, 'timed out after', Math.round(elapsed / 1000) + 's');
|
||||
updateTracker.set(node.id, updateTracker.resolve(tracker, 'timeout', UPDATE_TIMEOUT_MSG));
|
||||
resolveTerminal(node, tracker, 'timeout', UPDATE_TIMEOUT_MSG);
|
||||
} else if (node.type === 'remote') {
|
||||
if (remoteUpdateError) {
|
||||
if (debug) console.debug('[Fleet:debug] Node', node.id, 'reported pull failure:', remoteUpdateError);
|
||||
updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', remoteUpdateError));
|
||||
resolveTerminal(node, tracker, 'failed', remoteUpdateError);
|
||||
} else if (!remoteOnline) {
|
||||
if (!tracker.wasOffline) {
|
||||
if (debug) console.debug('[Fleet:debug] Node', node.id, 'went offline (restarting)');
|
||||
updateTracker.set(node.id, { ...tracker, wasOffline: true });
|
||||
}
|
||||
} else if (version !== tracker.previousVersion) {
|
||||
} else if (isValidVersion(version) && version !== tracker.previousVersion) {
|
||||
// Signal 1: a valid, different version. A null/unparseable version
|
||||
// from a transient /api/meta blip is NOT a version change, so it
|
||||
// must not complete a still-running, same-process node here.
|
||||
if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 1 (version changed):', tracker.previousVersion, '->', version);
|
||||
updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
|
||||
} else if (
|
||||
@@ -747,8 +785,18 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
|
||||
) {
|
||||
if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 2 (process restarted):', tracker.previousProcessStart, '->', remoteStartedAt);
|
||||
updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
|
||||
} else if (tracker.wasOffline && remoteOnline) {
|
||||
if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 3 (offline then online)');
|
||||
} else if (
|
||||
tracker.wasOffline &&
|
||||
remoteOnline &&
|
||||
(remoteStartedAt === null || tracker.previousProcessStart === null)
|
||||
) {
|
||||
// Signal 3: offline-then-online is only trustworthy as a completion
|
||||
// signal when we cannot read the remote process start time. When
|
||||
// startedAt IS known and unchanged (signal 2 above did not fire),
|
||||
// the process never restarted, so a brief unreachable blip on the
|
||||
// same version must not be reported as a completed update; it falls
|
||||
// through to the early-fail / timeout heuristics instead.
|
||||
if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 3 (offline then online, startedAt unavailable)');
|
||||
updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
|
||||
} else if (
|
||||
elapsed > 15_000 &&
|
||||
@@ -764,7 +812,7 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
|
||||
updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed'));
|
||||
} else if (elapsed > EARLY_FAIL_MS) {
|
||||
if (debug) console.debug('[Fleet:debug] Node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's - no signals detected');
|
||||
updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', 'Update may have failed. The node is still running and its version has not changed.'));
|
||||
resolveTerminal(node, tracker, 'failed', 'Update may have failed. The node is still running and its version has not changed.');
|
||||
}
|
||||
} else if (node.type === 'local') {
|
||||
// Local node has only two failure signals: an explicit pull/spawn
|
||||
@@ -776,18 +824,18 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp
|
||||
const localError = selfUpdate.getLastError();
|
||||
if (localError) {
|
||||
if (debug) console.debug('[Fleet:debug] Local node', node.id, 'update failed:', localError);
|
||||
updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', localError));
|
||||
resolveTerminal(node, tracker, 'failed', localError);
|
||||
selfUpdate.clearLastError();
|
||||
} else if (elapsed > EARLY_FAIL_MS) {
|
||||
if (debug) console.debug('[Fleet:debug] Local node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's');
|
||||
updateTracker.set(node.id, updateTracker.resolve(tracker, 'failed', 'Local update did not complete. The container may not have restarted; check Docker logs on the host.'));
|
||||
resolveTerminal(node, tracker, 'failed', 'Local update did not complete. The container may not have restarted; check Docker logs on the host.');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-expire completed entries 60s after they resolved so the badge
|
||||
// is visible briefly after completion.
|
||||
if (tracker?.status === 'completed' && tracker.resolvedAt && Date.now() - tracker.resolvedAt > 60_000) {
|
||||
// Auto-expire completed entries after their visibility window so the
|
||||
// badge is visible briefly after completion.
|
||||
if (tracker?.status === 'completed' && tracker.resolvedAt && Date.now() - tracker.resolvedAt > TERMINAL_TTL_MS) {
|
||||
updateTracker.delete(node.id);
|
||||
}
|
||||
|
||||
@@ -1012,6 +1060,7 @@ fleetRouter.post('/update-all', authMiddleware, async (req: Request, res: Respon
|
||||
});
|
||||
|
||||
fleetRouter.delete('/nodes/:nodeId/update-status', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
try {
|
||||
const nodeId = parseIntParam(req, res, 'nodeId', 'node ID');
|
||||
if (nodeId === null) return;
|
||||
@@ -1029,16 +1078,25 @@ fleetRouter.delete('/nodes/:nodeId/update-status', authMiddleware, async (req: R
|
||||
});
|
||||
|
||||
fleetRouter.delete('/update-status', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
// Pre-fetch fresh latest version so the next GET has up-to-date data.
|
||||
if (!requireAdmin(req, res)) return;
|
||||
// Optionally pre-fetch a fresh latest version so the next GET compares against
|
||||
// it. Throttled so a caller cannot loop this to hammer the upstream registries;
|
||||
// `rechecked` tells the client whether the forced refresh actually ran.
|
||||
let rechecked = false;
|
||||
if (req.query.recheck === 'true') {
|
||||
await getLatestVersion(true);
|
||||
const now = Date.now();
|
||||
if (now - lastForcedRecheckAt >= FORCED_RECHECK_COOLDOWN_MS) {
|
||||
lastForcedRecheckAt = now;
|
||||
await getLatestVersion(true);
|
||||
rechecked = true;
|
||||
}
|
||||
}
|
||||
for (const [nodeId, tracker] of updateTracker.entries()) {
|
||||
if (tracker.status === 'timeout' || tracker.status === 'failed' || tracker.status === 'completed') {
|
||||
updateTracker.delete(nodeId);
|
||||
}
|
||||
}
|
||||
res.status(204).send();
|
||||
res.status(200).json({ rechecked });
|
||||
});
|
||||
|
||||
// ─── Fleet Actions: gateway-orchestrated endpoints (multi-node) ───
|
||||
|
||||
Reference in New Issue
Block a user