Files
sencho/backend/src/services/FleetUpdateTrackerService.ts
T
Anso 381ed2a91f feat: add Admiral Hardened Build channel and business assurance surfaces (#1629)
* feat: add Admiral Hardened Build channel and business assurance surfaces

Introduce Studio Saelix entitlement-backed Hardened Build switching, a
single-flight image operation coordinator, Recovery Vault naming, Admiral
Account settings, and typed Fleet update failures while preserving Community
custom-repo and targetless pull-current updates.

* fix: harden image-op paths and clear CI CodeQL/pilot flake

Validate operation IDs before filesystem use, use hostname checks in Fleet
fetch mocks, sanitize registry probe logs, and swallow expected TCP teardown
errors in the pilot reverse-route post-handshake test.

* fix: sanitize image-op docker config write and probe logs

Allowlist-copy registry host keys and base64 auth before writing the
temp DOCKER_CONFIG, and log registry probe failures with a fixed message
so CodeQL no longer flags network-to-file and log-injection mediums.

* fix: address Admiral Hardened Build audit blockers

Expose imageChannel so hardened Fleet peers still POST for typed rejection, claim community updates before 202, terminalize helper failures, gate Hardened on paid, and align support/docs/e2e wording.

* fix: terminalize image ops on helper survival and aborted claims

* fix: prevent recreating persist from overwriting helper-exit failure

* test: assert helper-exit failure lands before recreating persist

* fix: keep current pointer when acknowledging a stale image operation
2026-07-14 10:47:54 -04:00

117 lines
4.2 KiB
TypeScript

export interface UpdateTracker {
status: 'updating' | 'completed' | 'timeout' | 'failed';
startedAt: number;
previousVersion: string | null;
error?: string;
/** Machine-readable failure code returned by a remote update request. */
code?: string;
/** Process start time of the remote node before the update was triggered. */
previousProcessStart: number | null;
/** True when the node became unreachable at least once during the update window. */
wasOffline: boolean;
/** Timestamp when the tracker transitioned to a terminal state (completed/failed/timeout). */
resolvedAt?: number;
}
export type TerminalStatus = 'completed' | 'failed' | 'timeout';
/** Hard ceiling for an in-flight update before it is declared timed out. */
export const UPDATE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
export const UPDATE_TIMEOUT_MSG = 'Node did not come back online within 5 minutes.';
/** How long a resolved `completed` tracker lingers before it is reaped, so the
* badge stays briefly visible after the update lands. */
export const TERMINAL_TTL_MS = 60 * 1000;
/**
* In-memory tracker for in-flight fleet node updates. Keyed by node id.
*
* State is intentionally process-local: a restart clears all trackers, which
* is correct because the primary's own restart means it cannot observe remote
* update progress anyway. Fleet routes consume this service to render and
* clear update status.
*/
export class FleetUpdateTrackerService {
private static instance: FleetUpdateTrackerService;
private readonly trackers = new Map<number, UpdateTracker>();
public static getInstance(): FleetUpdateTrackerService {
if (!FleetUpdateTrackerService.instance) {
FleetUpdateTrackerService.instance = new FleetUpdateTrackerService();
}
return FleetUpdateTrackerService.instance;
}
public get(nodeId: number): UpdateTracker | undefined {
return this.trackers.get(nodeId);
}
public set(nodeId: number, tracker: UpdateTracker): void {
this.trackers.set(nodeId, tracker);
}
public delete(nodeId: number): boolean {
return this.trackers.delete(nodeId);
}
public entries(): IterableIterator<[number, UpdateTracker]> {
return this.trackers.entries();
}
public size(): number {
return this.trackers.size;
}
/** Create a new tracker with `startedAt=now` and resolvedAt set if terminal. */
public create(
status: UpdateTracker['status'],
previousVersion: string | null,
previousProcessStart: number | null,
error?: string,
code?: string,
): UpdateTracker {
const now = Date.now();
return {
status,
startedAt: now,
previousVersion,
previousProcessStart,
wasOffline: false,
error,
code,
resolvedAt: status !== 'updating' ? now : undefined,
};
}
/** Return a copy of `tracker` transitioned to a terminal state, with resolvedAt=now. */
public resolve(tracker: UpdateTracker, status: TerminalStatus, error?: string): UpdateTracker {
return { ...tracker, status, resolvedAt: Date.now(), error };
}
/**
* Safety-net sweep driven off the monitor tick rather than the frontend poll.
* The `/api/fleet/update-status` poll is the primary resolver, but it only
* runs while a client is watching; this bounds trackers when nothing polls:
* an in-flight tracker past the ceiling is timed out, and a resolved
* `completed` badge past its visibility window is reaped (mirroring the
* poll's auto-expire). Failed/timeout trackers persist until the operator
* dismisses them, matching the poll's behaviour. Returns counts for logging.
*/
public sweepStale(): { timedOut: number; reaped: number } {
const now = Date.now();
let timedOut = 0;
let reaped = 0;
for (const [nodeId, tracker] of this.trackers) {
if (tracker.status === 'updating') {
if (now - tracker.startedAt > UPDATE_TIMEOUT_MS) {
this.trackers.set(nodeId, this.resolve(tracker, 'timeout', UPDATE_TIMEOUT_MSG));
timedOut++;
}
} else if (tracker.status === 'completed' && tracker.resolvedAt && now - tracker.resolvedAt > TERMINAL_TTL_MS) {
this.trackers.delete(nodeId);
reaped++;
}
}
return { timedOut, reaped };
}
}