Files
sencho/backend/src/services/SelfUpdateService.ts
T
Anso 3d69746eee fix(fleet): make local self-update flow reliable end-to-end (#472)
The "Updating Sencho..." overlay used to dismiss prematurely while the
image pull was still running, after which the local node card would get
stuck in "updating" and eventually surface a generic "Timed Out" error
while the container remained on the old version.

Three root causes are addressed:

1. The image pull was synchronous (`execFileSync`), which blocked the
   Node event loop. The overlay's health probe saw the server come back
   the moment the pull finished and reloaded the page, even though the
   container had not restarted yet. The pull is now async via
   `promisify(execFile)`, so /api/health and /api/fleet/update-status
   keep serving throughout.

2. The overlay reloaded on the first 200 from /api/health regardless of
   whether the underlying process had actually restarted. /api/health
   now exposes the gateway boot timestamp, and the overlay captures it
   pre-update and only reloads when it observes a different value. A
   wasOffline-then-online fallback handles the case where the pre-update
   fetch failed.

3. Helper container spawn errors from `docker run` were silently
   discarded, so a failed compose recreate never surfaced anywhere.
   Errors are now captured into `lastUpdateError` via the execFile
   callback and surfaced through the existing /api/fleet/update-status
   error path.

A 3-minute early-fail heuristic on the local node block surfaces a clear
failure message when the helper fails silently, instead of waiting the
full 5-minute timeout for an unknown failure.
2026-04-10 12:00:44 -04:00

146 lines
5.4 KiB
TypeScript

import { execFileSync, execFile } from 'child_process';
import { promisify } from 'util';
import DockerController from './DockerController';
import { disableCapability } from './CapabilityRegistry';
const execFileAsync = promisify(execFile);
interface ComposeContext {
workingDir: string;
configFiles: string;
serviceName: string;
imageName: string;
}
class SelfUpdateService {
private static instance: SelfUpdateService;
private canSelfUpdate = false;
private composeContext: ComposeContext | null = null;
private lastUpdateError: string | null = null;
public static getInstance(): SelfUpdateService {
if (!SelfUpdateService.instance) {
SelfUpdateService.instance = new SelfUpdateService();
}
return SelfUpdateService.instance;
}
async initialize(): Promise<void> {
const hostname = process.env.HOSTNAME;
if (!hostname) {
console.log('[SelfUpdate] HOSTNAME not set - self-update unavailable (not running in Docker?)');
disableCapability('self-update');
return;
}
try {
const docker = DockerController.getInstance().getDocker();
const container = docker.getContainer(hostname);
const info = await container.inspect();
const labels = info.Config?.Labels ?? {};
const workingDir = labels['com.docker.compose.project.working_dir'];
const configFiles = labels['com.docker.compose.project.config_files'];
const serviceName = labels['com.docker.compose.service'];
if (!workingDir || !configFiles || !serviceName) {
console.log('[SelfUpdate] Container lacks Docker Compose labels - self-update unavailable');
disableCapability('self-update');
return;
}
// Verify docker compose CLI is available inside the container
try {
execFileSync('docker', ['compose', 'version'], { stdio: 'pipe', timeout: 5000 });
} catch {
console.log('[SelfUpdate] docker compose CLI not available in container');
disableCapability('self-update');
return;
}
// Read the container's own image name for direct docker pull
const imageName = info.Config?.Image;
if (!imageName) {
console.log('[SelfUpdate] Could not determine container image name');
disableCapability('self-update');
return;
}
this.composeContext = { workingDir, configFiles, serviceName, imageName };
this.canSelfUpdate = true;
console.log(`[SelfUpdate] Ready - service="${serviceName}" image="${imageName}" in ${workingDir}`);
} catch (error) {
console.log('[SelfUpdate] Could not inspect own container - self-update unavailable:', (error as Error).message);
disableCapability('self-update');
}
}
isAvailable(): boolean {
return this.canSelfUpdate;
}
/** Returns the error message from the last failed update attempt, or null. */
getLastError(): string | null {
return this.lastUpdateError;
}
/** Clears the stored update error (call after reading it). */
clearLastError(): void {
this.lastUpdateError = null;
}
async triggerUpdate(): Promise<void> {
if (!this.composeContext) return;
const { workingDir, configFiles, serviceName, imageName } = this.composeContext;
const env = { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' };
this.lastUpdateError = null;
// Async pull: a sync execFileSync blocks the event loop, which lets the frontend
// overlay see a false "online" response between the pull finishing and the restart.
console.log(`[SelfUpdate] Pulling latest image: ${imageName}...`);
try {
await execFileAsync('docker', ['pull', imageName], {
env,
timeout: 300_000, // 5 min max for pull
});
} catch (error) {
const stderr = (error as { stderr?: Buffer | string })?.stderr?.toString().trim();
this.lastUpdateError = stderr || (error as Error).message;
console.error('[SelfUpdate] Pull failed:', this.lastUpdateError);
return;
}
// Step 2: Spawn a helper container to run docker compose recreate.
// The main container cannot access the compose file because the host path
// from Docker labels does not exist inside this container. The helper
// explicitly mounts the compose working directory from the host, so the
// compose file is accessible at the original path.
console.log(`[SelfUpdate] Spawning updater container... (last breath)`);
const fFlags = configFiles.split(',').flatMap(f => ['-f', f.trim()]);
const composeCmd = ['sleep 3 && docker compose', ...fFlags, 'up -d --force-recreate', serviceName].join(' ');
const args = [
'run', '--rm', '-d',
'--user', 'root',
'--entrypoint', 'sh',
'-v', '/var/run/docker.sock:/var/run/docker.sock',
'-v', `${workingDir}:${workingDir}:ro`,
'-w', workingDir,
imageName,
'-c', composeCmd,
];
// Capture spawn errors (bad image, missing socket, permission denied) so they
// land in lastUpdateError instead of vanishing silently.
execFile('docker', args, { env }, (err, _stdout, stderr) => {
if (err) {
const stderrText = stderr?.toString().trim();
this.lastUpdateError = stderrText || err.message || 'Helper container failed to spawn';
console.error('[SelfUpdate] Helper container spawn failed:', this.lastUpdateError);
}
});
// No code after this point is guaranteed to run: the helper recreates this container.
}
}
export default SelfUpdateService;