mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-07 09:24:09 +00:00
fix(fleet): capture local self-update helper errors (#495)
The helper container that runs `docker compose up --force-recreate` was spawned with `docker run -d`, so the command returned immediately with just the container ID. Any failure happening INSIDE the helper (bad compose file, image mismatch, permission issue, socket problem) was invisible: execFile's callback only fired for `docker run` command errors, never for errors inside the detached helper. The UI fell back to the generic 3-minute "Local update did not complete" heuristic with no actionable information. The helper now runs attached, so execFile's callback receives the helper's exit code and stderr directly for any failure that happens before the recreate kills this process. Additionally, the helper persists exit code + stderr to `/app/data/.sencho-update-error` before exiting, so the error survives the gateway's own death. On startup, `SelfUpdateService.recoverPreviousError()` reads and deletes that file, routing the real error through the existing `getLastError()` path so the freshly booted gateway reports exactly why the previous attempt failed instead of the generic timeout.
This commit is contained in:
Anso
@@ -1,15 +1,22 @@
|
||||
import { execFileSync, execFile } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
import * as fs from 'fs';
|
||||
import DockerController from './DockerController';
|
||||
import { disableCapability } from './CapabilityRegistry';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
// Error file written by the helper container on a failed compose recreate.
|
||||
// Must live under /app/data so both the helper (via host-path bind mount) and
|
||||
// the NEW gateway process (which always mounts /app/data) can reach it.
|
||||
const UPDATE_ERROR_FILE = '/app/data/.sencho-update-error';
|
||||
|
||||
interface ComposeContext {
|
||||
workingDir: string;
|
||||
configFiles: string;
|
||||
serviceName: string;
|
||||
imageName: string;
|
||||
dataDirHost: string | null;
|
||||
}
|
||||
|
||||
class SelfUpdateService {
|
||||
@@ -66,9 +73,22 @@ class SelfUpdateService {
|
||||
return;
|
||||
}
|
||||
|
||||
this.composeContext = { workingDir, configFiles, serviceName, imageName };
|
||||
// Find the host path backing /app/data. The helper container will bind-mount
|
||||
// it at the same path so compose recreate errors can be persisted to a file
|
||||
// that survives the gateway's restart and is readable by the NEW process.
|
||||
const mounts = (info.Mounts ?? []) as Array<{ Source?: string; Destination?: string }>;
|
||||
const dataDirHost = mounts.find(m => m.Destination === '/app/data')?.Source ?? null;
|
||||
if (!dataDirHost) {
|
||||
console.log('[SelfUpdate] /app/data mount not found - update error recovery will be unavailable');
|
||||
}
|
||||
|
||||
this.composeContext = { workingDir, configFiles, serviceName, imageName, dataDirHost };
|
||||
this.canSelfUpdate = true;
|
||||
console.log(`[SelfUpdate] Ready - service="${serviceName}" image="${imageName}" in ${workingDir}`);
|
||||
|
||||
// Surface any error from a previous failed update attempt (persisted by
|
||||
// the helper container) so the new process can report it to the user.
|
||||
this.recoverPreviousError();
|
||||
} catch (error) {
|
||||
console.log('[SelfUpdate] Could not inspect own container - self-update unavailable:', (error as Error).message);
|
||||
disableCapability('self-update');
|
||||
@@ -89,12 +109,31 @@ class SelfUpdateService {
|
||||
this.lastUpdateError = null;
|
||||
}
|
||||
|
||||
/** Surfaces any error the helper container persisted before the previous
|
||||
* gateway process died, then deletes the file. */
|
||||
private recoverPreviousError(): void {
|
||||
try {
|
||||
const content = fs.readFileSync(UPDATE_ERROR_FILE, 'utf8').trim();
|
||||
if (content) {
|
||||
this.lastUpdateError = content;
|
||||
console.error('[SelfUpdate] Recovered error from previous update attempt:', content);
|
||||
}
|
||||
fs.unlinkSync(UPDATE_ERROR_FILE);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
console.error('[SelfUpdate] Failed to recover previous update error:', (error as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async triggerUpdate(): Promise<void> {
|
||||
if (!this.composeContext) return;
|
||||
const { workingDir, configFiles, serviceName, imageName } = this.composeContext;
|
||||
const { workingDir, configFiles, serviceName, imageName, dataDirHost } = this.composeContext;
|
||||
const env = { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' };
|
||||
this.lastUpdateError = null;
|
||||
|
||||
try { fs.unlinkSync(UPDATE_ERROR_FILE); } catch { /* absent is the steady state */ }
|
||||
|
||||
// Async pull: a sync execFileSync blocks the event loop, which lets the frontend
|
||||
// overlay see a false "online" response between the pull finishing and the restart.
|
||||
console.log(`[SelfUpdate] Pulling latest image: ${imageName}...`);
|
||||
@@ -110,32 +149,44 @@ class SelfUpdateService {
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 2: Spawn a helper container to run docker compose recreate.
|
||||
// The main container cannot access the compose file because the host path
|
||||
// from Docker labels does not exist inside this container. The helper
|
||||
// explicitly mounts the compose working directory from the host, so the
|
||||
// compose file is accessible at the original path.
|
||||
// The main container cannot access the compose file at its host path,
|
||||
// so the helper bind-mounts the compose working directory from the host.
|
||||
// Run attached (no -d): if compose recreate fails before it kills us,
|
||||
// execFile's callback receives the helper's exit code + stderr directly.
|
||||
console.log(`[SelfUpdate] Spawning updater container... (last breath)`);
|
||||
const fFlags = configFiles.split(',').flatMap(f => ['-f', f.trim()]);
|
||||
const composeCmd = ['sleep 3 && docker compose', ...fFlags, 'up -d --force-recreate', serviceName].join(' ');
|
||||
|
||||
// On failure, persist exit code + stderr to UPDATE_ERROR_FILE (host-mounted)
|
||||
// so the NEW gateway can read it after restart if we die mid-execution.
|
||||
const stderrTmp = '/tmp/_sencho_err';
|
||||
const composeCmd = [
|
||||
'sleep 3',
|
||||
['docker compose', ...fFlags, 'up -d --force-recreate', serviceName, `2>${stderrTmp}`].join(' '),
|
||||
'ec=$?',
|
||||
`if [ $ec -ne 0 ]; then { echo "exit=$ec"; cat ${stderrTmp}; } > ${UPDATE_ERROR_FILE} 2>/dev/null; fi`,
|
||||
`cat ${stderrTmp} >&2 2>/dev/null`,
|
||||
'exit $ec',
|
||||
].join('; ');
|
||||
|
||||
const args = [
|
||||
'run', '--rm', '-d',
|
||||
'run', '--rm',
|
||||
'--user', 'root',
|
||||
'--entrypoint', 'sh',
|
||||
'-v', '/var/run/docker.sock:/var/run/docker.sock',
|
||||
'-v', `${workingDir}:${workingDir}:ro`,
|
||||
...(dataDirHost ? ['-v', `${dataDirHost}:/app/data:rw`] : []),
|
||||
'-w', workingDir,
|
||||
imageName,
|
||||
'-c', composeCmd,
|
||||
];
|
||||
|
||||
// Capture spawn errors (bad image, missing socket, permission denied) so they
|
||||
// land in lastUpdateError instead of vanishing silently.
|
||||
// Callback may never fire on success (we die mid-call during recreate);
|
||||
// that is fine because the restart itself is the success signal.
|
||||
execFile('docker', args, { env }, (err, _stdout, stderr) => {
|
||||
if (err) {
|
||||
const stderrText = stderr?.toString().trim();
|
||||
this.lastUpdateError = stderrText || err.message || 'Helper container failed to spawn';
|
||||
console.error('[SelfUpdate] Helper container spawn failed:', this.lastUpdateError);
|
||||
this.lastUpdateError = stderrText || err.message || 'Helper container failed';
|
||||
console.error('[SelfUpdate] Helper container failed:', this.lastUpdateError);
|
||||
}
|
||||
});
|
||||
// No code after this point is guaranteed to run: the helper recreates this container.
|
||||
|
||||
Reference in New Issue
Block a user