From 4003e7c04730c8ff3d2f8ae9f1885f020aafc8db Mon Sep 17 00:00:00 2001 From: Anso Date: Fri, 10 Apr 2026 17:39:01 -0400 Subject: [PATCH] fix(fleet): capture local self-update helper errors (#495) The helper container that runs `docker compose up --force-recreate` was spawned with `docker run -d`, so the command returned immediately with just the container ID. Any failure happening INSIDE the helper (bad compose file, image mismatch, permission issue, socket problem) was invisible: execFile's callback only fired for `docker run` command errors, never for errors inside the detached helper. The UI fell back to the generic 3-minute "Local update did not complete" heuristic with no actionable information. The helper now runs attached, so execFile's callback receives the helper's exit code and stderr directly for any failure that happens before the recreate kills this process. Additionally, the helper persists exit code + stderr to `/app/data/.sencho-update-error` before exiting, so the error survives the gateway's own death. On startup, `SelfUpdateService.recoverPreviousError()` reads and deletes that file, routing the real error through the existing `getLastError()` path so the freshly booted gateway reports exactly why the previous attempt failed instead of the generic timeout. --- CHANGELOG.md | 1 + backend/src/services/SelfUpdateService.ts | 77 +++++++++++++++++++---- 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af5eb252..e9f72e76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +* **fleet:** surface real errors from failed local self-updates. Previously the helper container that runs `docker compose up --force-recreate` was launched with `docker run -d`, so `docker run` returned the container ID immediately and any failure happening inside the helper (bad compose file, image mismatch, permission issue) was invisible. The UI only saw the generic 3-minute "Local update did not complete" heuristic fallback. The helper now runs attached so `execFile` captures its exit code and stderr directly, AND it persists failure details to `/app/data/.sencho-update-error` before exiting so the error survives the gateway's death. On startup, `SelfUpdateService.recoverPreviousError()` reads that file, surfaces the real error through `getLastError()`, and deletes it, so the freshly restarted gateway reports exactly why the previous attempt failed instead of the generic timeout message. * **deps:** migrate SSO OIDC integration from `openid-client` v5 to v6. The v5 `Issuer` / `Client` / `generators` API was removed upstream; the service now uses `Configuration`, `discovery`, `buildAuthorizationUrl`, `authorizationCodeGrant`, and `fetchUserInfo`. Discovery metadata is cached for 5 minutes via `CacheService` so a single login flow does not pay the HTTPS round trip twice, and the cache is invalidated on every SSO config write. `fetchUserInfo` failures now log the underlying error instead of silently falling back to id_token claims. ## [0.43.0](https://github.com/AnsoCode/Sencho/compare/v0.42.7...v0.43.0) (2026-04-10) diff --git a/backend/src/services/SelfUpdateService.ts b/backend/src/services/SelfUpdateService.ts index 4694a2fd..c517a0d5 100644 --- a/backend/src/services/SelfUpdateService.ts +++ b/backend/src/services/SelfUpdateService.ts @@ -1,15 +1,22 @@ import { execFileSync, execFile } from 'child_process'; import { promisify } from 'util'; +import * as fs from 'fs'; import DockerController from './DockerController'; import { disableCapability } from './CapabilityRegistry'; const execFileAsync = promisify(execFile); +// Error file written by the helper container on a failed compose recreate. +// Must live under /app/data so both the helper (via host-path bind mount) and +// the NEW gateway process (which always mounts /app/data) can reach it. +const UPDATE_ERROR_FILE = '/app/data/.sencho-update-error'; + interface ComposeContext { workingDir: string; configFiles: string; serviceName: string; imageName: string; + dataDirHost: string | null; } class SelfUpdateService { @@ -66,9 +73,22 @@ class SelfUpdateService { return; } - this.composeContext = { workingDir, configFiles, serviceName, imageName }; + // Find the host path backing /app/data. The helper container will bind-mount + // it at the same path so compose recreate errors can be persisted to a file + // that survives the gateway's restart and is readable by the NEW process. + const mounts = (info.Mounts ?? []) as Array<{ Source?: string; Destination?: string }>; + const dataDirHost = mounts.find(m => m.Destination === '/app/data')?.Source ?? null; + if (!dataDirHost) { + console.log('[SelfUpdate] /app/data mount not found - update error recovery will be unavailable'); + } + + this.composeContext = { workingDir, configFiles, serviceName, imageName, dataDirHost }; this.canSelfUpdate = true; console.log(`[SelfUpdate] Ready - service="${serviceName}" image="${imageName}" in ${workingDir}`); + + // Surface any error from a previous failed update attempt (persisted by + // the helper container) so the new process can report it to the user. + this.recoverPreviousError(); } catch (error) { console.log('[SelfUpdate] Could not inspect own container - self-update unavailable:', (error as Error).message); disableCapability('self-update'); @@ -89,12 +109,31 @@ class SelfUpdateService { this.lastUpdateError = null; } + /** Surfaces any error the helper container persisted before the previous + * gateway process died, then deletes the file. */ + private recoverPreviousError(): void { + try { + const content = fs.readFileSync(UPDATE_ERROR_FILE, 'utf8').trim(); + if (content) { + this.lastUpdateError = content; + console.error('[SelfUpdate] Recovered error from previous update attempt:', content); + } + fs.unlinkSync(UPDATE_ERROR_FILE); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + console.error('[SelfUpdate] Failed to recover previous update error:', (error as Error).message); + } + } + } + async triggerUpdate(): Promise { if (!this.composeContext) return; - const { workingDir, configFiles, serviceName, imageName } = this.composeContext; + const { workingDir, configFiles, serviceName, imageName, dataDirHost } = this.composeContext; const env = { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' }; this.lastUpdateError = null; + try { fs.unlinkSync(UPDATE_ERROR_FILE); } catch { /* absent is the steady state */ } + // Async pull: a sync execFileSync blocks the event loop, which lets the frontend // overlay see a false "online" response between the pull finishing and the restart. console.log(`[SelfUpdate] Pulling latest image: ${imageName}...`); @@ -110,32 +149,44 @@ class SelfUpdateService { return; } - // Step 2: Spawn a helper container to run docker compose recreate. - // The main container cannot access the compose file because the host path - // from Docker labels does not exist inside this container. The helper - // explicitly mounts the compose working directory from the host, so the - // compose file is accessible at the original path. + // The main container cannot access the compose file at its host path, + // so the helper bind-mounts the compose working directory from the host. + // Run attached (no -d): if compose recreate fails before it kills us, + // execFile's callback receives the helper's exit code + stderr directly. console.log(`[SelfUpdate] Spawning updater container... (last breath)`); const fFlags = configFiles.split(',').flatMap(f => ['-f', f.trim()]); - const composeCmd = ['sleep 3 && docker compose', ...fFlags, 'up -d --force-recreate', serviceName].join(' '); + + // On failure, persist exit code + stderr to UPDATE_ERROR_FILE (host-mounted) + // so the NEW gateway can read it after restart if we die mid-execution. + const stderrTmp = '/tmp/_sencho_err'; + const composeCmd = [ + 'sleep 3', + ['docker compose', ...fFlags, 'up -d --force-recreate', serviceName, `2>${stderrTmp}`].join(' '), + 'ec=$?', + `if [ $ec -ne 0 ]; then { echo "exit=$ec"; cat ${stderrTmp}; } > ${UPDATE_ERROR_FILE} 2>/dev/null; fi`, + `cat ${stderrTmp} >&2 2>/dev/null`, + 'exit $ec', + ].join('; '); + const args = [ - 'run', '--rm', '-d', + 'run', '--rm', '--user', 'root', '--entrypoint', 'sh', '-v', '/var/run/docker.sock:/var/run/docker.sock', '-v', `${workingDir}:${workingDir}:ro`, + ...(dataDirHost ? ['-v', `${dataDirHost}:/app/data:rw`] : []), '-w', workingDir, imageName, '-c', composeCmd, ]; - // Capture spawn errors (bad image, missing socket, permission denied) so they - // land in lastUpdateError instead of vanishing silently. + // Callback may never fire on success (we die mid-call during recreate); + // that is fine because the restart itself is the success signal. execFile('docker', args, { env }, (err, _stdout, stderr) => { if (err) { const stderrText = stderr?.toString().trim(); - this.lastUpdateError = stderrText || err.message || 'Helper container failed to spawn'; - console.error('[SelfUpdate] Helper container spawn failed:', this.lastUpdateError); + this.lastUpdateError = stderrText || err.message || 'Helper container failed'; + console.error('[SelfUpdate] Helper container failed:', this.lastUpdateError); } }); // No code after this point is guaranteed to run: the helper recreates this container.