fix(fleet): make local self-update flow reliable end-to-end (#472)

The "Updating Sencho..." overlay used to dismiss prematurely while the
image pull was still running, after which the local node card would get
stuck in "updating" and eventually surface a generic "Timed Out" error
while the container remained on the old version.

Three root causes are addressed:

1. The image pull was synchronous (`execFileSync`), which blocked the
   Node event loop. The overlay's health probe saw the server come back
   the moment the pull finished and reloaded the page, even though the
   container had not restarted yet. The pull is now async via
   `promisify(execFile)`, so /api/health and /api/fleet/update-status
   keep serving throughout.

2. The overlay reloaded on the first 200 from /api/health regardless of
   whether the underlying process had actually restarted. /api/health
   now exposes the gateway boot timestamp, and the overlay captures it
   pre-update and only reloads when it observes a different value. A
   wasOffline-then-online fallback handles the case where the pre-update
   fetch failed.

3. Helper container spawn errors from `docker run` were silently
   discarded, so a failed compose recreate never surfaced anywhere.
   Errors are now captured into `lastUpdateError` via the execFile
   callback and surfaced through the existing /api/fleet/update-status
   error path.

A 3-minute early-fail heuristic on the local node block surfaces a clear
failure message when the helper fails silently, instead of waiting the
full 5-minute timeout for an unknown failure.

CHANGELOG.md

@@ -44,6 +44,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+* **fleet:** fix local (gateway) self-update flow end-to-end. The "Updating Sencho..."
+  overlay no longer dismisses prematurely while the image pull is still running; it now
+  compares the gateway's boot timestamp via `/api/health` and only reloads after a real
+  container restart is observed. The image pull was made fully async so the API stays
+  responsive throughout (previously a sync `execFileSync` blocked the event loop, which
+  caused the overlay to reload the moment the pull finished, while still on the old
+  process). Helper container spawn errors are now captured into `lastUpdateError` instead
+  of being silently swallowed, so a failed `docker run` for the compose recreate surfaces
+  immediately on the Fleet Overview. A 3-minute early-fail heuristic surfaces a clear
+  failure message when the helper container fails silently, instead of users waiting the
+  full 5-minute timeout for an unknown failure.
+
 * **api:** add tiered rate limiting to prevent dashboard polling lockouts. High-frequency
   polling endpoints (stats, system stats, stack statuses) are now exempt from the global
   rate limiter and governed by a separate 300 req/min safety net. The global limit is

backend/src/index.ts

@@ -491,15 +491,17 @@ const authRateLimiter = rateLimit({
   message: { error: 'Too many attempts. Please try again in 15 minutes.' },
 });
 
+// Captured at boot. Exposed via /api/health and /api/meta so the Fleet update overlay
+// can distinguish a brand-new process from the old one still mid-pull.
+const processStartedAt = Date.now();
+
 // Public health endpoint - no auth required (used by Docker HEALTHCHECK and uptime monitors)
 app.get('/api/health', (_req: Request, res: Response): void => {
-  res.json({ status: 'ok', uptime: process.uptime() });
+  res.json({ status: 'ok', uptime: process.uptime(), startedAt: processStartedAt });
 });
 
 // Public meta endpoint - returns this instance's version and supported capabilities.
 // No auth required (like /health). Used by remote nodes during connection tests.
-const processStartedAt = Date.now();
-
 app.get('/api/meta', (_req: Request, res: Response): void => {
   const updateError = SelfUpdateService.getInstance().getLastError();
   res.json({
@@ -1243,7 +1245,13 @@ app.get('/api/license/billing-portal', async (_req: Request, res: Response): Pro
 function scheduleLocalUpdate(res: Response, message: string): void {
   res.status(202).json({ message });
   res.on('finish', () => {
-    setTimeout(() => SelfUpdateService.getInstance().triggerUpdate(), 500);
+    setTimeout(() => {
+      // Defense in depth: triggerUpdate records its own errors into lastUpdateError,
+      // but guard against an unexpected throw becoming an unhandled rejection.
+      SelfUpdateService.getInstance().triggerUpdate().catch((err) => {
+        console.error('[SelfUpdate] Unexpected error during triggerUpdate:', err);
+      });
+    }, 500);
   });
 }
 
@@ -1575,11 +1583,22 @@ app.get('/api/fleet/update-status', async (_req: Request, res: Response): Promis
               });
             }
           } else if (node.type === 'local') {
-            // Local node: check if SelfUpdateService reported a pull failure
-            const localError = SelfUpdateService.getInstance().getLastError();
+            // Local node has only two failure signals: an explicit pull/spawn error,
+            // or the early-fail heuristic. Success is observed by the frontend overlay
+            // (it reloads the page when /api/health reports a new startedAt), at which
+            // point the new process starts with an empty tracker map.
+            const selfUpdate = SelfUpdateService.getInstance();
+            const localError = selfUpdate.getLastError();
             if (localError) {
               updateTracker.set(node.id, { ...tracker, status: 'failed', error: localError });
-              SelfUpdateService.getInstance().clearLastError();
+              selfUpdate.clearLastError();
+            } else if (elapsed > EARLY_FAIL_MS) {
+              // Helper container likely failed silently. Surface failure before the 5 min timeout.
+              updateTracker.set(node.id, {
+                ...tracker,
+                status: 'failed',
+                error: 'Local update did not complete. The container may not have restarted; check Docker logs on the host.',
+              });
             }
           }
         }

backend/src/services/SelfUpdateService.ts

@@ -1,7 +1,10 @@
 import { execFileSync, execFile } from 'child_process';
+import { promisify } from 'util';
 import DockerController from './DockerController';
 import { disableCapability } from './CapabilityRegistry';
 
+const execFileAsync = promisify(execFile);
+
 interface ComposeContext {
   workingDir: string;
   configFiles: string;
@@ -86,22 +89,22 @@ class SelfUpdateService {
     this.lastUpdateError = null;
   }
 
-  triggerUpdate(): void {
+  async triggerUpdate(): Promise<void> {
     if (!this.composeContext) return;
     const { workingDir, configFiles, serviceName, imageName } = this.composeContext;
     const env = { ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' };
     this.lastUpdateError = null;
 
-    // Step 1: Pull latest image directly (no compose file needed)
+    // Async pull: a sync execFileSync blocks the event loop, which lets the frontend
+    // overlay see a false "online" response between the pull finishing and the restart.
     console.log(`[SelfUpdate] Pulling latest image: ${imageName}...`);
     try {
-      execFileSync('docker', ['pull', imageName], {
+      await execFileAsync('docker', ['pull', imageName], {
         env,
-        stdio: 'pipe',
         timeout: 300_000, // 5 min max for pull
       });
     } catch (error) {
-      const stderr = (error as { stderr?: Buffer })?.stderr?.toString().trim();
+      const stderr = (error as { stderr?: Buffer | string })?.stderr?.toString().trim();
       this.lastUpdateError = stderr || (error as Error).message;
       console.error('[SelfUpdate] Pull failed:', this.lastUpdateError);
       return;
@@ -126,8 +129,16 @@ class SelfUpdateService {
       '-c', composeCmd,
     ];
 
-    execFile('docker', args, { env });
-    // Process will be killed by Docker during recreate; no code runs after this
+    // Capture spawn errors (bad image, missing socket, permission denied) so they
+    // land in lastUpdateError instead of vanishing silently.
+    execFile('docker', args, { env }, (err, _stdout, stderr) => {
+      if (err) {
+        const stderrText = stderr?.toString().trim();
+        this.lastUpdateError = stderrText || err.message || 'Helper container failed to spawn';
+        console.error('[SelfUpdate] Helper container spawn failed:', this.lastUpdateError);
+      }
+    });
+    // No code after this point is guaranteed to run: the helper recreates this container.
   }
 }
 

docs/features/remote-updates.mdx

@@ -74,10 +74,10 @@ When you update the local (gateway) node:
 
 1. A confirmation dialog appears explaining that the dashboard will briefly disconnect.
 2. After confirming, the server pulls the latest image and restarts.
-3. A reconnecting overlay appears and polls the server every few seconds.
-4. The page automatically reloads when the server comes back.
+3. A reconnecting overlay appears and polls the server every few seconds. It stays visible for the entire pull and restart cycle, even if the API briefly responds in between, so the page does not reload until the new container has actually booted.
+4. Once the new container reports a fresh boot timestamp, the page automatically reloads with the new version.
 
-If the server does not return within 5 minutes, a timeout message appears with a manual reload option.
+If the new container does not come up within 5 minutes, a timeout message appears with a manual reload option. If the gateway can detect that the update did not complete (for example, the image pull failed or the restart helper container could not spawn), the local node card surfaces a "Failed" badge with the underlying error within about 3 minutes instead of waiting for the full timeout.
 
 ## What happens during an update
 

frontend/src/components/FleetView.tsx

@@ -349,7 +349,12 @@ function UpdateStatusBadge({ status, error, onRetry, onDismiss }: {
     return null;
 }
 
-function ReconnectingOverlay() {
+interface ReconnectingOverlayProps {
+    /** Gateway boot timestamp captured pre-update. Null falls back to offline-then-online detection. */
+    preUpdateStartedAt: number | null;
+}
+
+function ReconnectingOverlay({ preUpdateStartedAt }: ReconnectingOverlayProps) {
     const [elapsed, setElapsed] = useState(0);
     const timedOut = elapsed >= 300; // 5 minutes
 
@@ -360,14 +365,35 @@ function ReconnectingOverlay() {
 
     useEffect(() => {
         if (timedOut) return;
+        let sawOffline = false;
         const poll = setInterval(async () => {
             try {
                 const res = await fetch('/api/health');
-                if (res.ok) window.location.reload();
-            } catch { /* still down */ }
+                if (!res.ok) {
+                    sawOffline = true;
+                    return;
+                }
+                const data = await res.json().catch(() => null) as { startedAt?: number } | null;
+                const currentStartedAt = typeof data?.startedAt === 'number' ? data.startedAt : null;
+
+                if (preUpdateStartedAt !== null && currentStartedAt !== null) {
+                    if (currentStartedAt !== preUpdateStartedAt) {
+                        window.location.reload();
+                    }
+                    return;
+                }
+
+                // Fallback when we don't know the original startedAt: require an offline
+                // response first so we don't reload while the old process is still mid-pull.
+                if (sawOffline) {
+                    window.location.reload();
+                }
+            } catch {
+                sawOffline = true;
+            }
         }, 3000);
         return () => clearInterval(poll);
-    }, [timedOut]);
+    }, [timedOut, preUpdateStartedAt]);
 
     return (
         <div className="fixed inset-0 z-50 flex items-center justify-center bg-background/80 backdrop-blur-[10px] backdrop-saturate-[1.15]">
@@ -637,6 +663,7 @@ export function FleetView({ onNavigateToNode }: FleetViewProps) {
     const [updateStatuses, setUpdateStatuses] = useState<NodeUpdateStatus[]>([]);
     const [updatingNodeId, setUpdatingNodeId] = useState<number | null>(null);
     const [reconnecting, setReconnecting] = useState(false);
+    const [preUpdateStartedAt, setPreUpdateStartedAt] = useState<number | null>(null);
     const [localUpdateConfirm, setLocalUpdateConfirm] = useState<number | null>(null);
     const [showUpdateModal, setShowUpdateModal] = useState(false);
     const [checkingUpdates, setCheckingUpdates] = useState(false);
@@ -727,8 +754,20 @@ export function FleetView({ onNavigateToNode }: FleetViewProps) {
 
         setUpdatingNodeId(nodeId);
         try {
+            // Capture pre-update boot timestamp so the overlay can detect a real restart
+            // vs a false "online" response from the still-running old process mid-pull.
+            let bootBefore: number | null = null;
+            try {
+                const healthRes = await fetch('/api/health');
+                if (healthRes.ok) {
+                    const data = await healthRes.json();
+                    if (typeof data?.startedAt === 'number') bootBefore = data.startedAt;
+                }
+            } catch { /* fall back to offline-then-online detection */ }
+
             const res = await apiFetch(`/fleet/nodes/${nodeId}/update`, { method: 'POST', localOnly: true });
             if (res.ok) {
+                setPreUpdateStartedAt(bootBefore);
                 setReconnecting(true);
             } else {
                 const err = await res.json().catch(() => ({}));
@@ -1192,7 +1231,7 @@ export function FleetView({ onNavigateToNode }: FleetViewProps) {
             </Tabs>
 
             {/* Reconnecting overlay — shown when local node is updating */}
-            {reconnecting && <ReconnectingOverlay />}
+            {reconnecting && <ReconnectingOverlay preUpdateStartedAt={preUpdateStartedAt} />}
 
             {/* Node Updates modal */}
             <Dialog open={showUpdateModal} onOpenChange={(open) => { setShowUpdateModal(open); if (!open) setModalSearch(''); }}>