feat: detect stalled stack updates and add in-app recovery actions (#1347)

* feat: detect stalled stack updates and add in-app recovery actions

Add a backend idle-output backstop that stops a deploy/update compose step
that has gone silent (SENCHO_COMPOSE_STALL_TIMEOUT_MS, default 10m), so a
hung image pull surfaces a fast failure instead of spinning indefinitely.

Surface failed, timed-out, and stalled operations with recovery actions on
the stack page: a desktop chip plus popover menu and an inline mobile card
offering retry, restart, roll back (when a backup exists), refresh state,
and copy diagnostics, all gated by deploy permission. The streaming
deploy/update progress modal is now on by default and warns when output
goes quiet. Container state is refreshed after a failed or stalled
operation, and the UI never sits in an indefinite spinner.

* fix: harden rollback against policy-blocked file mutation and refine recovery

Address review findings on the stalled-update recovery work:

- The rollback route restored backup files before running the policy gate, so
  a policy-blocked rollback could leave the on-disk config rolled back while the
  deployed containers were unchanged. Snapshot the current files first and
  revert them when the gate blocks; if that revert itself fails, escalate it on
  the persistent alert feed since the 409 is already sent.
- Refresh container state after a successful manual rollback (rollback
  redeploys), without mis-recording a refetch failure as a rollback failure.
- Suppress the stalled-output warning once live progress is unavailable.

* test: mock snapshotStackFiles in the atomic-deploy rollback route tests

The rollback route now snapshots stack files before restoring a backup, so its
FileSystemService mock needs snapshotStackFiles. Without it the mocked call
threw and the route returned 500, failing the success-path rollback assertions.

backend/src/services/ComposeService.ts

@@ -50,6 +50,22 @@ function getComposeCommandTimeoutMs(): number {
   return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS;
 }
 
+// Idle backstop for long-running pull/recreate steps: if the child emits no
+// output for this window while still running, the step is treated as stalled
+// and terminated, so a hung `docker compose pull` surfaces a fast failure
+// instead of spinning until the much longer command timeout above. Conservative
+// by default because a working pull can be briefly silent while a large layer
+// extracts; operators on slow links or heavy local builds can raise it.
+const DEFAULT_COMPOSE_STALL_TIMEOUT_MS = 10 * 60 * 1000;
+
+function getComposeStallTimeoutMs(): number {
+  const configured = Number(process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS);
+  if (Number.isFinite(configured) && configured > 0) {
+    return configured;
+  }
+  return DEFAULT_COMPOSE_STALL_TIMEOUT_MS;
+}
+
 /**
  * ComposeService - local docker compose CLI execution.
  *
@@ -98,7 +114,11 @@ export class ComposeService {
     cwd: string,
     ws?: WebSocket,
     throwOnError = true,
-    env?: Record<string, string | undefined>
+    env?: Record<string, string | undefined>,
+    // When set, terminate the child if it emits no output for this long while
+    // still running (idle stall backstop). Appended last so the existing
+    // registry-auth call sites that pass `env` are unaffected.
+    idleTimeoutMs?: number
   ): Promise<void> {
     return new Promise((resolve, reject) => {
       const child = spawn(command, args, {
@@ -116,6 +136,7 @@ export class ComposeService {
       const timeoutMs = getComposeCommandTimeoutMs();
       let timeout: ReturnType<typeof setTimeout> | null = null;
       let forceKillTimeout: ReturnType<typeof setTimeout> | null = null;
+      let idleTimeout: ReturnType<typeof setTimeout> | null = null;
 
       const sendOutput = (text: string) => {
         if (ws && ws.readyState === WebSocket.OPEN) {
@@ -132,6 +153,10 @@ export class ComposeService {
           clearTimeout(forceKillTimeout);
           forceKillTimeout = null;
         }
+        if (idleTimeout) {
+          clearTimeout(idleTimeout);
+          idleTimeout = null;
+        }
       };
 
       const finish = (complete: () => void) => {
@@ -159,21 +184,39 @@ export class ComposeService {
         }, 5000);
       };
 
+      // Idle stall backstop. Armed once below and reset on every output chunk;
+      // if it ever fires, the step has been silent for idleTimeoutMs while still
+      // running, so terminate it. Never rearmed after a termination is pending or
+      // the child has exited, so it cannot re-fire during the SIGTERM grace.
+      const armIdleTimeout = () => {
+        if (idleTimeoutMs === undefined) return;
+        if (exited || settled || pendingTerminationError) return;
+        if (idleTimeout) clearTimeout(idleTimeout);
+        idleTimeout = setTimeout(() => {
+          const seconds = Math.round(idleTimeoutMs / 1000);
+          sendOutput(`=== No output for ${seconds}s; the operation appears stalled and was stopped ===\n`);
+          terminateChild(new Error(`STACK_STALLED_OUTPUT: no output for ${seconds}s`));
+        }, idleTimeoutMs);
+      };
+
       // The progress socket is output-only: a deploy/update/down is owned by the
       // HTTP request that started it, so closing or losing the socket (the user
       // minimizes the panel, navigates away, or the connection blips) must not
       // terminate the compose process. Termination is driven solely by the
-      // command timeout below.
+      // command timeout here and the optional idle stall backstop above.
       timeout = setTimeout(() => {
         const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`;
         sendOutput(`${message}\n`);
         terminateChild(new Error(message));
       }, timeoutMs);
 
+      armIdleTimeout();
+
       const onData = (data: Buffer) => {
         const text = data.toString();
         errorLog += text;
         sendOutput(text);
+        armIdleTimeout();
       };
 
       child.stdout.on('data', onData);
@@ -328,7 +371,7 @@ export class ComposeService {
       }
 
       await this.withRegistryAuth(async (env) => {
-        await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env);
+        await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs());
       }, sendOutput);
 
       // Post-Deploy Health Probe
@@ -505,10 +548,10 @@ export class ComposeService {
 
       await this.withRegistryAuth(async (env) => {
         sendOutput('=== Pulling latest images ===\n');
-        await this.execute('docker', ['compose', 'pull'], stackDir, ws, true, env);
+        await this.execute('docker', ['compose', 'pull'], stackDir, ws, true, env, getComposeStallTimeoutMs());
 
         sendOutput('=== Recreating containers ===\n');
-        await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env);
+        await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs());
       }, sendOutput);
 
       // Post-Update Health Probe

backend/src/services/FileSystemService.ts

@@ -874,6 +874,52 @@ export class FileSystemService {
     if (debug) console.debug(`[FileSystemService:debug] Restore completed in ${Date.now() - t0}ms`, { stackName, restored: items.filter(i => i !== '.timestamp').length, removedOrphans });
   }
 
+  /**
+   * Capture the current managed stack files (PROTECTED_STACK_FILES) in memory and
+   * return a function that puts them back, faithfully (writing the captured
+   * contents and removing any managed file that did not exist when captured).
+   *
+   * Used by the rollback route to undo a restored backup when the policy gate
+   * blocks before the deploy commits: restoreStackFiles has already overwritten
+   * the on-disk files, so without this a blocked rollback would leave disk holding
+   * the rolled-back configuration while the deployed containers are unchanged.
+   */
+  async snapshotStackFiles(stackName: string): Promise<() => Promise<void>> {
+    const stackDir = this.resolveStackDir(stackName);
+    // Canonical js/path-injection barrier inline with the read/write sinks, the
+    // same pattern restoreStackFiles uses: resolve against the base and confirm
+    // containment so static analysis credits the barrier.
+    const baseResolved = path.resolve(this.baseDir);
+    const snapshot = new Map<string, Buffer>();
+    for (const file of PROTECTED_STACK_FILES) {
+      const target = path.resolve(baseResolved, path.join(stackDir, file));
+      if (!target.startsWith(baseResolved + path.sep)) {
+        throw Object.assign(new Error('Path escapes compose directory'), { code: 'INVALID_PATH' });
+      }
+      try {
+        snapshot.set(file, await fsPromises.readFile(target));
+      } catch (e: unknown) {
+        if ((e as NodeJS.ErrnoException)?.code !== 'ENOENT') throw e;
+      }
+    }
+    return async () => {
+      for (const file of PROTECTED_STACK_FILES) {
+        const target = path.resolve(baseResolved, path.join(stackDir, file));
+        if (!target.startsWith(baseResolved + path.sep)) continue;
+        const saved = snapshot.get(file);
+        if (saved !== undefined) {
+          await fsPromises.writeFile(target, saved);
+        } else {
+          try {
+            await fsPromises.unlink(target);
+          } catch (e: unknown) {
+            if ((e as NodeJS.ErrnoException)?.code !== 'ENOENT') throw e;
+          }
+        }
+      }
+    };
+  }
+
   async getBackupInfo(stackName: string): Promise<{ exists: boolean; timestamp: number | null }> {
     const backupDir = this.getBackupDir(stackName);
     try {