feat: detect stalled stack updates and add in-app recovery actions (#1347)

* feat: detect stalled stack updates and add in-app recovery actions

Add a backend idle-output backstop that stops a deploy/update compose step
that has gone silent (SENCHO_COMPOSE_STALL_TIMEOUT_MS, default 10m), so a
hung image pull surfaces a fast failure instead of spinning indefinitely.

Surface failed, timed-out, and stalled operations with recovery actions on
the stack page: a desktop chip plus popover menu and an inline mobile card
offering retry, restart, roll back (when a backup exists), refresh state,
and copy diagnostics, all gated by deploy permission. The streaming
deploy/update progress modal is now on by default and warns when output
goes quiet. Container state is refreshed after a failed or stalled
operation, and the UI never sits in an indefinite spinner.

* fix: harden rollback against policy-blocked file mutation and refine recovery

Address review findings on the stalled-update recovery work:

- The rollback route restored backup files before running the policy gate, so
  a policy-blocked rollback could leave the on-disk config rolled back while the
  deployed containers were unchanged. Snapshot the current files first and
  revert them when the gate blocks; if that revert itself fails, escalate it on
  the persistent alert feed since the 409 is already sent.
- Refresh container state after a successful manual rollback (rollback
  redeploys), without mis-recording a refetch failure as a rollback failure.
- Suppress the stalled-output warning once live progress is unavailable.

* test: mock snapshotStackFiles in the atomic-deploy rollback route tests

The rollback route now snapshots stack files before restoring a backup, so its
FileSystemService mock needs snapshotStackFiles. Without it the mocked call
threw and the route returned 500, failing the success-path rollback assertions.

backend/src/__tests__/atomic-deploy-hardening.test.ts

@@ -17,10 +17,12 @@ const {
   mockDeployStack,
   mockGetBackupInfo,
   mockRestoreStackFiles,
+  mockSnapshotStackFiles,
 } = vi.hoisted(() => ({
   mockDeployStack: vi.fn(),
   mockGetBackupInfo: vi.fn(),
   mockRestoreStackFiles: vi.fn(),
+  mockSnapshotStackFiles: vi.fn(),
 }));
 
 vi.mock('../services/ComposeService', async () => {
@@ -43,6 +45,7 @@ vi.mock('../services/FileSystemService', () => ({
       hasComposeFile: vi.fn().mockResolvedValue(true),
       getBackupInfo: mockGetBackupInfo,
       restoreStackFiles: mockRestoreStackFiles,
+      snapshotStackFiles: mockSnapshotStackFiles,
     }),
   },
 }));
@@ -83,6 +86,7 @@ beforeEach(async () => {
   mockDeployStack.mockReset();
   mockGetBackupInfo.mockReset().mockResolvedValue({ exists: true, timestamp: Date.now() });
   mockRestoreStackFiles.mockReset().mockResolvedValue(undefined);
+  mockSnapshotStackFiles.mockReset().mockResolvedValue(async () => {});
   const { StackOpLockService } = await import('../services/StackOpLockService');
   StackOpLockService.resetForTests();
 });

backend/src/__tests__/compose-service.test.ts

@@ -110,6 +110,7 @@ vi.mock('../services/LogFormatter', () => ({
 import { ComposeService, getComposeRollbackInfo } from '../services/ComposeService';
 
 const originalComposeTimeout = process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS;
+const originalStallTimeout = process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS;
 
 /** Creates an EventEmitter that mimics a child_process spawn result */
 function createMockProcess() {
@@ -169,6 +170,11 @@ afterEach(() => {
   } else {
     process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS = originalComposeTimeout;
   }
+  if (originalStallTimeout === undefined) {
+    delete process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS;
+  } else {
+    process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = originalStallTimeout;
+  }
 });
 
 // ── runCommand ─────────────────────────────────────────────────────────
@@ -689,3 +695,122 @@ describe('ComposeService - downStack', () => {
     await expect(svc.downStack('my-stack')).resolves.toBeUndefined();
   });
 });
+
+// ── stall (idle-output) backstop ───────────────────────────────────────
+
+describe('ComposeService - idle-output stall backstop', () => {
+  it('terminates a silent update step and rejects with STACK_STALLED_OUTPUT', async () => {
+    process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000';
+    const proc = createMockProcess();
+    mockSpawn.mockReturnValue(proc);
+
+    const svc = ComposeService.getInstance(1);
+    // The pull emits nothing; the idle backstop should fire after 1s.
+    const result = svc.updateStack('my-stack').then(() => null, (e: Error) => e);
+    await vi.advanceTimersByTimeAsync(1000);
+
+    expect(proc.kill).toHaveBeenCalledWith('SIGTERM');
+    proc.emit('close', null);
+    const error = await result;
+    expect(error).not.toBeNull();
+    expect(error!.message).toContain('STACK_STALLED_OUTPUT');
+  });
+
+  it('sends a stalled marker to the WebSocket before terminating', async () => {
+    process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000';
+    const proc = createMockProcess();
+    mockSpawn.mockReturnValue(proc);
+    const ws = createMockWs();
+
+    const svc = ComposeService.getInstance(1);
+    const result = svc.updateStack('my-stack', ws).then(() => null, (e: Error) => e);
+    await vi.advanceTimersByTimeAsync(1000);
+    proc.emit('close', null);
+    await result;
+
+    const sendCalls = ws.send.mock.calls.map(c => c[0] as string);
+    expect(sendCalls.some(msg => msg.includes('appears stalled and was stopped'))).toBe(true);
+  });
+
+  it('does not stall when output keeps arriving within the idle window', async () => {
+    process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000';
+    mockListContainers.mockResolvedValue([]);
+    const proc = createMockProcess();
+    mockSpawn.mockReturnValue(proc);
+
+    const svc = ComposeService.getInstance(1);
+    // deployStack spawns a single `up`; emit output every 600ms (< 1s window)
+    // so the idle timer resets and never fires.
+    const promise = svc.deployStack('my-stack');
+    await vi.advanceTimersByTimeAsync(600);
+    proc.stdout.emit('data', Buffer.from('pulling layer a...'));
+    await vi.advanceTimersByTimeAsync(600);
+    proc.stdout.emit('data', Buffer.from('pulling layer b...'));
+    await vi.advanceTimersByTimeAsync(600);
+    expect(proc.kill).not.toHaveBeenCalled();
+
+    proc.emit('close', 0);
+    await vi.advanceTimersByTimeAsync(3100); // health probe
+    await promise;
+    expect(proc.kill).not.toHaveBeenCalled();
+  });
+
+  it('does not arm the idle backstop for runCommand (down/restart/stop stay silent-safe)', async () => {
+    process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000';
+    const proc = createMockProcess();
+    mockSpawn.mockReturnValue(proc);
+
+    const svc = ComposeService.getInstance(1);
+    const promise = svc.runCommand('my-stack', 'restart');
+    // A silent restart longer than the stall window must not be killed: the
+    // idle backstop is only armed for deploy/update compose steps.
+    await vi.advanceTimersByTimeAsync(1500);
+    expect(proc.kill).not.toHaveBeenCalled();
+
+    proc.emit('close', 0);
+    await promise;
+  });
+
+  it('falls back to the default stall window when the env value is invalid', async () => {
+    process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '0'; // invalid → default (10min)
+    mockListContainers.mockResolvedValue([]);
+    const proc = createMockProcess();
+    mockSpawn.mockReturnValue(proc);
+
+    const svc = ComposeService.getInstance(1);
+    const promise = svc.deployStack('my-stack');
+    // Far below the 10-minute default: a '0' that leaked through would fire at 0ms.
+    await vi.advanceTimersByTimeAsync(2000);
+    expect(proc.kill).not.toHaveBeenCalled();
+
+    proc.emit('close', 0);
+    await vi.advanceTimersByTimeAsync(3100); // health probe
+    await promise;
+  });
+
+  it('preserves STACK_STALLED_OUTPUT through the atomic rollback wrapper', async () => {
+    process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000';
+    const pullProc = createMockProcess();
+    let call = 0;
+    // First spawn is the stalling pull; later spawns (the rollback restore's
+    // `up`) close cleanly, proving the restore is not idle-timeout armed.
+    mockSpawn.mockImplementation(() => {
+      call += 1;
+      if (call === 1) return pullProc;
+      const p = createMockProcess();
+      Promise.resolve().then(() => p.emit('close', 0));
+      return p;
+    });
+
+    const svc = ComposeService.getInstance(1);
+    const result = svc.updateStack('my-stack', undefined, true).then(() => null, (e: Error) => e);
+    await vi.advanceTimersByTimeAsync(1000); // idle backstop fires on the silent pull
+    pullProc.emit('close', null);
+    await vi.runAllTimersAsync();
+
+    const error = await result;
+    expect(error).not.toBeNull();
+    expect(error!.message).toContain('STACK_STALLED_OUTPUT');
+    expect(getComposeRollbackInfo(error)).toMatchObject({ attempted: true });
+  });
+});

backend/src/__tests__/filesystem-backup.test.ts

@@ -228,6 +228,58 @@ describe('FileSystemService backup location', () => {
     await expect(service.restoreStackFiles(stackName)).rejects.toThrow(/Rollback aborted/i);
   });
 
+  it('snapshotStackFiles reverts a restore so a policy-blocked rollback leaves current files in place', async () => {
+    const stackName = 'revert';
+    const stackDir = path.join(composeDir, stackName);
+    await fsPromises.mkdir(stackDir, { recursive: true });
+    // The rollback target (older config) is captured into the backup slot.
+    await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: old\n', 'utf-8');
+
+    const service = FileSystemService.getInstance();
+    await service.backupStackFiles(stackName);
+
+    // The current in-use config differs from the backup and adds a managed file
+    // (.env) the backup does not have.
+    await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: current\n', 'utf-8');
+    await fsPromises.writeFile(path.join(stackDir, '.env'), 'TOKEN=current\n', 'utf-8');
+
+    // Snapshot current, then restore the backup, mirroring the rollback route.
+    const revert = await service.snapshotStackFiles(stackName);
+    await service.restoreStackFiles(stackName);
+    expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: old\n');
+    await expect(fsPromises.access(path.join(stackDir, '.env'))).rejects.toMatchObject({ code: 'ENOENT' });
+
+    // The policy gate blocks the restored target: revert must put the current
+    // files back exactly (content restored, the removed .env recreated).
+    await revert();
+    expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: current\n');
+    expect(await fsPromises.readFile(path.join(stackDir, '.env'), 'utf-8')).toBe('TOKEN=current\n');
+  });
+
+  it('snapshotStackFiles revert removes a managed file the snapshot did not have', async () => {
+    const stackName = 'revert-orphan';
+    const stackDir = path.join(composeDir, stackName);
+    await fsPromises.mkdir(stackDir, { recursive: true });
+    // Backup target has compose + .env; current has only compose.
+    await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: target\n', 'utf-8');
+    await fsPromises.writeFile(path.join(stackDir, '.env'), 'TOKEN=target\n', 'utf-8');
+
+    const service = FileSystemService.getInstance();
+    await service.backupStackFiles(stackName);
+
+    await fsPromises.rm(path.join(stackDir, '.env'));
+    await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'name: current\n', 'utf-8');
+
+    const revert = await service.snapshotStackFiles(stackName);
+    await service.restoreStackFiles(stackName); // brings back the .env from the backup
+    await expect(fsPromises.access(path.join(stackDir, '.env'))).resolves.toBeUndefined();
+
+    await revert();
+    // The current state had no .env, so revert must remove the restored one.
+    await expect(fsPromises.access(path.join(stackDir, '.env'))).rejects.toMatchObject({ code: 'ENOENT' });
+    expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe('name: current\n');
+  });
+
   it('restoreStackFiles leaves non-managed files untouched', async () => {
     const stackName = 'userdata';
     const stackDir = path.join(composeDir, stackName);

backend/src/__tests__/stacks-failure-notifications.test.ts

@@ -12,6 +12,7 @@ import request from 'supertest';
 import jwt from 'jsonwebtoken';
 import { setupTestDb, cleanupTestDb, loginAsTestAdmin, TEST_JWT_SECRET } from './helpers/setupTestDb';
 import { ComposeRollbackError } from '../services/ComposeService';
+import * as policyGate from '../helpers/policyGate';
 
 // ── Hoisted mocks (must come before importing the app) ──────────────────────
 
@@ -26,6 +27,9 @@ const {
   mockIsTrivyAvailable,
   mockGetImageDigest,
   mockRunScanAndPersist,
+  mockGetBackupInfo,
+  mockRestoreStackFiles,
+  mockSnapshotStackFiles,
 } = vi.hoisted(() => ({
   mockDeployStack: vi.fn(),
   mockRunCommand: vi.fn(),
@@ -37,6 +41,9 @@ const {
   mockIsTrivyAvailable: vi.fn(),
   mockGetImageDigest: vi.fn(),
   mockRunScanAndPersist: vi.fn(),
+  mockGetBackupInfo: vi.fn(),
+  mockRestoreStackFiles: vi.fn(),
+  mockSnapshotStackFiles: vi.fn(),
 }));
 
 vi.mock('../services/ComposeService', async () => {
@@ -100,6 +107,9 @@ vi.mock('../services/FileSystemService', () => ({
       getBaseDir: () => '/tmp/compose',
       readComposeFile: vi.fn().mockResolvedValue(''),
       hasComposeFile: vi.fn().mockResolvedValue(true),
+      getBackupInfo: mockGetBackupInfo,
+      restoreStackFiles: mockRestoreStackFiles,
+      snapshotStackFiles: mockSnapshotStackFiles,
     }),
   },
 }));
@@ -145,6 +155,12 @@ beforeEach(() => {
     critical_count: 0,
     high_count: 0,
   });
+  mockGetBackupInfo.mockReset();
+  mockRestoreStackFiles.mockReset();
+  mockSnapshotStackFiles.mockReset();
+  mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: 1 });
+  mockRestoreStackFiles.mockResolvedValue(undefined);
+  mockSnapshotStackFiles.mockResolvedValue(async () => {});
   dispatchAlertSpy.mockClear();
 });
 
@@ -354,3 +370,63 @@ describe('deploy_failure notification on /update error', () => {
     expect(mockUpdateStack.mock.calls[0][2]).toBe(true);
   });
 });
+
+describe('rollback file-revert safety on a policy-blocked rollback', () => {
+  it('does not deploy and alerts the operator when the post-block file revert fails', async () => {
+    // The restored backup is blocked by policy after files were already restored.
+    const gateSpy = vi
+      .spyOn(policyGate, 'runPolicyGate')
+      .mockImplementation(async (_req, res) => {
+        res.status(409).json({ error: 'Rollback blocked by policy' });
+        return false;
+      });
+    // The revert that should undo the restore itself fails (e.g. EACCES on a
+    // chowned bind mount), leaving disk inconsistent with the deployed stack.
+    mockSnapshotStackFiles.mockResolvedValue(async () => {
+      throw Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' });
+    });
+    try {
+      const res = await request(app)
+        .post('/api/stacks/myapp/rollback')
+        .set('Cookie', authCookie);
+
+      expect(res.status).toBe(409);
+      // The rollback must not have deployed the blocked target.
+      expect(mockDeployStack).not.toHaveBeenCalled();
+      await new Promise(resolve => setImmediate(resolve));
+      // The revert failure is escalated on the persistent alert feed.
+      expect(dispatchAlertSpy).toHaveBeenCalledWith(
+        'error',
+        'deploy_failure',
+        expect.stringContaining('EACCES'),
+        { stackName: 'myapp', actor: 'testadmin' },
+      );
+    } finally {
+      gateSpy.mockRestore();
+    }
+  });
+
+  it('reverts cleanly and stays quiet when the policy block revert succeeds', async () => {
+    const revert = vi.fn().mockResolvedValue(undefined);
+    const gateSpy = vi
+      .spyOn(policyGate, 'runPolicyGate')
+      .mockImplementation(async (_req, res) => {
+        res.status(409).json({ error: 'Rollback blocked by policy' });
+        return false;
+      });
+    mockSnapshotStackFiles.mockResolvedValue(revert);
+    try {
+      const res = await request(app)
+        .post('/api/stacks/myapp/rollback')
+        .set('Cookie', authCookie);
+
+      expect(res.status).toBe(409);
+      expect(revert).toHaveBeenCalledTimes(1);
+      expect(mockDeployStack).not.toHaveBeenCalled();
+      await new Promise(resolve => setImmediate(resolve));
+      expect(dispatchAlertSpy).not.toHaveBeenCalled();
+    } finally {
+      gateSpy.mockRestore();
+    }
+  });
+});

backend/src/routes/stacks.ts

@@ -1421,8 +1421,24 @@ stacksRouter.post('/:stackName/rollback', async (req: Request, res: Response) =>
       return;
     }
     dlog(`[Stacks] Rollback initiated: ${sanitizeForLog(stackName)}`);
+    // Snapshot the current files before restoring so a policy gate that blocks
+    // the restored target can be undone: restoreStackFiles commits to disk, and
+    // without this a blocked rollback would leave disk rolled back while the
+    // deployed state is unchanged.
+    const revertRestore = await fsSvc.snapshotStackFiles(stackName);
     await fsSvc.restoreStackFiles(stackName);
-    if (!(await runPolicyGate(req, res, stackName, req.nodeId))) return;
+    if (!(await runPolicyGate(req, res, stackName, req.nodeId))) {
+      try {
+        await revertRestore();
+      } catch (revertError) {
+        console.error('[Stacks] Failed to revert files after a policy-blocked rollback: %s', sanitizeForLog(stackName), revertError);
+        // The 409 is already sent and the on-disk config now diverges from the
+        // running stack; surface it on the persistent alert feed so the operator
+        // can repair it rather than discovering it on the next deploy.
+        notifyActionFailure('rollback', stackName, revertError, req.user?.username ?? 'system');
+      }
+      return;
+    }
     await ComposeService.getInstance(req.nodeId).deployStack(stackName, getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), false);
     invalidateNodeCaches(req.nodeId);
     dlog(`[Stacks] Rollback completed: ${sanitizeForLog(stackName)}`);

backend/src/services/ComposeService.ts

@@ -50,6 +50,22 @@ function getComposeCommandTimeoutMs(): number {
   return DEFAULT_COMPOSE_COMMAND_TIMEOUT_MS;
 }
 
+// Idle backstop for long-running pull/recreate steps: if the child emits no
+// output for this window while still running, the step is treated as stalled
+// and terminated, so a hung `docker compose pull` surfaces a fast failure
+// instead of spinning until the much longer command timeout above. Conservative
+// by default because a working pull can be briefly silent while a large layer
+// extracts; operators on slow links or heavy local builds can raise it.
+const DEFAULT_COMPOSE_STALL_TIMEOUT_MS = 10 * 60 * 1000;
+
+function getComposeStallTimeoutMs(): number {
+  const configured = Number(process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS);
+  if (Number.isFinite(configured) && configured > 0) {
+    return configured;
+  }
+  return DEFAULT_COMPOSE_STALL_TIMEOUT_MS;
+}
+
 /**
  * ComposeService - local docker compose CLI execution.
  *
@@ -98,7 +114,11 @@ export class ComposeService {
     cwd: string,
     ws?: WebSocket,
     throwOnError = true,
-    env?: Record<string, string | undefined>
+    env?: Record<string, string | undefined>,
+    // When set, terminate the child if it emits no output for this long while
+    // still running (idle stall backstop). Appended last so the existing
+    // registry-auth call sites that pass `env` are unaffected.
+    idleTimeoutMs?: number
   ): Promise<void> {
     return new Promise((resolve, reject) => {
       const child = spawn(command, args, {
@@ -116,6 +136,7 @@ export class ComposeService {
       const timeoutMs = getComposeCommandTimeoutMs();
       let timeout: ReturnType<typeof setTimeout> | null = null;
       let forceKillTimeout: ReturnType<typeof setTimeout> | null = null;
+      let idleTimeout: ReturnType<typeof setTimeout> | null = null;
 
       const sendOutput = (text: string) => {
         if (ws && ws.readyState === WebSocket.OPEN) {
@@ -132,6 +153,10 @@ export class ComposeService {
           clearTimeout(forceKillTimeout);
           forceKillTimeout = null;
         }
+        if (idleTimeout) {
+          clearTimeout(idleTimeout);
+          idleTimeout = null;
+        }
       };
 
       const finish = (complete: () => void) => {
@@ -159,21 +184,39 @@ export class ComposeService {
         }, 5000);
       };
 
+      // Idle stall backstop. Armed once below and reset on every output chunk;
+      // if it ever fires, the step has been silent for idleTimeoutMs while still
+      // running, so terminate it. Never rearmed after a termination is pending or
+      // the child has exited, so it cannot re-fire during the SIGTERM grace.
+      const armIdleTimeout = () => {
+        if (idleTimeoutMs === undefined) return;
+        if (exited || settled || pendingTerminationError) return;
+        if (idleTimeout) clearTimeout(idleTimeout);
+        idleTimeout = setTimeout(() => {
+          const seconds = Math.round(idleTimeoutMs / 1000);
+          sendOutput(`=== No output for ${seconds}s; the operation appears stalled and was stopped ===\n`);
+          terminateChild(new Error(`STACK_STALLED_OUTPUT: no output for ${seconds}s`));
+        }, idleTimeoutMs);
+      };
+
       // The progress socket is output-only: a deploy/update/down is owned by the
       // HTTP request that started it, so closing or losing the socket (the user
       // minimizes the panel, navigates away, or the connection blips) must not
       // terminate the compose process. Termination is driven solely by the
-      // command timeout below.
+      // command timeout here and the optional idle stall backstop above.
       timeout = setTimeout(() => {
         const message = `Command timed out after ${Math.round(timeoutMs / 1000)}s`;
         sendOutput(`${message}\n`);
         terminateChild(new Error(message));
       }, timeoutMs);
 
+      armIdleTimeout();
+
       const onData = (data: Buffer) => {
         const text = data.toString();
         errorLog += text;
         sendOutput(text);
+        armIdleTimeout();
       };
 
       child.stdout.on('data', onData);
@@ -328,7 +371,7 @@ export class ComposeService {
       }
 
       await this.withRegistryAuth(async (env) => {
-        await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env);
+        await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs());
       }, sendOutput);
 
       // Post-Deploy Health Probe
@@ -505,10 +548,10 @@ export class ComposeService {
 
       await this.withRegistryAuth(async (env) => {
         sendOutput('=== Pulling latest images ===\n');
-        await this.execute('docker', ['compose', 'pull'], stackDir, ws, true, env);
+        await this.execute('docker', ['compose', 'pull'], stackDir, ws, true, env, getComposeStallTimeoutMs());
 
         sendOutput('=== Recreating containers ===\n');
-        await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env);
+        await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs());
       }, sendOutput);
 
       // Post-Update Health Probe

backend/src/services/FileSystemService.ts

@@ -874,6 +874,52 @@ export class FileSystemService {
     if (debug) console.debug(`[FileSystemService:debug] Restore completed in ${Date.now() - t0}ms`, { stackName, restored: items.filter(i => i !== '.timestamp').length, removedOrphans });
   }
 
+  /**
+   * Capture the current managed stack files (PROTECTED_STACK_FILES) in memory and
+   * return a function that puts them back, faithfully (writing the captured
+   * contents and removing any managed file that did not exist when captured).
+   *
+   * Used by the rollback route to undo a restored backup when the policy gate
+   * blocks before the deploy commits: restoreStackFiles has already overwritten
+   * the on-disk files, so without this a blocked rollback would leave disk holding
+   * the rolled-back configuration while the deployed containers are unchanged.
+   */
+  async snapshotStackFiles(stackName: string): Promise<() => Promise<void>> {
+    const stackDir = this.resolveStackDir(stackName);
+    // Canonical js/path-injection barrier inline with the read/write sinks, the
+    // same pattern restoreStackFiles uses: resolve against the base and confirm
+    // containment so static analysis credits the barrier.
+    const baseResolved = path.resolve(this.baseDir);
+    const snapshot = new Map<string, Buffer>();
+    for (const file of PROTECTED_STACK_FILES) {
+      const target = path.resolve(baseResolved, path.join(stackDir, file));
+      if (!target.startsWith(baseResolved + path.sep)) {
+        throw Object.assign(new Error('Path escapes compose directory'), { code: 'INVALID_PATH' });
+      }
+      try {
+        snapshot.set(file, await fsPromises.readFile(target));
+      } catch (e: unknown) {
+        if ((e as NodeJS.ErrnoException)?.code !== 'ENOENT') throw e;
+      }
+    }
+    return async () => {
+      for (const file of PROTECTED_STACK_FILES) {
+        const target = path.resolve(baseResolved, path.join(stackDir, file));
+        if (!target.startsWith(baseResolved + path.sep)) continue;
+        const saved = snapshot.get(file);
+        if (saved !== undefined) {
+          await fsPromises.writeFile(target, saved);
+        } else {
+          try {
+            await fsPromises.unlink(target);
+          } catch (e: unknown) {
+            if ((e as NodeJS.ErrnoException)?.code !== 'ENOENT') throw e;
+          }
+        }
+      }
+    };
+  }
+
   async getBackupInfo(stackName: string): Promise<{ exists: boolean; timestamp: number | null }> {
     const backupDir = this.getBackupDir(stackName);
     try {