feat: detect stalled stack updates and add in-app recovery actions (#1347)

* feat: detect stalled stack updates and add in-app recovery actions

Add a backend idle-output backstop that stops a deploy/update compose step
that has gone silent (SENCHO_COMPOSE_STALL_TIMEOUT_MS, default 10m), so a
hung image pull surfaces a fast failure instead of spinning indefinitely.

Surface failed, timed-out, and stalled operations with recovery actions on
the stack page: a desktop chip plus popover menu and an inline mobile card
offering retry, restart, roll back (when a backup exists), refresh state,
and copy diagnostics, all gated by deploy permission. The streaming
deploy/update progress modal is now on by default and warns when output
goes quiet. Container state is refreshed after a failed or stalled
operation, and the UI never sits in an indefinite spinner.

* fix: harden rollback against policy-blocked file mutation and refine recovery

Address review findings on the stalled-update recovery work:

- The rollback route restored backup files before running the policy gate, so
  a policy-blocked rollback could leave the on-disk config rolled back while the
  deployed containers were unchanged. Snapshot the current files first and
  revert them when the gate blocks; if that revert itself fails, escalate it on
  the persistent alert feed since the 409 is already sent.
- Refresh container state after a successful manual rollback (rollback
  redeploys), without mis-recording a refetch failure as a rollback failure.
- Suppress the stalled-output warning once live progress is unavailable.

* test: mock snapshotStackFiles in the atomic-deploy rollback route tests

The rollback route now snapshots stack files before restoring a backup, so its
FileSystemService mock needs snapshotStackFiles. Without it the mocked call
threw and the route returned 500, failing the success-path rollback assertions.

frontend/src/components/EditorLayout/__tests__/RecoveryChip.test.tsx

@@ -0,0 +1,60 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { render, screen, fireEvent } from '@testing-library/react';
+import { RecoveryChip } from '../RecoveryChip';
+import type { StackActionResult } from '../EditorView';
+
+vi.mock('@/lib/clipboard', () => ({ copyToClipboard: vi.fn().mockResolvedValue(undefined) }));
+vi.mock('@/components/ui/toast-store', () => ({ toast: { success: vi.fn(), error: vi.fn() } }));
+
+const baseResult: StackActionResult = {
+  action: 'update',
+  rolledBack: false,
+  errorMessage: 'pull failed: connection reset',
+  startedAt: 1000,
+  endedAt: 4000,
+};
+
+function setup(over: Partial<Parameters<typeof RecoveryChip>[0]> = {}) {
+  const props = {
+    stackName: 'web',
+    result: baseResult,
+    activeNode: null,
+    backupInfo: { exists: false, timestamp: null },
+    canDeploy: true,
+    onRetry: vi.fn(),
+    onRestart: vi.fn(),
+    onRollback: vi.fn(),
+    onRefreshState: vi.fn(),
+    onDismiss: vi.fn(),
+    ...over,
+  };
+  render(<RecoveryChip {...props} />);
+  return props;
+}
+
+describe('RecoveryChip', () => {
+  beforeEach(() => vi.clearAllMocks());
+
+  it('renders a chip with the failed action and keeps actions hidden until opened', () => {
+    setup();
+    expect(screen.getByTestId('recovery-chip')).toBeInTheDocument();
+    expect(screen.getByText(/Update failed/)).toBeInTheDocument();
+    expect(screen.queryByText('Retry update')).not.toBeInTheDocument();
+  });
+
+  it('reveals the recovery actions when the chip is clicked', () => {
+    const props = setup();
+    fireEvent.click(screen.getByTestId('recovery-chip'));
+    expect(screen.getByText('Retry update')).toBeInTheDocument();
+    expect(screen.getByText('Refresh')).toBeInTheDocument();
+    fireEvent.click(screen.getByText('Retry update'));
+    expect(props.onRetry).toHaveBeenCalledTimes(1);
+  });
+
+  it('dismisses from inside the popover', () => {
+    const props = setup();
+    fireEvent.click(screen.getByTestId('recovery-chip'));
+    fireEvent.click(screen.getByText('Dismiss'));
+    expect(props.onDismiss).toHaveBeenCalledTimes(1);
+  });
+});

frontend/src/components/EditorLayout/__tests__/RecoveryPanel.test.tsx

@@ -0,0 +1,92 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { render, screen, fireEvent } from '@testing-library/react';
+import { RecoveryPanel } from '../RecoveryPanel';
+import type { StackActionResult } from '../EditorView';
+
+vi.mock('@/lib/clipboard', () => ({ copyToClipboard: vi.fn().mockResolvedValue(undefined) }));
+vi.mock('@/components/ui/toast-store', () => ({ toast: { success: vi.fn(), error: vi.fn() } }));
+
+import { copyToClipboard } from '@/lib/clipboard';
+
+const baseResult: StackActionResult = {
+  action: 'update',
+  rolledBack: false,
+  errorMessage: 'pull failed: connection reset',
+  startedAt: 1000,
+  endedAt: 4000,
+};
+
+function setup(over: Partial<Parameters<typeof RecoveryPanel>[0]> = {}) {
+  const props = {
+    stackName: 'web',
+    result: baseResult,
+    activeNode: null,
+    backupInfo: { exists: false, timestamp: null },
+    canDeploy: true,
+    onRetry: vi.fn(),
+    onRestart: vi.fn(),
+    onRollback: vi.fn(),
+    onRefreshState: vi.fn(),
+    onDismiss: vi.fn(),
+    ...over,
+  };
+  render(<RecoveryPanel {...props} />);
+  return props;
+}
+
+describe('RecoveryPanel', () => {
+  beforeEach(() => vi.clearAllMocks());
+
+  it('shows the failed action title and error message', () => {
+    setup();
+    expect(screen.getByText(/Update failed/)).toBeInTheDocument();
+    expect(screen.getByText(/pull failed: connection reset/)).toBeInTheDocument();
+  });
+
+  it('calls onRetry from the retry button', () => {
+    const props = setup();
+    fireEvent.click(screen.getByText('Retry update'));
+    expect(props.onRetry).toHaveBeenCalledTimes(1);
+  });
+
+  it('hides retry/restart/rollback without the deploy permission', () => {
+    setup({ canDeploy: false, backupInfo: { exists: true, timestamp: 1 } });
+    expect(screen.queryByText('Retry update')).not.toBeInTheDocument();
+    expect(screen.queryByText('Restart')).not.toBeInTheDocument();
+    expect(screen.queryByText('Roll back')).not.toBeInTheDocument();
+    // Read-level actions remain available.
+    expect(screen.getByText('Refresh')).toBeInTheDocument();
+    expect(screen.getByText('Copy details')).toBeInTheDocument();
+  });
+
+  it('offers rollback only when a backup exists', () => {
+    setup({ backupInfo: { exists: false, timestamp: null } });
+    expect(screen.queryByText('Roll back')).not.toBeInTheDocument();
+    setup({ backupInfo: { exists: true, timestamp: 123 } });
+    expect(screen.getByText('Roll back')).toBeInTheDocument();
+  });
+
+  it('does not show a redundant restart button when the failed action was a restart', () => {
+    setup({ result: { ...baseResult, action: 'restart' } });
+    expect(screen.getByText('Retry restart')).toBeInTheDocument();
+    expect(screen.queryByText('Restart')).not.toBeInTheDocument();
+  });
+
+  it('copies session-safe diagnostics including stack and error', () => {
+    setup({ result: { ...baseResult, lastOutputLine: 'pulling app ...' } });
+    fireEvent.click(screen.getByText('Copy details'));
+    expect(copyToClipboard).toHaveBeenCalledTimes(1);
+    const blob = vi.mocked(copyToClipboard).mock.calls[0][0];
+    expect(blob).toContain('Stack: web');
+    expect(blob).toContain('pull failed: connection reset');
+    expect(blob).toContain('Last output: pulling app ...');
+  });
+
+  it('wires refresh and dismiss callbacks', () => {
+    const props = setup();
+    fireEvent.click(screen.getByText('Refresh'));
+    fireEvent.click(screen.getByLabelText('Dismiss recovery panel'));
+    expect(props.onRefreshState).toHaveBeenCalledTimes(1);
+    expect(props.onDismiss).toHaveBeenCalledTimes(1);
+  });
+});