feat: detect stalled stack updates and add in-app recovery actions (#1347)

* feat: detect stalled stack updates and add in-app recovery actions

Add a backend idle-output backstop that stops a deploy/update compose step
that has gone silent (SENCHO_COMPOSE_STALL_TIMEOUT_MS, default 10m), so a
hung image pull surfaces a fast failure instead of spinning indefinitely.

Surface failed, timed-out, and stalled operations with recovery actions on
the stack page: a desktop chip plus popover menu and an inline mobile card
offering retry, restart, roll back (when a backup exists), refresh state,
and copy diagnostics, all gated by deploy permission. The streaming
deploy/update progress modal is now on by default and warns when output
goes quiet. Container state is refreshed after a failed or stalled
operation, and the UI never sits in an indefinite spinner.

* fix: harden rollback against policy-blocked file mutation and refine recovery

Address review findings on the stalled-update recovery work:

- The rollback route restored backup files before running the policy gate, so
  a policy-blocked rollback could leave the on-disk config rolled back while the
  deployed containers were unchanged. Snapshot the current files first and
  revert them when the gate blocks; if that revert itself fails, escalate it on
  the persistent alert feed since the 409 is already sent.
- Refresh container state after a successful manual rollback (rollback
  redeploys), without mis-recording a refetch failure as a rollback failure.
- Suppress the stalled-output warning once live progress is unavailable.

* test: mock snapshotStackFiles in the atomic-deploy rollback route tests

The rollback route now snapshots stack files before restoring a backup, so its
FileSystemService mock needs snapshotStackFiles. Without it the mocked call
threw and the route returned 500, failing the success-path rollback assertions.
This commit is contained in:
Anso
2026-06-10 10:12:24 -04:00
committed by GitHub
parent a3033a848e
commit d369b03a38
31 changed files with 1580 additions and 76 deletions
@@ -0,0 +1,42 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { renderHook, act } from '@testing-library/react';
import { useDeployFeedbackEnabled, DEPLOY_FEEDBACK_KEY } from '../use-deploy-feedback-enabled';
describe('useDeployFeedbackEnabled (opt-out default)', () => {
beforeEach(() => localStorage.clear());
afterEach(() => localStorage.clear());
it('defaults to enabled when no value is stored', () => {
const { result } = renderHook(() => useDeployFeedbackEnabled());
expect(result.current[0]).toBe(true);
});
it('is disabled only when explicitly set to false', () => {
localStorage.setItem(DEPLOY_FEEDBACK_KEY, 'false');
const { result } = renderHook(() => useDeployFeedbackEnabled());
expect(result.current[0]).toBe(false);
});
it('treats any non-false value as enabled', () => {
localStorage.setItem(DEPLOY_FEEDBACK_KEY, 'true');
const { result } = renderHook(() => useDeployFeedbackEnabled());
expect(result.current[0]).toBe(true);
});
it('re-enables when a storage event clears the key (newValue null)', () => {
localStorage.setItem(DEPLOY_FEEDBACK_KEY, 'false');
const { result } = renderHook(() => useDeployFeedbackEnabled());
expect(result.current[0]).toBe(false);
act(() => {
window.dispatchEvent(new StorageEvent('storage', { key: DEPLOY_FEEDBACK_KEY, newValue: null }));
});
expect(result.current[0]).toBe(true);
});
it('setEnabled(false) persists false and disables', () => {
const { result } = renderHook(() => useDeployFeedbackEnabled());
act(() => result.current[1](false));
expect(result.current[0]).toBe(false);
expect(localStorage.getItem(DEPLOY_FEEDBACK_KEY)).toBe('false');
});
});
@@ -3,12 +3,15 @@ import { SENCHO_SETTINGS_CHANGED } from '@/lib/events';
export const DEPLOY_FEEDBACK_KEY = 'sencho.deploy-feedback.enabled';
// Default on (opt-out): the live deploy/update output panel shows unless the
// user has explicitly turned it off, so update progress and the stalled-output
// warning are visible without a setting hunt. Only a stored 'false' disables it.
function readStored(): boolean {
if (typeof window === 'undefined') return false;
if (typeof window === 'undefined') return true;
try {
return window.localStorage.getItem(DEPLOY_FEEDBACK_KEY) === 'true';
return window.localStorage.getItem(DEPLOY_FEEDBACK_KEY) !== 'false';
} catch {
return false;
return true;
}
}
@@ -26,7 +29,9 @@ export function useDeployFeedbackEnabled(): [boolean, (next: boolean) => void] {
useEffect(() => {
function onStorage(event: StorageEvent) {
if (event.key !== DEPLOY_FEEDBACK_KEY) return;
setEnabledState(event.newValue === 'true');
// Opt-out: anything other than an explicit 'false' (including a
// cleared key) means enabled.
setEnabledState(event.newValue !== 'false');
}
window.addEventListener('storage', onStorage);
return () => window.removeEventListener('storage', onStorage);