Files
sencho/e2e/auto-heal-policies.spec.ts
T
Anso 5bb4b01953 feat: auto-heal policies for unhealthy containers (#671)
* feat(db): add auto_heal_policies and auto_heal_history schema and CRUD

Adds two new SQLite tables (auto_heal_policies, auto_heal_history) to
DatabaseService.initSchema() and exposes CRUD methods: getAutoHealPolicies,
getAutoHealPolicy, addAutoHealPolicy, updateAutoHealPolicy,
deleteAutoHealPolicy, recordAutoHealHistory, getAutoHealHistory,
incrementConsecutiveFailures, resetConsecutiveFailures, setPolicyEnabled.
Also adds AutoHealPolicy and AutoHealHistoryEntry TypeScript interfaces.

* feat(events): track health-status duration and expose state accessors

- Add healthStatus and unhealthySince fields to InternalContainerState
- onHealthStatus now records unhealthySince timestamp on first transition
  to unhealthy, and clears it when the container recovers or restarts
- onStart resets both fields so a restarted container begins from 'starting'
- Add listContainerStates() and getContainerState() public accessors for
  use by the upcoming AutoHealService evaluator

* fix(auto-heal): key allowlist in updateAutoHealPolicy, cascade delete, extract ContainerHealthSnapshot

* feat: add AutoHealService evaluator singleton

Polls every 30 s, matches containers to enabled policies via Compose
labels, and restarts containers that have been unhealthy beyond the
configured threshold. Enforces cooldown, per-hour rate cap, and
recent-user-action suppression; auto-disables policies after repeated
consecutive failures. Also adds DockerEventManager.getService() accessor
required by the evaluator.

* fix(auto-heal): prune stale restartTimestamps, guard undefined policy id

- Prune restartTimestamps entries for containers no longer running after
  each container list fetch, preventing unbounded map growth from dead
  container IDs.
- Guard against policies with undefined id at the start of the per-policy
  loop; warn and skip rather than proceed with a non-null assertion.
- Extract handleAutoDisable private helper to bring executeHeal under 30
  lines and isolate the auto-disable side-effect sequence.
- Move ContainerInfo type to module scope.

* feat: add auto-heal API routes and wire AutoHealService lifecycle

Registers five REST endpoints under /api/auto-heal/policies (list, create,
patch, delete, history) with requirePaid + requireAdmin guards and Zod
validation. Wires AutoHealService.start()/stop() into the server startup
and graceful-shutdown blocks alongside MonitorService.

* test: add AutoHealService and DatabaseService auto-heal unit tests

- 15 unit tests for AutoHealService.shouldHeal covering all decision branches
  (healthy state, duration threshold, user-action suppression, cooldown,
  rate limiting, and correct skipReason values)
- 13 integration tests for DatabaseService auto-heal CRUD: policy round-trip,
  stack-name filter, partial update, cascade delete, history ordering/limit,
  consecutive failure counters, and setPolicyEnabled toggle

* fix: log AutoHealService shutdown errors consistently

* fix(api): requireAdmin-first guard order and try/catch on auto-heal routes

* feat(ui): add StackAutoHealSheet component

* feat(ui): add Auto-Heal context menu item to EditorLayout

* fix(ui): StackAutoHealSheet label, token, a11y, and useEffect fixes

- Rename 'All services in stack' to 'All services' in combobox options and placeholder
- Replace text-green-600 with text-success design token in actionColorClass
- Add htmlFor/id pairs to all four numeric form inputs for accessibility
- Inline fetch logic into useEffect, removing stale closure risk and eslint-disable comment
- Remove now-unused fetchPolicies and fetchServices standalone functions
- Update 'Auto-disable after' label to 'Auto-disable after (failures)' for clarity
- Add toast.error in policy fetch failure path; services fetch silently skips as before

* docs: add auto-heal-policies feature documentation

* test(e2e): add auto-heal policies CRUD spec

* fix(docs): correct auto-heal-policies nav position in docs.json
2026-04-17 22:45:06 -04:00

102 lines
4.4 KiB
TypeScript

/**
* Auto-Heal Policies E2E tests - happy-path CRUD via the sheet UI.
*
* Opens the Auto-Heal sheet from the stack sidebar context menu, creates a
* policy, verifies it appears in the list, then deletes it.
*
* Requires a paid license (Skipper or Admiral) on the test instance. The test
* skips gracefully when the PaidGate upgrade prompt is detected instead.
*/
import { test, expect } from '@playwright/test';
import { loginAs } from './helpers';
/** Wait for the stacks sidebar to finish loading. */
async function waitForStacksLoaded(page: import('@playwright/test').Page) {
await expect(page.getByRole('button', { name: 'Create Stack' })).toBeVisible({ timeout: 15_000 });
await expect(page.locator('[data-stacks-loaded="true"]')).toBeAttached({ timeout: 15_000 });
}
test.describe('Auto-Heal Policies', () => {
test.beforeEach(async ({ page }) => {
await loginAs(page);
await waitForStacksLoaded(page);
});
test('CRUD: create and delete a policy via the sheet', async ({ page }) => {
// Find the first stack item in the sidebar. cmdk renders items with role="option".
const stackItems = page.locator('[data-stacks-loaded="true"] [role="option"]');
const count = await stackItems.count();
if (count === 0) {
test.skip();
return;
}
// Right-click the first stack to open the context menu
await stackItems.first().click({ button: 'right' });
// Wait for the Radix context menu to appear
await expect(page.locator('[role="menu"]')).toBeVisible({ timeout: 5_000 });
// Click "Auto-Heal" menu item
await page.locator('[role="menu"]').getByText('Auto-Heal').click();
// The sheet title should be visible
await expect(page.getByText(/Auto-Heal Policies/)).toBeVisible({ timeout: 5_000 });
// Detect PaidGate: skip if the upgrade prompt blocks the UI (community instance)
const upgradePrompt = page.getByText(/requires a paid license/i);
if (await upgradePrompt.isVisible({ timeout: 2_000 }).catch(() => false)) {
test.skip();
return;
}
// Wait for the policy list and form to be ready
await expect(page.getByText('Active Policies')).toBeVisible({ timeout: 5_000 });
await expect(page.getByRole('button', { name: 'Add Policy' })).toBeVisible({ timeout: 8_000 });
// ── Fill in the add-policy form ─────────────────────────────────────────
// Leave Service as default "All services"
const unhealthyInput = page.locator('#unhealthy-duration');
await unhealthyInput.clear();
await unhealthyInput.fill('2');
const cooldownInput = page.locator('#cooldown');
await cooldownInput.clear();
await cooldownInput.fill('5');
const maxRestartsInput = page.locator('#max-restarts');
await maxRestartsInput.clear();
await maxRestartsInput.fill('3');
const autoDisableInput = page.locator('#auto-disable');
await autoDisableInput.clear();
await autoDisableInput.fill('5');
// ── Submit ──────────────────────────────────────────────────────────────
await page.getByRole('button', { name: 'Add Policy' }).click();
// Wait for the save to complete (button re-enables) then verify the row appears
await expect(page.getByRole('button', { name: 'Add Policy' })).toBeEnabled({ timeout: 8_000 });
// The PolicyRow subtitle shows "Unhealthy for 2 min" for the value we entered
const policySubtitle = page.getByText(/Unhealthy for 2 min/i).first();
await expect(policySubtitle).toBeVisible({ timeout: 8_000 });
// ── Delete the policy ───────────────────────────────────────────────────
// Find the policy card containing the subtitle and click its delete button
const policyCard = page
.locator('.rounded-lg.border')
.filter({ hasText: 'Unhealthy for 2 min' })
.first();
await expect(policyCard).toBeVisible({ timeout: 5_000 });
const deleteBtn = policyCard.getByRole('button', { name: 'Delete policy' });
await expect(deleteBtn).toBeVisible({ timeout: 5_000 });
await deleteBtn.click();
// Confirm the policy row is removed
await expect(policySubtitle).not.toBeVisible({ timeout: 8_000 });
});
});