mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-21 07:36:40 +00:00
feat: auto-heal policies for unhealthy containers (#671)
* feat(db): add auto_heal_policies and auto_heal_history schema and CRUD Adds two new SQLite tables (auto_heal_policies, auto_heal_history) to DatabaseService.initSchema() and exposes CRUD methods: getAutoHealPolicies, getAutoHealPolicy, addAutoHealPolicy, updateAutoHealPolicy, deleteAutoHealPolicy, recordAutoHealHistory, getAutoHealHistory, incrementConsecutiveFailures, resetConsecutiveFailures, setPolicyEnabled. Also adds AutoHealPolicy and AutoHealHistoryEntry TypeScript interfaces. * feat(events): track health-status duration and expose state accessors - Add healthStatus and unhealthySince fields to InternalContainerState - onHealthStatus now records unhealthySince timestamp on first transition to unhealthy, and clears it when the container recovers or restarts - onStart resets both fields so a restarted container begins from 'starting' - Add listContainerStates() and getContainerState() public accessors for use by the upcoming AutoHealService evaluator * fix(auto-heal): key allowlist in updateAutoHealPolicy, cascade delete, extract ContainerHealthSnapshot * feat: add AutoHealService evaluator singleton Polls every 30 s, matches containers to enabled policies via Compose labels, and restarts containers that have been unhealthy beyond the configured threshold. Enforces cooldown, per-hour rate cap, and recent-user-action suppression; auto-disables policies after repeated consecutive failures. Also adds DockerEventManager.getService() accessor required by the evaluator. * fix(auto-heal): prune stale restartTimestamps, guard undefined policy id - Prune restartTimestamps entries for containers no longer running after each container list fetch, preventing unbounded map growth from dead container IDs. - Guard against policies with undefined id at the start of the per-policy loop; warn and skip rather than proceed with a non-null assertion. - Extract handleAutoDisable private helper to bring executeHeal under 30 lines and isolate the auto-disable side-effect sequence. - Move ContainerInfo type to module scope. * feat: add auto-heal API routes and wire AutoHealService lifecycle Registers five REST endpoints under /api/auto-heal/policies (list, create, patch, delete, history) with requirePaid + requireAdmin guards and Zod validation. Wires AutoHealService.start()/stop() into the server startup and graceful-shutdown blocks alongside MonitorService. * test: add AutoHealService and DatabaseService auto-heal unit tests - 15 unit tests for AutoHealService.shouldHeal covering all decision branches (healthy state, duration threshold, user-action suppression, cooldown, rate limiting, and correct skipReason values) - 13 integration tests for DatabaseService auto-heal CRUD: policy round-trip, stack-name filter, partial update, cascade delete, history ordering/limit, consecutive failure counters, and setPolicyEnabled toggle * fix: log AutoHealService shutdown errors consistently * fix(api): requireAdmin-first guard order and try/catch on auto-heal routes * feat(ui): add StackAutoHealSheet component * feat(ui): add Auto-Heal context menu item to EditorLayout * fix(ui): StackAutoHealSheet label, token, a11y, and useEffect fixes - Rename 'All services in stack' to 'All services' in combobox options and placeholder - Replace text-green-600 with text-success design token in actionColorClass - Add htmlFor/id pairs to all four numeric form inputs for accessibility - Inline fetch logic into useEffect, removing stale closure risk and eslint-disable comment - Remove now-unused fetchPolicies and fetchServices standalone functions - Update 'Auto-disable after' label to 'Auto-disable after (failures)' for clarity - Add toast.error in policy fetch failure path; services fetch silently skips as before * docs: add auto-heal-policies feature documentation * test(e2e): add auto-heal policies CRUD spec * fix(docs): correct auto-heal-policies nav position in docs.json
This commit is contained in:
Anso
@@ -0,0 +1,168 @@
|
||||
/**
|
||||
* Unit tests for AutoHealService.shouldHeal decision logic.
|
||||
*
|
||||
* shouldHeal is private; accessed via type cast (service as any) to avoid
|
||||
* exposing it in production API surface. All tests are pure (no I/O, no
|
||||
* timers) - they exercise the decision function directly.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { INTENTIONAL_KILL_WINDOW_MS } from '../services/ContainerLifecycleClassifier';
|
||||
import { AutoHealService } from '../services/AutoHealService';
|
||||
|
||||
describe('AutoHealService.shouldHeal', () => {
|
||||
let service: any;
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset singleton so each test starts with a clean restartTimestamps map
|
||||
(AutoHealService as any).instance = undefined;
|
||||
service = AutoHealService.getInstance();
|
||||
});
|
||||
|
||||
const basePolicy = {
|
||||
id: 1,
|
||||
stack_name: 'mystack',
|
||||
service_name: null,
|
||||
unhealthy_duration_mins: 5,
|
||||
cooldown_mins: 10,
|
||||
max_restarts_per_hour: 3,
|
||||
auto_disable_after_failures: 5,
|
||||
enabled: 1,
|
||||
consecutive_failures: 0,
|
||||
last_fired_at: 0,
|
||||
created_at: Date.now(),
|
||||
updated_at: Date.now(),
|
||||
};
|
||||
|
||||
const baseState = {
|
||||
id: 'container123',
|
||||
name: 'mystack-web-1',
|
||||
stackName: 'mystack',
|
||||
healthStatus: 'unhealthy' as const,
|
||||
unhealthySince: Date.now() - 6 * 60_000, // 6 minutes ago (past 5 min threshold)
|
||||
lastKillAt: undefined,
|
||||
};
|
||||
|
||||
it('returns heal:true when all conditions are met', () => {
|
||||
const result = service.shouldHeal(baseState, basePolicy, 'container123', Date.now());
|
||||
expect(result.heal).toBe(true);
|
||||
});
|
||||
|
||||
it('returns heal:false when healthStatus is not unhealthy', () => {
|
||||
const result = service.shouldHeal(
|
||||
{ ...baseState, healthStatus: 'healthy' },
|
||||
basePolicy,
|
||||
'container123',
|
||||
Date.now(),
|
||||
);
|
||||
expect(result.heal).toBe(false);
|
||||
});
|
||||
|
||||
it('returns heal:false when healthStatus is undefined', () => {
|
||||
const result = service.shouldHeal(
|
||||
{ ...baseState, healthStatus: undefined },
|
||||
basePolicy,
|
||||
'container123',
|
||||
Date.now(),
|
||||
);
|
||||
expect(result.heal).toBe(false);
|
||||
});
|
||||
|
||||
it('returns heal:false when state is undefined', () => {
|
||||
const result = service.shouldHeal(undefined, basePolicy, 'container123', Date.now());
|
||||
expect(result.heal).toBe(false);
|
||||
});
|
||||
|
||||
it('returns heal:false when unhealthySince is undefined', () => {
|
||||
const result = service.shouldHeal(
|
||||
{ ...baseState, unhealthySince: undefined },
|
||||
basePolicy,
|
||||
'container123',
|
||||
Date.now(),
|
||||
);
|
||||
expect(result.heal).toBe(false);
|
||||
});
|
||||
|
||||
it('returns heal:false when duration threshold is not yet met', () => {
|
||||
// Only 2 minutes, threshold is 5
|
||||
const state = { ...baseState, unhealthySince: Date.now() - 2 * 60_000 };
|
||||
const result = service.shouldHeal(state, basePolicy, 'container123', Date.now());
|
||||
expect(result.heal).toBe(false);
|
||||
expect(result.skipReason).toBe('duration_not_met');
|
||||
});
|
||||
|
||||
it('returns skipped_user_action when lastKillAt is within the window', () => {
|
||||
// 30s ago, well within the 60s INTENTIONAL_KILL_WINDOW_MS
|
||||
const state = { ...baseState, lastKillAt: Date.now() - 30_000 };
|
||||
const result = service.shouldHeal(state, basePolicy, 'container123', Date.now());
|
||||
expect(result.heal).toBe(false);
|
||||
expect(result.skipReason).toBe('skipped_user_action');
|
||||
});
|
||||
|
||||
it('does not suppress when lastKillAt is outside the intentional kill window', () => {
|
||||
const state = {
|
||||
...baseState,
|
||||
lastKillAt: Date.now() - (INTENTIONAL_KILL_WINDOW_MS + 5_000),
|
||||
};
|
||||
const result = service.shouldHeal(state, basePolicy, 'container123', Date.now());
|
||||
expect(result.heal).toBe(true);
|
||||
});
|
||||
|
||||
it('returns skipped_cooldown when last_fired_at is within cooldown period', () => {
|
||||
// Fired 5 min ago, cooldown is 10 min
|
||||
const policy = { ...basePolicy, last_fired_at: Date.now() - 5 * 60_000, cooldown_mins: 10 };
|
||||
const result = service.shouldHeal(baseState, policy, 'container123', Date.now());
|
||||
expect(result.heal).toBe(false);
|
||||
expect(result.skipReason).toBe('skipped_cooldown');
|
||||
});
|
||||
|
||||
it('does not apply cooldown when last_fired_at is 0', () => {
|
||||
const policy = { ...basePolicy, last_fired_at: 0 };
|
||||
const result = service.shouldHeal(baseState, policy, 'container123', Date.now());
|
||||
expect(result.heal).toBe(true);
|
||||
});
|
||||
|
||||
it('does not apply cooldown when last_fired_at exceeds the cooldown window', () => {
|
||||
// Fired 15 min ago, cooldown is 10 min
|
||||
const policy = { ...basePolicy, last_fired_at: Date.now() - 15 * 60_000, cooldown_mins: 10 };
|
||||
const result = service.shouldHeal(baseState, policy, 'container123', Date.now());
|
||||
expect(result.heal).toBe(true);
|
||||
});
|
||||
|
||||
it('returns skipped_rate_limit when hourly restart count is at the configured max', () => {
|
||||
const now = Date.now();
|
||||
// Pre-populate with 3 entries within the last hour (policy max is 3)
|
||||
const map = (service as any).restartTimestamps as Map<string, number[]>;
|
||||
map.set('container123', [now - 10_000, now - 20_000, now - 30_000]);
|
||||
const result = service.shouldHeal(baseState, basePolicy, 'container123', now);
|
||||
expect(result.heal).toBe(false);
|
||||
expect(result.skipReason).toBe('skipped_rate_limit');
|
||||
});
|
||||
|
||||
it('does not rate-limit when all timestamps are older than one hour', () => {
|
||||
const now = Date.now();
|
||||
const map = (service as any).restartTimestamps as Map<string, number[]>;
|
||||
// All entries are >1 hour old, so they fall outside the rate-limit window
|
||||
map.set('container123', [now - 70 * 60_000, now - 80 * 60_000, now - 90 * 60_000]);
|
||||
const result = service.shouldHeal(baseState, basePolicy, 'container123', now);
|
||||
expect(result.heal).toBe(true);
|
||||
});
|
||||
|
||||
it('counts only recent timestamps toward the rate limit', () => {
|
||||
const now = Date.now();
|
||||
const map = (service as any).restartTimestamps as Map<string, number[]>;
|
||||
// 2 old (outside window) + 1 recent = 1 active restart; max is 3, so still allowed
|
||||
map.set('container123', [now - 70 * 60_000, now - 80 * 60_000, now - 5_000]);
|
||||
const result = service.shouldHeal(baseState, basePolicy, 'container123', now);
|
||||
expect(result.heal).toBe(true);
|
||||
});
|
||||
|
||||
it('returns not_unhealthy as skipReason when container is healthy', () => {
|
||||
const result = service.shouldHeal(
|
||||
{ ...baseState, healthStatus: 'healthy' },
|
||||
basePolicy,
|
||||
'container123',
|
||||
Date.now(),
|
||||
);
|
||||
expect(result.skipReason).toBe('not_unhealthy');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* Integration tests for DatabaseService auto-heal CRUD.
|
||||
*
|
||||
* Uses a real temp SQLite database (same pattern as database-metrics.test.ts).
|
||||
* All operations are tested against live SQL; no mocking.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
|
||||
let tmpDir: string;
|
||||
let db: any;
|
||||
|
||||
const makePolicy = (overrides: Record<string, unknown> = {}) => {
|
||||
const now = Date.now();
|
||||
return {
|
||||
stack_name: 'teststack',
|
||||
service_name: null,
|
||||
unhealthy_duration_mins: 5,
|
||||
cooldown_mins: 10,
|
||||
max_restarts_per_hour: 3,
|
||||
auto_disable_after_failures: 5,
|
||||
enabled: 1,
|
||||
consecutive_failures: 0,
|
||||
last_fired_at: 0,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
...overrides,
|
||||
};
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
const { DatabaseService } = await import('../services/DatabaseService');
|
||||
db = DatabaseService.getInstance();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
describe('DatabaseService - auto-heal policy CRUD', () => {
|
||||
it('addAutoHealPolicy + getAutoHealPolicy round-trip preserves all fields', () => {
|
||||
const input = makePolicy({
|
||||
stack_name: 'roundtrip-stack',
|
||||
service_name: 'web',
|
||||
unhealthy_duration_mins: 3,
|
||||
cooldown_mins: 15,
|
||||
max_restarts_per_hour: 5,
|
||||
auto_disable_after_failures: 10,
|
||||
enabled: 1,
|
||||
consecutive_failures: 0,
|
||||
last_fired_at: 0,
|
||||
});
|
||||
|
||||
const created = db.addAutoHealPolicy(input);
|
||||
expect(created.id).toBeDefined();
|
||||
expect(typeof created.id).toBe('number');
|
||||
|
||||
const fetched = db.getAutoHealPolicy(created.id);
|
||||
expect(fetched).toBeDefined();
|
||||
expect(fetched.stack_name).toBe('roundtrip-stack');
|
||||
expect(fetched.service_name).toBe('web');
|
||||
expect(fetched.unhealthy_duration_mins).toBe(3);
|
||||
expect(fetched.cooldown_mins).toBe(15);
|
||||
expect(fetched.max_restarts_per_hour).toBe(5);
|
||||
expect(fetched.auto_disable_after_failures).toBe(10);
|
||||
expect(fetched.enabled).toBe(1);
|
||||
expect(fetched.consecutive_failures).toBe(0);
|
||||
expect(fetched.last_fired_at).toBe(0);
|
||||
expect(fetched.created_at).toBe(input.created_at);
|
||||
});
|
||||
|
||||
it('getAutoHealPolicies without filter returns all policies', () => {
|
||||
const before = db.getAutoHealPolicies().length;
|
||||
|
||||
db.addAutoHealPolicy(makePolicy({ stack_name: 'all-stack-a' }));
|
||||
db.addAutoHealPolicy(makePolicy({ stack_name: 'all-stack-b' }));
|
||||
|
||||
const after = db.getAutoHealPolicies();
|
||||
expect(after.length).toBe(before + 2);
|
||||
});
|
||||
|
||||
it('getAutoHealPolicies with stackName filter returns only matching rows', () => {
|
||||
db.addAutoHealPolicy(makePolicy({ stack_name: 'filter-stack-x' }));
|
||||
db.addAutoHealPolicy(makePolicy({ stack_name: 'filter-stack-y' }));
|
||||
|
||||
const xPolicies = db.getAutoHealPolicies('filter-stack-x');
|
||||
const yPolicies = db.getAutoHealPolicies('filter-stack-y');
|
||||
|
||||
expect(xPolicies.length).toBeGreaterThanOrEqual(1);
|
||||
expect(xPolicies.every((p: any) => p.stack_name === 'filter-stack-x')).toBe(true);
|
||||
|
||||
expect(yPolicies.length).toBeGreaterThanOrEqual(1);
|
||||
expect(yPolicies.every((p: any) => p.stack_name === 'filter-stack-y')).toBe(true);
|
||||
});
|
||||
|
||||
it('updateAutoHealPolicy partial update changes only specified fields', () => {
|
||||
const created = db.addAutoHealPolicy(makePolicy({ stack_name: 'update-stack', cooldown_mins: 10 }));
|
||||
|
||||
db.updateAutoHealPolicy(created.id, { cooldown_mins: 30, max_restarts_per_hour: 7 });
|
||||
|
||||
const updated = db.getAutoHealPolicy(created.id);
|
||||
// Changed fields
|
||||
expect(updated.cooldown_mins).toBe(30);
|
||||
expect(updated.max_restarts_per_hour).toBe(7);
|
||||
// Unchanged fields must be preserved
|
||||
expect(updated.stack_name).toBe('update-stack');
|
||||
expect(updated.unhealthy_duration_mins).toBe(5);
|
||||
// updated_at should be refreshed
|
||||
expect(updated.updated_at).toBeGreaterThanOrEqual(created.updated_at);
|
||||
});
|
||||
|
||||
it('deleteAutoHealPolicy removes the policy row', () => {
|
||||
const created = db.addAutoHealPolicy(makePolicy({ stack_name: 'delete-stack' }));
|
||||
expect(db.getAutoHealPolicy(created.id)).toBeDefined();
|
||||
|
||||
db.deleteAutoHealPolicy(created.id);
|
||||
|
||||
expect(db.getAutoHealPolicy(created.id)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('deleteAutoHealPolicy cascades and removes history rows', () => {
|
||||
const created = db.addAutoHealPolicy(makePolicy({ stack_name: 'cascade-stack' }));
|
||||
const policyId: number = created.id;
|
||||
|
||||
// Write a history entry tied to this policy
|
||||
db.recordAutoHealHistory({
|
||||
policy_id: policyId,
|
||||
stack_name: 'cascade-stack',
|
||||
service_name: null,
|
||||
container_name: 'cascade-stack-web-1',
|
||||
container_id: 'cascade-abc',
|
||||
action: 'restarted',
|
||||
reason: 'test cascade',
|
||||
success: 1,
|
||||
error: null,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
const beforeDelete = db.getAutoHealHistory(policyId);
|
||||
expect(beforeDelete.length).toBe(1);
|
||||
|
||||
db.deleteAutoHealPolicy(policyId);
|
||||
|
||||
const afterDelete = db.getAutoHealHistory(policyId);
|
||||
expect(afterDelete.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DatabaseService - auto-heal history', () => {
|
||||
it('recordAutoHealHistory + getAutoHealHistory orders results by timestamp DESC', () => {
|
||||
const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'history-order-stack' }));
|
||||
const policyId: number = policy.id;
|
||||
const baseTs = Date.now();
|
||||
|
||||
db.recordAutoHealHistory({
|
||||
policy_id: policyId,
|
||||
stack_name: 'history-order-stack',
|
||||
service_name: null,
|
||||
container_name: 'history-order-stack-web-1',
|
||||
container_id: 'hist-001',
|
||||
action: 'restarted',
|
||||
reason: 'first',
|
||||
success: 1,
|
||||
error: null,
|
||||
timestamp: baseTs,
|
||||
});
|
||||
db.recordAutoHealHistory({
|
||||
policy_id: policyId,
|
||||
stack_name: 'history-order-stack',
|
||||
service_name: null,
|
||||
container_name: 'history-order-stack-web-1',
|
||||
container_id: 'hist-001',
|
||||
action: 'skipped_cooldown',
|
||||
reason: 'second',
|
||||
success: 0,
|
||||
error: null,
|
||||
timestamp: baseTs + 1_000,
|
||||
});
|
||||
|
||||
const history = db.getAutoHealHistory(policyId);
|
||||
expect(history.length).toBe(2);
|
||||
// Most recent first
|
||||
expect(history[0].reason).toBe('second');
|
||||
expect(history[1].reason).toBe('first');
|
||||
});
|
||||
|
||||
it('getAutoHealHistory respects the limit parameter', () => {
|
||||
const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'history-limit-stack' }));
|
||||
const policyId: number = policy.id;
|
||||
const baseTs = Date.now();
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
db.recordAutoHealHistory({
|
||||
policy_id: policyId,
|
||||
stack_name: 'history-limit-stack',
|
||||
service_name: null,
|
||||
container_name: 'history-limit-stack-web-1',
|
||||
container_id: 'hist-limit',
|
||||
action: 'restarted',
|
||||
reason: `entry-${i}`,
|
||||
success: 1,
|
||||
error: null,
|
||||
timestamp: baseTs + i,
|
||||
});
|
||||
}
|
||||
|
||||
const limited = db.getAutoHealHistory(policyId, 3);
|
||||
expect(limited.length).toBe(3);
|
||||
// Should contain the 3 most recent entries (highest timestamps)
|
||||
expect(limited[0].reason).toBe('entry-9');
|
||||
expect(limited[1].reason).toBe('entry-8');
|
||||
expect(limited[2].reason).toBe('entry-7');
|
||||
});
|
||||
});
|
||||
|
||||
describe('DatabaseService - consecutive failure counters', () => {
|
||||
it('incrementConsecutiveFailures increments the counter each call', () => {
|
||||
const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'fail-inc-stack', consecutive_failures: 0 }));
|
||||
const id: number = policy.id;
|
||||
|
||||
db.incrementConsecutiveFailures(id);
|
||||
expect(db.getAutoHealPolicy(id).consecutive_failures).toBe(1);
|
||||
|
||||
db.incrementConsecutiveFailures(id);
|
||||
expect(db.getAutoHealPolicy(id).consecutive_failures).toBe(2);
|
||||
});
|
||||
|
||||
it('resetConsecutiveFailures sets the counter back to 0', () => {
|
||||
const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'fail-reset-stack', consecutive_failures: 0 }));
|
||||
const id: number = policy.id;
|
||||
|
||||
db.incrementConsecutiveFailures(id);
|
||||
db.incrementConsecutiveFailures(id);
|
||||
expect(db.getAutoHealPolicy(id).consecutive_failures).toBe(2);
|
||||
|
||||
db.resetConsecutiveFailures(id);
|
||||
expect(db.getAutoHealPolicy(id).consecutive_failures).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DatabaseService - setPolicyEnabled', () => {
|
||||
it('setPolicyEnabled(id, false) sets enabled to 0', () => {
|
||||
const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'enable-toggle-stack', enabled: 1 }));
|
||||
const id: number = policy.id;
|
||||
|
||||
db.setPolicyEnabled(id, false);
|
||||
expect(db.getAutoHealPolicy(id).enabled).toBe(0);
|
||||
});
|
||||
|
||||
it('setPolicyEnabled(id, true) sets enabled to 1', () => {
|
||||
const policy = db.addAutoHealPolicy(makePolicy({ stack_name: 'enable-on-stack', enabled: 0 }));
|
||||
const id: number = policy.id;
|
||||
|
||||
db.setPolicyEnabled(id, true);
|
||||
expect(db.getAutoHealPolicy(id).enabled).toBe(1);
|
||||
});
|
||||
|
||||
it('getAutoHealPolicies returns only enabled policies after filtering', () => {
|
||||
const enabledPolicy = db.addAutoHealPolicy(makePolicy({ stack_name: 'enabled-filter-stack', enabled: 1 }));
|
||||
const disabledPolicy = db.addAutoHealPolicy(makePolicy({ stack_name: 'disabled-filter-stack', enabled: 0 }));
|
||||
|
||||
const allPolicies = db.getAutoHealPolicies();
|
||||
const enabledIds = allPolicies.filter((p: any) => p.enabled === 1).map((p: any) => p.id);
|
||||
const disabledIds = allPolicies.filter((p: any) => p.enabled === 0).map((p: any) => p.id);
|
||||
|
||||
expect(enabledIds).toContain(enabledPolicy.id);
|
||||
expect(disabledIds).toContain(disabledPolicy.id);
|
||||
expect(enabledIds).not.toContain(disabledPolicy.id);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user