mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-24 17:36:42 +00:00
feat(auto-heal): restart crashed containers and harden the heal loop (#1258)
* feat(auto-heal): restart crashed containers and harden the heal loop Auto-Heal now restarts containers that crash (non-zero exit) and stay down past the policy threshold, in addition to those that fail their Docker healthcheck. Crash detection reuses the container event classifier so a container that exits cleanly or that an operator stopped is never restarted; only classified crashes set the heal signal. Also hardens the existing loop: - A paid controlling instance refreshes proxied remotes' entitlement on a background interval so a remote node's policies keep evaluating between operator visits instead of lapsing a few minutes after the sheet was last opened. A node that stays unreachable surfaces a warning. - Overlapping policies (all-services plus a service-specific one) restart a given container at most once per evaluation pass, so the hourly cap holds. - A failed restart now counts toward the cooldown and hourly cap, so a broken setup is retried on the cooldown interval rather than every poll. - Diagnostic logging behind developer mode for evaluation, heal decisions, timing, and lease refresh. * docs(auto-heal): document crash healing and refresh troubleshooting Cover the two heal conditions (unhealthy and crashed), note that clean exits and operator stops are never restarted and that crash healing acts on crashes observed while Sencho is running, and update the troubleshooting and tab visibility entries accordingly. * fix(auto-heal): close stale crash-signal race and harden lease refresh A crash signal could outlive the crash it described. The exit classifier is deferred 500ms, so an immediate restart could let it stamp the crash marker after the container was already running, and a later clean or operator-initiated exit did not clear it; the next poll could then restart a container that had exited cleanly. Now a clean or intentional exit always clears the marker, a die that a start has superseded is not stamped, and the die's own time is captured at arrival rather than at the deferred classification so the supersede check is accurate. Also: - Crash state survives the event service's idle-prune window, so crash healing works for any configured threshold rather than only short ones. - An exited or dead container is matched before any health-text parsing, so it can never fall into the healthcheck path. - A remote with no reachable proxy target counts toward the lease-refresh failure warning instead of being silently skipped.
This commit is contained in:
Anso
@@ -8,6 +8,7 @@ let DockerController: typeof import('../services/DockerController').default;
|
||||
let DockerEventManager: typeof import('../services/DockerEventManager').DockerEventManager;
|
||||
let LicenseService: typeof import('../services/LicenseService').LicenseService;
|
||||
let NotificationService: typeof import('../services/NotificationService').NotificationService;
|
||||
let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
|
||||
|
||||
function makePolicy(
|
||||
db: import('../services/DatabaseService').DatabaseService,
|
||||
@@ -46,6 +47,7 @@ beforeAll(async () => {
|
||||
({ DockerEventManager } = await import('../services/DockerEventManager'));
|
||||
({ LicenseService } = await import('../services/LicenseService'));
|
||||
({ NotificationService } = await import('../services/NotificationService'));
|
||||
({ NodeRegistry } = await import('../services/NodeRegistry'));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -77,9 +79,9 @@ describe('AutoHealService.evaluate', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
makePolicy(db, { proxy_entitled_until: Date.now() + 60_000 });
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const getRunningContainers = vi.fn().mockResolvedValue([]);
|
||||
const getAllContainers = vi.fn().mockResolvedValue([]);
|
||||
const getInstance = vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getRunningContainers,
|
||||
getAllContainers,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
|
||||
await resetAutoHealSingleton().evaluate();
|
||||
@@ -93,7 +95,7 @@ describe('AutoHealService.evaluate', () => {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
const restartContainer = vi.fn().mockResolvedValue(undefined);
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getRunningContainers: vi.fn().mockResolvedValue([{
|
||||
getAllContainers: vi.fn().mockResolvedValue([{
|
||||
Id: 'container-1',
|
||||
Names: ['/heal-stack-web-1'],
|
||||
Labels: {
|
||||
@@ -118,6 +120,190 @@ describe('AutoHealService.evaluate', () => {
|
||||
expect(restartContainer).toHaveBeenCalledWith('container-1');
|
||||
const history = db.getAutoHealHistory(policy.id!);
|
||||
expect(history[0]).toMatchObject({ action: 'restarted', success: 1 });
|
||||
expect(history[0].reason).toContain('unhealthy');
|
||||
});
|
||||
|
||||
it('heals a crashed container that stayed down past the threshold', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const policy = makePolicy(db);
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
const restartContainer = vi.fn().mockResolvedValue(undefined);
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getAllContainers: vi.fn().mockResolvedValue([{
|
||||
Id: 'crash-1',
|
||||
Names: ['/heal-stack-worker-1'],
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'heal-stack',
|
||||
'com.docker.compose.service': 'worker',
|
||||
},
|
||||
State: 'exited',
|
||||
Status: 'Exited (1) 2 minutes ago',
|
||||
}]),
|
||||
restartContainer,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
|
||||
getContainerState: () => ({ id: 'crash-1', crashedAt: Date.now() - 2 * 60_000 }),
|
||||
} as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
|
||||
|
||||
await resetAutoHealSingleton().evaluate();
|
||||
|
||||
expect(restartContainer).toHaveBeenCalledWith('crash-1');
|
||||
const history = db.getAutoHealHistory(policy.id!);
|
||||
expect(history[0]).toMatchObject({ action: 'restarted', success: 1 });
|
||||
expect(history[0].reason).toContain('crashed');
|
||||
});
|
||||
|
||||
it('heals a container in the "dead" state when it crashed', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
makePolicy(db);
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
const restartContainer = vi.fn().mockResolvedValue(undefined);
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getAllContainers: vi.fn().mockResolvedValue([{
|
||||
Id: 'dead-1',
|
||||
Names: ['/heal-stack-worker-1'],
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'heal-stack',
|
||||
'com.docker.compose.service': 'worker',
|
||||
},
|
||||
State: 'dead',
|
||||
Status: 'Dead',
|
||||
}]),
|
||||
restartContainer,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
|
||||
getContainerState: () => ({ id: 'dead-1', crashedAt: Date.now() - 2 * 60_000 }),
|
||||
} as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
|
||||
|
||||
await resetAutoHealSingleton().evaluate();
|
||||
|
||||
expect(restartContainer).toHaveBeenCalledWith('dead-1');
|
||||
});
|
||||
|
||||
it('does not heal a recovered container that still carries a stale crash signal', async () => {
|
||||
// Container crashed earlier (crashedAt set) but is running again now; the
|
||||
// live running state must win so we do not restart a healthy container.
|
||||
const db = DatabaseService.getInstance();
|
||||
makePolicy(db);
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
const restartContainer = vi.fn().mockResolvedValue(undefined);
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getAllContainers: vi.fn().mockResolvedValue([{
|
||||
Id: 'recovered-1',
|
||||
Names: ['/heal-stack-worker-1'],
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'heal-stack',
|
||||
'com.docker.compose.service': 'worker',
|
||||
},
|
||||
State: 'running',
|
||||
Status: 'Up 30 seconds',
|
||||
}]),
|
||||
restartContainer,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
|
||||
getContainerState: () => ({ id: 'recovered-1', crashedAt: Date.now() - 5 * 60_000 }),
|
||||
} as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
|
||||
|
||||
await resetAutoHealSingleton().evaluate();
|
||||
|
||||
expect(restartContainer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not heal an exited container with no crash signal (operator stop / clean exit)', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
makePolicy(db);
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
const restartContainer = vi.fn().mockResolvedValue(undefined);
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getAllContainers: vi.fn().mockResolvedValue([{
|
||||
Id: 'stopped-1',
|
||||
Names: ['/heal-stack-worker-1'],
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'heal-stack',
|
||||
'com.docker.compose.service': 'worker',
|
||||
},
|
||||
State: 'exited',
|
||||
Status: 'Exited (0) 2 minutes ago',
|
||||
}]),
|
||||
restartContainer,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
|
||||
getContainerState: () => ({ id: 'stopped-1' }), // no crashedAt: not a crash
|
||||
} as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
|
||||
|
||||
await resetAutoHealSingleton().evaluate();
|
||||
|
||||
expect(restartContainer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('restarts a container at most once per pass when policies overlap', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
makePolicy(db); // all services
|
||||
makePolicy(db, { service_name: 'worker' }); // service-specific, same container
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
const restartContainer = vi.fn().mockResolvedValue(undefined);
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getAllContainers: vi.fn().mockResolvedValue([{
|
||||
Id: 'dup-1',
|
||||
Names: ['/heal-stack-worker-1'],
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'heal-stack',
|
||||
'com.docker.compose.service': 'worker',
|
||||
},
|
||||
State: 'running',
|
||||
Status: 'Up 5 minutes (unhealthy)',
|
||||
}]),
|
||||
restartContainer,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
|
||||
getContainerState: () => undefined,
|
||||
} as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
|
||||
|
||||
const service = resetAutoHealSingleton();
|
||||
(service as unknown as { observedUnhealthySince: Map<string, number> }).observedUnhealthySince
|
||||
.set('1:dup-1', Date.now() - 2 * 60_000);
|
||||
|
||||
await service.evaluate();
|
||||
|
||||
expect(restartContainer).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('counts a failed restart toward cooldown and the hourly cap', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const policy = makePolicy(db);
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
const restartContainer = vi.fn().mockRejectedValue(new Error('no such container'));
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getAllContainers: vi.fn().mockResolvedValue([{
|
||||
Id: 'fail-1',
|
||||
Names: ['/heal-stack-web-1'],
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'heal-stack',
|
||||
'com.docker.compose.service': 'web',
|
||||
},
|
||||
State: 'running',
|
||||
Status: 'Up 1 minute (unhealthy)',
|
||||
}]),
|
||||
restartContainer,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
vi.spyOn(DockerEventManager.getInstance(), 'getService').mockReturnValue({
|
||||
getContainerState: () => undefined,
|
||||
} as unknown as ReturnType<ReturnType<typeof DockerEventManager.getInstance>['getService']>);
|
||||
|
||||
const service = resetAutoHealSingleton();
|
||||
(service as unknown as { observedUnhealthySince: Map<string, number> }).observedUnhealthySince
|
||||
.set('1:fail-1', Date.now() - 2 * 60_000);
|
||||
|
||||
await service.evaluate();
|
||||
|
||||
expect(restartContainer).toHaveBeenCalledTimes(1);
|
||||
const updated = db.getAutoHealPolicy(policy.id!);
|
||||
expect(updated!.last_fired_at).toBeGreaterThan(0);
|
||||
expect(updated!.consecutive_failures).toBe(1);
|
||||
const ts = (service as unknown as { restartTimestamps: Map<string, number[]> }).restartTimestamps.get('1:fail-1');
|
||||
expect(ts).toHaveLength(1);
|
||||
const history = db.getAutoHealHistory(policy.id!);
|
||||
expect(history[0]).toMatchObject({ action: 'failed', success: 0 });
|
||||
});
|
||||
|
||||
it('records Docker unavailable history once per throttle window when listing containers fails', async () => {
|
||||
@@ -125,7 +311,7 @@ describe('AutoHealService.evaluate', () => {
|
||||
const policy = makePolicy(db);
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getRunningContainers: vi.fn().mockRejectedValue(new Error('permission denied')),
|
||||
getAllContainers: vi.fn().mockRejectedValue(new Error('permission denied')),
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
|
||||
const service = resetAutoHealSingleton();
|
||||
@@ -153,9 +339,9 @@ describe('AutoHealService.evaluate', () => {
|
||||
});
|
||||
makePolicy(db, { node_id: secondNodeId, stack_name: 'second-stack' });
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
const getRunningContainers = vi.fn().mockResolvedValue([]);
|
||||
const getAllContainers = vi.fn().mockResolvedValue([]);
|
||||
const getInstance = vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getRunningContainers,
|
||||
getAllContainers,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
|
||||
await resetAutoHealSingleton().evaluate();
|
||||
@@ -176,7 +362,7 @@ describe('AutoHealService.evaluate', () => {
|
||||
});
|
||||
const policy = makePolicy(db, { node_id: secondNodeId, stack_name: 'second-rate-stack' });
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getRunningContainers: vi.fn().mockResolvedValue([]),
|
||||
getAllContainers: vi.fn().mockResolvedValue([]),
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
|
||||
const service = resetAutoHealSingleton();
|
||||
@@ -195,7 +381,7 @@ describe('AutoHealService.evaluate', () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const policy = makePolicy(db, { stack_name: 'cleanup-stack' });
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getRunningContainers: vi.fn().mockResolvedValue([]),
|
||||
getAllContainers: vi.fn().mockResolvedValue([]),
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
|
||||
const service = resetAutoHealSingleton();
|
||||
@@ -214,6 +400,120 @@ describe('AutoHealService.evaluate', () => {
|
||||
expect(internals.historyTimestamps.size).toBe(0);
|
||||
});
|
||||
|
||||
it('refreshes proxied remote leases from a paid controlling instance', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.addNode({
|
||||
name: 'lease-remote',
|
||||
type: 'remote',
|
||||
compose_dir: '',
|
||||
is_default: false,
|
||||
api_url: 'http://remote:1852',
|
||||
api_token: 'tok',
|
||||
});
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getProxyHeaders').mockReturnValue(
|
||||
{ tier: 'paid', variant: 'admiral' } as ReturnType<ReturnType<typeof LicenseService.getInstance>['getProxyHeaders']>,
|
||||
);
|
||||
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
|
||||
apiUrl: 'http://remote:1852',
|
||||
apiToken: 'tok',
|
||||
});
|
||||
const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ ok: true } as Response);
|
||||
|
||||
const service = resetAutoHealSingleton();
|
||||
await (service as unknown as { refreshRemoteLeases: () => Promise<void> }).refreshRemoteLeases();
|
||||
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
||||
const [url, opts] = fetchSpy.mock.calls[0];
|
||||
expect(String(url)).toContain('/api/auto-heal/policies');
|
||||
expect((opts as RequestInit).headers).toMatchObject({ 'x-sencho-tier': 'paid' });
|
||||
});
|
||||
|
||||
it('does not refresh remote leases when the controlling instance is not paid', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.addNode({
|
||||
name: 'lease-remote-community',
|
||||
type: 'remote',
|
||||
compose_dir: '',
|
||||
is_default: false,
|
||||
api_url: 'http://remote2:1852',
|
||||
api_token: 'tok2',
|
||||
});
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ ok: true } as Response);
|
||||
|
||||
const service = resetAutoHealSingleton();
|
||||
await (service as unknown as { refreshRemoteLeases: () => Promise<void> }).refreshRemoteLeases();
|
||||
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps refreshing other remotes when one node is unreachable', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.addNode({
|
||||
name: 'good-remote',
|
||||
type: 'remote',
|
||||
compose_dir: '',
|
||||
is_default: false,
|
||||
api_url: 'http://good-host:1852',
|
||||
api_token: 'tok',
|
||||
});
|
||||
db.addNode({
|
||||
name: 'unreachable-remote',
|
||||
type: 'remote',
|
||||
compose_dir: '',
|
||||
is_default: false,
|
||||
api_url: 'http://bad-host:1852',
|
||||
api_token: 'tok',
|
||||
});
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getProxyHeaders').mockReturnValue(
|
||||
{ tier: 'paid', variant: 'admiral' } as ReturnType<ReturnType<typeof LicenseService.getInstance>['getProxyHeaders']>,
|
||||
);
|
||||
const fetchSpy = vi.spyOn(global, 'fetch').mockImplementation(((input: unknown) => {
|
||||
const url = String(input);
|
||||
if (url.includes('bad-host')) return Promise.reject(new Error('ECONNREFUSED'));
|
||||
return Promise.resolve({ ok: true } as Response);
|
||||
}) as typeof fetch);
|
||||
|
||||
const service = resetAutoHealSingleton();
|
||||
// One node rejecting must not throw or block the others.
|
||||
await expect(
|
||||
(service as unknown as { refreshRemoteLeases: () => Promise<void> }).refreshRemoteLeases(),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
const calledUrls = fetchSpy.mock.calls.map(c => String(c[0]));
|
||||
expect(calledUrls.some(u => u.includes('good-host'))).toBe(true);
|
||||
expect(calledUrls.some(u => u.includes('bad-host'))).toBe(true);
|
||||
});
|
||||
|
||||
it('warns after repeated lease refreshes find no reachable proxy target', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
db.addNode({
|
||||
name: 'no-target-remote',
|
||||
type: 'remote',
|
||||
compose_dir: '',
|
||||
is_default: false,
|
||||
api_url: '',
|
||||
api_token: '',
|
||||
});
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getProxyHeaders').mockReturnValue(
|
||||
{ tier: 'paid', variant: 'admiral' } as ReturnType<ReturnType<typeof LicenseService.getInstance>['getProxyHeaders']>,
|
||||
);
|
||||
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue(null);
|
||||
const fetchSpy = vi.spyOn(global, 'fetch');
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
|
||||
const svc = resetAutoHealSingleton() as unknown as { refreshRemoteLeases: () => Promise<void> };
|
||||
await svc.refreshRemoteLeases();
|
||||
await svc.refreshRemoteLeases();
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
await svc.refreshRemoteLeases(); // third consecutive failure crosses the threshold
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not create duplicate timers when start is called twice', () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
@@ -224,9 +524,14 @@ describe('AutoHealService.evaluate', () => {
|
||||
service.start();
|
||||
service.start();
|
||||
|
||||
// One deferred-first-tick timeout, regardless of how many start() calls.
|
||||
expect(setTimeoutSpy).toHaveBeenCalledTimes(1);
|
||||
vi.advanceTimersByTime(10_000);
|
||||
// The lease-refresh interval is created immediately; the eval interval
|
||||
// only after the initial delay fires. Calling start() twice must not
|
||||
// duplicate either.
|
||||
expect(setIntervalSpy).toHaveBeenCalledTimes(1);
|
||||
vi.advanceTimersByTime(10_000);
|
||||
expect(setIntervalSpy).toHaveBeenCalledTimes(2);
|
||||
service.stop();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
|
||||
Reference in New Issue
Block a user