sencho/backend/src/__tests__/fleet-update-hardening.test.ts

/**
 * Hardening coverage for the fleet node self-update flow:
 *   - GET /api/fleet/update-status terminal resolution: hard timeout, early-fail,
 *     the version-change and process-restart completion signals, and the tightened
 *     offline-then-online rule (a bounce on the same version with an unchanged,
 *     known process start time must NOT be reported as completed).
 *   - The failure-class terminal transitions emit an operator-visible WARN.
 *   - POST /api/fleet/nodes/:id/update concurrency guard (409).
 *   - Authorization: both DELETE clear routes require admin.
 *   - The forced-recheck throttle on DELETE /update-status?recheck=true.
 */
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import type { RemoteMeta } from '../services/CapabilityRegistry';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
import { CacheService } from '../services/CacheService';

let tmpDir: string;
let app: import('express').Express;
let adminAuth: string;
let viewerAuth: string;
let proxyNodeId: number;
let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
let FleetUpdateTrackerService: typeof import('../services/FleetUpdateTrackerService').FleetUpdateTrackerService;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let SelfUpdateService: typeof import('../services/SelfUpdateService').default;
let UPDATE_TIMEOUT_MS: number;
let localNodeId: number;

// Recent enough to clear neither the early-fail (3 min) nor the timeout (5 min).
const RECENT_MS = 30_000;
// Past the early-fail heuristic but inside the hard timeout window.
const EARLY_FAIL_ELAPSED_MS = 240_000;

const ONLINE = (over: Partial<RemoteMeta> = {}): RemoteMeta => ({
  version: '0.83.0',
  capabilities: ['stacks', 'self-update'],
  startedAt: 1,
  updateError: null,
  online: true,
  imagePinKind: null,
  updateBlocked: false,
  imageChannel: null,
  ...over,
});

function mockMeta(meta: RemoteMeta) {
  vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(meta);
}

function mockTarget() {
  vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) =>
    id === proxyNodeId ? { apiUrl: 'http://192.168.1.99:1852', apiToken: 'proxy-token', trustedLoopback: false } : null,
  );
}

// getCompareTarget hits GitHub; pin it to a version above the node's so the
// node reads as outdated and never trips signal 4 unintentionally.
function mockCompareTargetFetch() {
  vi.spyOn(globalThis, 'fetch').mockImplementation(async () =>
    new Response(JSON.stringify({ tag_name: 'v0.99.0' }), {
      status: 200,
      headers: { 'content-type': 'application/json' },
    }),
  );
}

function setTracker(over: Partial<import('../services/FleetUpdateTrackerService').UpdateTracker>, nodeId = proxyNodeId) {
  FleetUpdateTrackerService.getInstance().set(nodeId, {
    status: 'updating',
    startedAt: Date.now() - RECENT_MS,
    previousVersion: '0.83.0',
    previousProcessStart: 1,
    wasOffline: false,
    operationKind: 'update',
    ...over,
  });
}

async function getStatus(nodeId = proxyNodeId): Promise<number | null | undefined> {
  const res = await request(app).get('/api/fleet/update-status').set('Authorization', adminAuth);
  expect(res.status).toBe(200);
  return res.body.nodes.find((n: { nodeId: number }) => n.nodeId === nodeId)?.updateStatus;
}

beforeAll(async () => {
  tmpDir = await setupTestDb();
  ({ app } = await import('../index'));
  ({ NodeRegistry } = await import('../services/NodeRegistry'));
  const trackerMod = await import('../services/FleetUpdateTrackerService');
  FleetUpdateTrackerService = trackerMod.FleetUpdateTrackerService;
  UPDATE_TIMEOUT_MS = trackerMod.UPDATE_TIMEOUT_MS;
  ({ DatabaseService } = await import('../services/DatabaseService'));
  SelfUpdateService = (await import('../services/SelfUpdateService')).default;

  const db = DatabaseService.getInstance();
  localNodeId = db.getNodes().find(n => n.type === 'local')!.id;
  proxyNodeId = db.addNode({
    name: 'proxy-hardening-test',
    type: 'remote',
    mode: 'proxy',
    compose_dir: '/tmp',
    is_default: false,
    api_url: 'http://192.168.1.99:1852',
    api_token: 'proxy-token',
  });
  db.addUser({ username: 'viewer-hardening-test', password_hash: 'unused', role: 'viewer' });

  adminAuth = `Bearer ${jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' })}`;
  viewerAuth = `Bearer ${jwt.sign({ username: 'viewer-hardening-test' }, TEST_JWT_SECRET, { expiresIn: '1m' })}`;
});

afterAll(() => cleanupTestDb(tmpDir));

afterEach(() => {
  vi.restoreAllMocks();
  const tracker = FleetUpdateTrackerService.getInstance();
  for (const [id] of tracker.entries()) tracker.delete(id);
});

describe('GET /api/fleet/update-status terminal resolution', () => {
  it('times out an in-flight tracker past the hard ceiling and warns', async () => {
    mockTarget();
    mockCompareTargetFetch();
    mockMeta(ONLINE());
    const warnSpy = vi.spyOn(console, 'warn');
    setTracker({ startedAt: Date.now() - (UPDATE_TIMEOUT_MS + 1_000) });

    expect(await getStatus()).toBe('timeout');
    expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Node update timeout'));
  });

  it('fails via the early-fail heuristic when the node stays online and unchanged, and warns', async () => {
    mockTarget();
    mockCompareTargetFetch();
    mockMeta(ONLINE()); // version unchanged, startedAt unchanged
    const warnSpy = vi.spyOn(console, 'warn');
    setTracker({ startedAt: Date.now() - EARLY_FAIL_ELAPSED_MS });

    expect(await getStatus()).toBe('failed');
    expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Node update failed'));
  });

  it('completes via signal 1 when the remote version changed, without warning', async () => {
    mockTarget();
    mockCompareTargetFetch();
    mockMeta(ONLINE({ version: '0.99.0', startedAt: 1 }));
    const warnSpy = vi.spyOn(console, 'warn');
    setTracker({ previousVersion: '0.83.0' });

    expect(await getStatus()).toBe('completed');
    expect(warnSpy).not.toHaveBeenCalledWith(expect.stringContaining('Node update'));
  });

  it('completes via signal 2 when the remote process restarted (startedAt changed)', async () => {
    mockTarget();
    mockCompareTargetFetch();
    mockMeta(ONLINE({ version: '0.83.0', startedAt: 2 })); // same version, new process start
    setTracker({ previousProcessStart: 1 });

    expect(await getStatus()).toBe('completed');
  });

  it('does NOT complete a same-version bounce when the process start time is unchanged', async () => {
    mockTarget();
    mockCompareTargetFetch();
    // Bounced offline then back, but same version AND same (known) process start:
    // the process never actually restarted, so this is a blip, not a completed update.
    mockMeta(ONLINE({ version: '0.83.0', startedAt: 1 }));
    setTracker({ wasOffline: true, previousProcessStart: 1 });

    expect(await getStatus()).toBe('updating');
  });

  it('does NOT complete when the remote version is momentarily unavailable (null) on an unchanged process', async () => {
    mockTarget();
    mockCompareTargetFetch();
    // /api/meta briefly omits or mangles the version (online, but version null)
    // with no process restart: this is not a version change and must not complete.
    mockMeta(ONLINE({ version: null, startedAt: 1 }));
    setTracker({ previousVersion: '0.83.0', previousProcessStart: 1, wasOffline: false });

    expect(await getStatus()).toBe('updating');
  });

  it('completes an offline->online bounce when the process start time is unavailable', async () => {
    mockTarget();
    mockCompareTargetFetch();
    // No startedAt reported by the remote: offline->online is the only restart
    // evidence available, so signal 3 remains a valid completion fallback.
    mockMeta(ONLINE({ version: '0.83.0', startedAt: null }));
    setTracker({ wasOffline: true, previousProcessStart: null });

    expect(await getStatus()).toBe('completed');
  });

  it('fails a local-node update via the early-fail heuristic and warns', async () => {
    // The local node reads its own version, never fetchMetaForNode; with no
    // recorded self-update error it resolves through the early-fail heuristic.
    mockCompareTargetFetch();
    vi.spyOn(SelfUpdateService.getInstance(), 'getLastError').mockReturnValue(null);
    const warnSpy = vi.spyOn(console, 'warn');
    setTracker({ startedAt: Date.now() - EARLY_FAIL_ELAPSED_MS }, localNodeId);

    expect(await getStatus(localNodeId)).toBe('failed');
    expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Node update failed'));
  });
});

describe('POST /api/fleet/nodes/:id/update concurrency', () => {
  it('returns 409 when an update is already in progress for the node', async () => {
    setTracker({ startedAt: Date.now() - RECENT_MS });

    const res = await request(app)
      .post(`/api/fleet/nodes/${proxyNodeId}/update`)
      .set('Authorization', adminAuth);

    expect(res.status).toBe(409);
    expect(res.body?.error).toMatch(/already in progress/i);
  });

  it('preserves a typed remote update failure in the response and tracker', async () => {
    mockTarget();
    mockMeta(ONLINE());
    vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
      try {
        if (new URL(String(input)).hostname === 'api.github.com') {
          return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 });
        }
      } catch {
        // Non-URL fetch inputs fall through to the remote-update mock.
      }
      return new Response(JSON.stringify({
        error: 'Hardened Build updates require a signed-in admin on that node.',
        code: 'HARDENED_REMOTE_UPDATE_UNSUPPORTED',
      }), { status: 403 });
    });

    const res = await request(app)
      .post(`/api/fleet/nodes/${proxyNodeId}/update`)
      .set('Authorization', adminAuth);

    expect(res.status).toBe(502);
    expect(res.body).toEqual({
      error: 'Hardened Build updates require a signed-in admin on that node.',
      code: 'HARDENED_REMOTE_UPDATE_UNSUPPORTED',
    });
    expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)?.code)
      .toBe('HARDENED_REMOTE_UPDATE_UNSUPPORTED');
  });
});


describe('POST /api/fleet/nodes/:id/update hardened digest pin', () => {
  it('still POSTs when updateBlocked and imageChannel is hardened', async () => {
    mockTarget();
    mockMeta(ONLINE({ updateBlocked: true, imageChannel: 'hardened', imagePinKind: 'digest' }));
    let remoteUpdateCalled = false;
    vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
      try {
        if (new URL(String(input)).hostname === 'api.github.com') {
          return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 });
        }
      } catch {
        // Non-URL fetch inputs fall through to the remote-update mock.
      }
      remoteUpdateCalled = true;
      return new Response(JSON.stringify({
        error: 'Hardened Build updates require a signed-in admin on that node.',
        code: 'HARDENED_REMOTE_UPDATE_UNSUPPORTED',
      }), { status: 403 });
    });

    const res = await request(app)
      .post(`/api/fleet/nodes/${proxyNodeId}/update`)
      .set('Authorization', adminAuth);

    expect(remoteUpdateCalled).toBe(true);
    expect(res.status).toBe(502);
    expect(res.body).toEqual({
      error: 'Hardened Build updates require a signed-in admin on that node.',
      code: 'HARDENED_REMOTE_UPDATE_UNSUPPORTED',
    });
    expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)?.code)
      .toBe('HARDENED_REMOTE_UPDATE_UNSUPPORTED');
  });
});

describe('POST /api/fleet/update-all typed failures', () => {
  it('reports remote rejections as failed instead of skipped', async () => {
    mockTarget();
    mockMeta(ONLINE());
    vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
      try {
        if (new URL(String(input)).hostname === 'api.github.com') {
          return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 });
        }
      } catch {
        // Non-URL fetch inputs fall through to the remote-update mock.
      }
      return new Response(JSON.stringify({
        error: 'Hardened Build updates require a signed-in admin on that node.',
        code: 'HARDENED_REMOTE_UPDATE_UNSUPPORTED',
      }), { status: 403 });
    });

    const res = await request(app)
      .post('/api/fleet/update-all')
      .set('Authorization', adminAuth);

    expect(res.status).toBe(202);
    expect(res.body).toEqual({
      updating: [],
      skipped: [],
      failed: [{
        nodeId: proxyNodeId,
        name: 'proxy-hardening-test',
        code: 'HARDENED_REMOTE_UPDATE_UNSUPPORTED',
        error: 'Hardened Build updates require a signed-in admin on that node.',
      }],
    });
  });
});

describe('clear-route authorization', () => {
  it('rejects DELETE /nodes/:id/update-status for a non-admin', async () => {
    const res = await request(app)
      .delete(`/api/fleet/nodes/${proxyNodeId}/update-status`)
      .set('Authorization', viewerAuth);
    expect(res.status).toBe(403);
  });

  it('rejects DELETE /update-status for a non-admin', async () => {
    const res = await request(app)
      .delete('/api/fleet/update-status?recheck=true')
      .set('Authorization', viewerAuth);
    expect(res.status).toBe(403);
  });

  it('allows DELETE /nodes/:id/update-status for an admin and clears the tracker', async () => {
    setTracker({ status: 'failed', error: 'boom', startedAt: Date.now() - RECENT_MS });
    const res = await request(app)
      .delete(`/api/fleet/nodes/${proxyNodeId}/update-status`)
      .set('Authorization', adminAuth);
    expect(res.status).toBe(204);
    expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)).toBeUndefined();
  });
});

describe('forced-recheck throttle', () => {
  it('forces the latest-version refresh once, then throttles within the cooldown', async () => {
    // Reset the module-scope throttle clock so this assertion does not depend on
    // whether an earlier test happened to force a recheck first.
    const { _resetForcedRecheckThrottleForTests } = await import('../routes/fleet');
    _resetForcedRecheckThrottleForTests();
    mockCompareTargetFetch();
    const invalidateSpy = vi.spyOn(CacheService.getInstance(), 'invalidate');

    const first = await request(app)
      .delete('/api/fleet/update-status?recheck=true')
      .set('Authorization', adminAuth);
    expect(first.status).toBe(200);
    expect(first.body.rechecked).toBe(true);
    expect(invalidateSpy).toHaveBeenCalledWith('latest-version-info');

    invalidateSpy.mockClear();

    // A terminal tracker set before the throttled call must still be cleared:
    // the cooldown gates only the upstream version refresh, not tracker cleanup.
    FleetUpdateTrackerService.getInstance().set(proxyNodeId, {
      status: 'failed', startedAt: Date.now(), previousVersion: null,
      previousProcessStart: null, wasOffline: false, resolvedAt: Date.now(), error: 'boom',
      operationKind: 'update',
    });

    const second = await request(app)
      .delete('/api/fleet/update-status?recheck=true')
      .set('Authorization', adminAuth);
    expect(second.status).toBe(200);
    expect(second.body.rechecked).toBe(false);
    expect(invalidateSpy).not.toHaveBeenCalledWith('latest-version');
    expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)).toBeUndefined();
  });
});

describe('GET /api/fleet/update-status/release-notes', () => {
  // Each case uses ?recheck=true so getLatestRelease force-invalidates the cache
  // and fetches fresh, keeping the assertion independent of prior cache state.
  it('binds the returned notes to the release version (normalized tag_name)', async () => {
    vi.spyOn(globalThis, 'fetch').mockImplementation(async () =>
      new Response(JSON.stringify({
        tag_name: 'v0.93.0',
        body: '## Notes for 0.93.0',
        html_url: 'https://github.com/studio-saelix/sencho/releases/tag/v0.93.0',
      }), { status: 200, headers: { 'content-type': 'application/json' } }),
    );
    const res = await request(app)
      .get('/api/fleet/update-status/release-notes?recheck=true')
      .set('Authorization', adminAuth);
    expect(res.status).toBe(200);
    expect(res.body.version).toBe('0.93.0');
    expect(res.body.releaseNotes).toBe('## Notes for 0.93.0');
    expect(res.body.htmlUrl).toContain('v0.93.0');
  });

  it('returns null fields when the upstream release lookup fails', async () => {
    vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('', { status: 500 }));
    const res = await request(app)
      .get('/api/fleet/update-status/release-notes?recheck=true')
      .set('Authorization', adminAuth);
    expect(res.status).toBe(200);
    expect(res.body.version).toBeNull();
    expect(res.body.releaseNotes).toBeNull();
    expect(res.body.htmlUrl).toBeNull();
  });
});

describe('compose reapply status and concurrency', () => {
  it('exposes canReapplyCompose for local when SelfUpdateService is available', async () => {
    vi.spyOn(SelfUpdateService.getInstance(), 'isAvailable').mockReturnValue(true);
    mockCompareTargetFetch();
    const res = await request(app).get('/api/fleet/update-status').set('Authorization', adminAuth);
    expect(res.status).toBe(200);
    const local = res.body.nodes.find((n: { type: string }) => n.type === 'local');
    expect(local.canReapplyCompose).toBe(true);
  });

  it('sets canReapplyCompose false for a remote without self-update capability', async () => {
    mockMeta(ONLINE({ capabilities: ['stacks'] }));
    mockCompareTargetFetch();
    const res = await request(app).get('/api/fleet/update-status').set('Authorization', adminAuth);
    expect(res.status).toBe(200);
    const remote = res.body.nodes.find((n: { nodeId: number }) => n.nodeId === proxyNodeId);
    expect(remote.canReapplyCompose).toBe(false);
  });

  it('resolves a reapply tracker via startedAt change without requiring a version bump', async () => {
    setTracker({
      operationKind: 'reapply_configuration',
      previousVersion: '0.83.0',
      previousProcessStart: 1,
      startedAt: Date.now() - RECENT_MS,
    });
    mockMeta(ONLINE({ version: '0.83.0', startedAt: 2 }));
    mockCompareTargetFetch();
    expect(await getStatus()).toBe('completed');
  });

  it('does not complete a reapply tracker via signal 4 when version is already current', async () => {
    setTracker({
      operationKind: 'reapply_configuration',
      previousVersion: '0.99.0',
      previousProcessStart: 1,
      startedAt: Date.now() - 20_000,
    });
    // Node already at compare target; signal 4 would false-complete an update,
    // but must not for reapply while startedAt is unchanged.
    mockMeta(ONLINE({ version: '0.99.0', startedAt: 1 }));
    mockCompareTargetFetch();
    expect(await getStatus()).toBe('updating');
  });

  it('returns 409 when reapply is requested while an update tracker is in flight', async () => {
    setTracker({ operationKind: 'update' });
    mockTarget();
    const res = await request(app)
      .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
      .set('Authorization', adminAuth);
    expect(res.status).toBe(409);
    expect(res.body?.error).toMatch(/already in progress/i);
  });

  it('dispatches remote reapply to /api/system/reapply-compose without updateBlocked gating', async () => {
    mockTarget();
    mockMeta(ONLINE({ updateBlocked: true, imagePinKind: 'digest', imageChannel: 'community' }));
    let reapplyUrl: string | null = null;
    vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
      const url = String(input);
      try {
        if (new URL(url).hostname === 'api.github.com') {
          return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 });
        }
      } catch { /* fall through */ }
      reapplyUrl = url;
      return new Response(JSON.stringify({ message: 'ok' }), { status: 202 });
    });

    const res = await request(app)
      .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
      .set('Authorization', adminAuth);

    expect(res.status).toBe(202);
    expect(reapplyUrl).toContain('/api/system/reapply-compose');
    expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)?.operationKind)
      .toBe('reapply_configuration');
  });

  it('reserves the tracker before remote dispatch so a concurrent reapply gets 409 without overwriting success', async () => {
    mockTarget();
    // Hold meta so the first request sits in the dispatch set before the
    // pollable tracker exists; the second must still 409 on that lock.
    let releaseMeta!: (value: RemoteMeta) => void;
    const metaHeld = new Promise<RemoteMeta>((resolve) => { releaseMeta = resolve; });
    vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockImplementation(async () => metaHeld);

    let releaseRemote!: (value: Response) => void;
    const remoteHeld = new Promise<Response>((resolve) => { releaseRemote = resolve; });
    vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
      const url = String(input);
      try {
        if (new URL(url).hostname === 'api.github.com') {
          return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 });
        }
      } catch { /* fall through */ }
      if (url.includes('/api/system/reapply-compose')) {
        return remoteHeld;
      }
      return new Response('{}', { status: 200 });
    });

    // Supertest is lazy until the thenable is consumed; start the request now.
    const firstPromise = request(app)
      .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
      .set('Authorization', adminAuth)
      .then((res) => res);

    await vi.waitFor(() => {
      expect(NodeRegistry.getInstance().fetchMetaForNode).toHaveBeenCalled();
    });

    const secondDuringMeta = await request(app)
      .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
      .set('Authorization', adminAuth);
    expect(secondDuringMeta.status).toBe(409);
    expect(secondDuringMeta.body?.error).toMatch(/already in progress/i);

    releaseMeta(ONLINE());

    await vi.waitFor(() => {
      expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)?.status).toBe('updating');
    });

    const secondDuringPost = await request(app)
      .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
      .set('Authorization', adminAuth);
    expect(secondDuringPost.status).toBe(409);

    releaseRemote(new Response(JSON.stringify({ message: 'ok' }), { status: 202 }));
    const first = await firstPromise;
    expect(first.status).toBe(202);
    const tracker = FleetUpdateTrackerService.getInstance().get(proxyNodeId);
    expect(tracker?.status).toBe('updating');
    expect(tracker?.operationKind).toBe('reapply_configuration');
    expect(tracker?.previousVersion).toBe('0.83.0');
    expect(tracker?.previousProcessStart).toBe(1);
    expect(tracker?.error).toBeUndefined();
  });

  it('marks a reserved remote reapply as failed when the peer rejects, without leaving a false updating row for a second request', async () => {
    mockTarget();
    mockMeta(ONLINE());
    vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => {
      const url = String(input);
      try {
        if (new URL(url).hostname === 'api.github.com') {
          return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 });
        }
      } catch { /* fall through */ }
      return new Response(JSON.stringify({
        error: 'An image operation is already in progress.',
        code: 'IMAGE_OPERATION_IN_FLIGHT',
      }), { status: 409 });
    });

    const res = await request(app)
      .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`)
      .set('Authorization', adminAuth);
    expect(res.status).toBe(502);
    const tracker = FleetUpdateTrackerService.getInstance().get(proxyNodeId);
    expect(tracker?.status).toBe('failed');
    expect(tracker?.error).toMatch(/already in progress/i);
  });
});