const { fetchDiscoveryData, fetchMetricsData, fetchPbsData } = require('../dataFetcher'); // Don't require the real apiClients, we will mock it // const { initializeApiClients } = require('../apiClients'); // Mock the modules used by dataFetcher jest.mock('axios'); // Keep this in case axios is used directly anywhere unexpected jest.mock('../pbsUtils', () => ({ // Ensure processPbsTasks returns the expected structure processPbsTasks: jest.fn().mockReturnValue({ backupTasks: [], verifyTasks: [], gcTasks: [] }), })); jest.mock('../apiClients'); // <-- MOCK apiClients module // --- REMOVE Mock for fetchPbsData within dataFetcher --- // jest.mock('../dataFetcher', ...); // --- END REMOVE --- // Import the mocked version AFTER mocking it const { initializeApiClients } = require('../apiClients'); process.env.NODE_ENV = 'test'; // Setup and teardown for console spies beforeEach(() => { // Create spies before each test jest.spyOn(console, 'log').mockImplementation(() => {}); jest.spyOn(console, 'warn').mockImplementation(() => {}); jest.spyOn(console, 'error').mockImplementation(() => {}); }); afterEach(() => { // Restore console after each test jest.restoreAllMocks(); // More comprehensive way to restore all spies/mocks }); describe('Data Fetcher', () => { // --- Declare variables used across tests/hooks --- let originalEnv; // <--- Declare here let mockPveClientInstance; let mockPveApiClient; let mockPbsClientInstance; let mockPbsApiClient; // --- End declare vars --- // Helper to set up a basic PBS client mock (MOVED TO OUTER SCOPE) const setupMockPbsClient = (id, configOverrides = {}, clientMocks = {}) => { mockPbsClientInstance = { get: jest.fn(), ...clientMocks // Allow overriding .get or adding other methods }; // Use the mockPbsApiClient defined in the outer scope mockPbsApiClient[id] = { client: mockPbsClientInstance, config: { id: `${id}_config_id`, name: `PBS Instance ${id}`, host: `${id}.pbs.example.com`, // Add other default config properties as needed ...configOverrides } }; return mockPbsClientInstance; // Return the mock instance for further configuration }; beforeEach(() => { // Store environment (assign to variable declared above) originalEnv = { ...process.env }; // Reset the mocked initializeApiClients function and other mocks jest.clearAllMocks(); // Define the *default* return value for the mocked initializer // Tests can override this if needed mockPveClientInstance = { get: jest.fn() }; mockPveApiClient = { primary: { client: mockPveClientInstance, config: { /* ... */ } } }; mockPbsClientInstance = { get: jest.fn() }; mockPbsApiClient = {}; initializeApiClients.mockResolvedValue({ apiClients: mockPveApiClient, pbsApiClients: mockPbsApiClient }); // --- Remove console mocks --- // jest.spyOn(console, 'warn').mockImplementation(() => {}); // jest.spyOn(console, 'log').mockImplementation(() => {}); // jest.spyOn(console, 'error').mockImplementation(() => {}); }); afterEach(() => { // Restore environment (can now access originalEnv) const currentEnvKeys = Object.keys(process.env); currentEnvKeys.forEach(key => delete process.env[key]); Object.keys(originalEnv).forEach(key => { process.env[key] = originalEnv[key]; }); // --- Remove console restore --- // jest.restoreAllMocks(); }); describe('fetchDiscoveryData', () => { test('should return empty structure when no PVE clients configured', async () => { // Arrange const mockPbsFunction = jest.fn().mockResolvedValue([]); // Mock PBS part as well const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); // Act const result = await fetchDiscoveryData({}, mockPbsApiClient, mockPbsFunction); // Pass empty PVE clients // Assert expect(result.nodes).toEqual([]); expect(result.vms).toEqual([]); expect(result.containers).toEqual([]); expect(result.pbs).toEqual([]); // PBS fetch should still run expect(consoleLogSpy).toHaveBeenCalledWith("[DataFetcher] Discovery cycle completed. Found: 0 PVE nodes, 0 VMs, 0 CTs, 0 PBS instances."); expect(mockPbsFunction).toHaveBeenCalled(); // Ensure PBS was still called consoleLogSpy.mockRestore(); }); test('should fetch PVE data correctly (1 node, 1 VM, 1 CT)', async () => { // Arrange: Configure the mock get method on the client instance const nodeName = 'mock-node'; const vmId = 100; const ctId = 101; const endpointId = 'primary'; // Assuming the default mock client is 'primary' // Configure mockPveClientInstance.get mockPveClientInstance.get // Calls within fetchDataForPveEndpoint for 'primary' .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'test-cluster' }] } }) // 1. /cluster/status (for endpoint 'primary') .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online' }] } }) // 2. /nodes (for endpoint 'primary' to get standaloneNodeName if cluster nodes <=1) .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online', maxcpu: 4, maxmem: 8 * 1024**3 , id: `node/${nodeName}` }] } }) // 3. /nodes (main call for endpoint 'primary' to get node list) // Calls within fetchDataForNode for 'mock-node' (the single node from the call above) .mockResolvedValueOnce({ data: { data: { cpu: 0.1, mem: 2 * 1024**3, rootfs: { total: 100*1024**3, used: 20*1024**3 }, uptime: 12345 } } }) // 4. /nodes/mock-node/status .mockResolvedValueOnce({ data: { data: [ { storage: 'local-lvm', type: 'lvmthin', content: 'images,rootdir', total: 500*1024**3, used: 150*1024**3 } ] } }) // 5. /nodes/mock-node/storage .mockResolvedValueOnce({ data: { data: [ { vmid: vmId, name: 'test-vm', status: 'running', cpu: 0.5, mem: 1 * 1024**3, maxmem: 2 * 1024**3, maxdisk: 32*1024**3 } ] } }) // 6. /nodes/mock-node/qemu .mockResolvedValueOnce({ data: { data: [ { vmid: ctId, name: 'test-ct', status: 'running', cpu: 0.2, mem: 512 * 1024**2, maxmem: 1 * 1024**3, maxdisk: 8*1024**3 } ] } }); // 7. /nodes/mock-node/lxc // Act: Call function with the clients provided by the (mocked) default setup const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert expect(mockPveClientInstance.get).toHaveBeenCalledTimes(7); // Updated to 7 expect(mockPveClientInstance.get).toHaveBeenNthCalledWith(1, '/cluster/status'); expect(mockPveClientInstance.get).toHaveBeenNthCalledWith(2, '/nodes'); // For standaloneNodeName expect(mockPveClientInstance.get).toHaveBeenNthCalledWith(3, '/nodes'); // Main nodes call expect(mockPveClientInstance.get).toHaveBeenNthCalledWith(4, `/nodes/${nodeName}/status`); expect(mockPveClientInstance.get).toHaveBeenNthCalledWith(5, `/nodes/${nodeName}/storage`); expect(mockPveClientInstance.get).toHaveBeenNthCalledWith(6, `/nodes/${nodeName}/qemu`); expect(mockPveClientInstance.get).toHaveBeenNthCalledWith(7, `/nodes/${nodeName}/lxc`); // Assert Nodes expect(result.nodes).toHaveLength(1); expect(result.nodes[0]).toMatchObject({ node: nodeName, status: 'online', // Should be updated by status call maxcpu: 4, maxmem: 8 * 1024**3, id: `${endpointId}-${nodeName}`, // Check constructed ID endpointId: endpointId, cpu: 0.1, mem: 2 * 1024**3, disk: 20 * 1024**3, maxdisk: 100 * 1024**3, uptime: 12345, }); expect(result.nodes[0].storage).toHaveLength(1); expect(result.nodes[0].storage[0]).toMatchObject({ storage: 'local-lvm', type: 'lvmthin' }); // Assert VMs expect(result.vms).toHaveLength(1); expect(result.vms[0]).toMatchObject({ vmid: vmId, name: 'test-vm', status: 'running', node: nodeName, endpointId: endpointId, type: 'qemu', id: `${endpointId}-${nodeName}-${vmId}` // Check constructed ID }); // Assert Containers expect(result.containers).toHaveLength(1); expect(result.containers[0]).toMatchObject({ vmid: ctId, name: 'test-ct', status: 'running', node: nodeName, endpointId: endpointId, type: 'lxc', id: `${endpointId}-${nodeName}-${ctId}` // Check constructed ID }); // Assert PBS (should be empty) expect(result.pbs).toBeDefined(); expect(result.pbs).toHaveLength(0); }); test('should handle invalid format for node storage response', async () => { // Arrange: Use default mock client const nodeName = 'node-bad-storage-format'; const endpointId = 'primary'; const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); mockPveClientInstance.get // Endpoint level calls .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'test-cluster' }] } }) // 1. /cluster/status .mockResolvedValueOnce({ data: { data: [{ node: nodeName }] } }) // 2. /nodes (for standaloneNodeName) .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online' }] } }) // 3. /nodes (main node list for endpoint) // Node level calls for nodeName .mockResolvedValueOnce({ data: { data: { cpu: 0.1, uptime: 10 } } }) // 4. /nodes/${nodeName}/status .mockResolvedValueOnce({ data: { data: { not_an_array: true } } }) // 5. /nodes/${nodeName}/storage (INVALID) .mockResolvedValueOnce({ data: { data: [] } }) // 6. /nodes/${nodeName}/qemu .mockResolvedValueOnce({ data: { data: [] } }); // 7. /nodes/${nodeName}/lxc // Act const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining(`[DataFetcher - ${endpointId}-${nodeName}] Node storage data is not an array as expected.`) ); // Ensure node was still processed and added (just without storage) expect(result.nodes).toHaveLength(1); expect(result.nodes[0].node).toBe(nodeName); expect(result.nodes[0].storage).toEqual([]); // Should default to empty array expect(result.vms).toHaveLength(0); expect(result.containers).toHaveLength(0); consoleWarnSpy.mockRestore(); }); test('should handle missing or invalid data.data for a node resource', async () => { // Arrange: Use default mock client and a specific node name const nodeName = 'node-missing-data-data'; // Can be the same or different, impact is on the mock const endpointId = 'primary'; const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); mockPveClientInstance.get // Endpoint level calls .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'test-cluster' }] } }) // 1. /cluster/status .mockResolvedValueOnce({ data: { data: [{ node: nodeName }] } }) // 2. /nodes (for standaloneNodeName) .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online' }] } }) // 3. /nodes (main node list for endpoint) // Node level calls for nodeName .mockResolvedValueOnce({ data: { data: null } }) // 4. /nodes/${nodeName}/status (INVALID data.data) .mockResolvedValueOnce({ data: { data: [] } }) // 5. /nodes/${nodeName}/storage .mockResolvedValueOnce({ data: { data: [] } }) // 6. /nodes/${nodeName}/qemu .mockResolvedValueOnce({ data: { data: [] } }); // 7. /nodes/${nodeName}/lxc // Act const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert // Verify the warning was logged by fetchNodeResource expect(consoleWarnSpy).toHaveBeenCalledWith( `[DataFetcher - ${endpointId}-${nodeName}] Node status data missing or invalid format.` ); // Verify that the node was still processed but status data is default (null/0) expect(result.nodes).toHaveLength(1); expect(result.nodes[0].node).toBe(nodeName); expect(result.nodes[0].cpu).toBeNull(); expect(result.nodes[0].uptime).toBe(0); expect(result.nodes[0].storage).toEqual([]); expect(result.vms).toHaveLength(0); expect(result.containers).toHaveLength(0); consoleWarnSpy.mockRestore(); }); test('should handle API error when fetching /nodes for an endpoint', async () => { const mockPveClientInstance1 = { get: jest.fn() }; const mockPveClientInstance2 = { get: jest.fn() }; // Separate instance for pve2 const mockClients = { // Custom clients for this test pve1: { client: mockPveClientInstance1, config: { id: 'pve1', name: 'PVE1 Endpoint' } }, pve2: { client: mockPveClientInstance2, config: { id: 'pve2', name: 'PVE2 Endpoint' } }, }; const nodesError = new Error('Network Error on PVE1'); const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); // Configure mockPveClientInstance1 (fails on main /nodes call) mockPveClientInstance1.get .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'pve1-cluster' }] } }) // 1. /cluster/status .mockResolvedValueOnce({ data: { data: [{ node: 'pve1-node-temp' }] } }) // 2. /nodes (standalone) .mockRejectedValueOnce(nodesError); // 3. /nodes (main list) -> FAILS // Configure mockPveClientInstance2 (succeeds fully, no VMs/CTs) mockPveClientInstance2.get .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'pve2-cluster' }] } }) // 1. /cluster/status .mockResolvedValueOnce({ data: { data: [{ node: 'node-pve2' }] } }) // 2. /nodes (standalone) .mockResolvedValueOnce({ data: { data: [{ node: 'node-pve2', status: 'online' }] } }) // 3. /nodes (main list) .mockResolvedValueOnce({ data: { data: { cpu: 0.1, uptime: 10 } } }) // 4. /nodes/node-pve2/status .mockResolvedValueOnce({ data: { data: [] } }) // 5. /nodes/node-pve2/storage .mockResolvedValueOnce({ data: { data: [] } }) // 6. /nodes/node-pve2/qemu .mockResolvedValueOnce({ data: { data: [] } }); // 7. /nodes/node-pve2/lxc <<< SHOULD BE EMPTY const result = await fetchDiscoveryData(mockClients, {}); // Pass custom clients // Assertions for pve1 (failed endpoint) expect(mockPveClientInstance1.get).toHaveBeenCalledTimes(3); // expect(consoleErrorSpy).toHaveBeenCalledWith( // expect.stringContaining(`[DataFetcher - PVE1 Endpoint] Error fetching PVE discovery data: ${nodesError.message}`) // ); // Assert mockPveClientInstance2 (successful endpoint) expect(mockPveClientInstance2.get).toHaveBeenCalledTimes(7); // Should still return data from the successful endpoint (pve2) expect(result.nodes).toHaveLength(1); expect(result.nodes[0].node).toBe('node-pve2'); expect(result.nodes[0].endpointId).toBe('pve2'); expect(result.vms).toHaveLength(0); expect(result.containers).toHaveLength(0); expect(result.pbs).toHaveLength(0); consoleErrorSpy.mockRestore(); }); test('should handle API error when fetching guests for a specific node', async () => { // Arrange: Uses the default mock clients from beforeEach const nodeNameGood = 'node-good'; const nodeNameBad = 'node-bad-guests'; // Use mockImplementation on the default mockPveClientInstance mockPveClientInstance.get.mockImplementation(async (url) => { console.log(`Mock API call: ${url}`); // Added for debugging if (url === '/nodes') { return { data: { data: [ { node: nodeNameGood, status: 'online', id: `node/${nodeNameGood}` }, { node: nodeNameBad, status: 'online', id: `node/${nodeNameBad}` } ]}}; } if (url === `/nodes/${nodeNameGood}/status`) { return { data: { data: { cpu: 0.1, uptime: 10 } } }; } if (url === `/nodes/${nodeNameGood}/storage`) { return { data: { data: [] } }; } if (url === `/nodes/${nodeNameGood}/qemu`) { return { data: { data: [ { vmid: 100, name: 'vm-good', status: 'running' } ] } }; } if (url === `/nodes/${nodeNameGood}/lxc`) { return { data: { data: [] } }; } if (url === `/nodes/${nodeNameBad}/status`) { return { data: { data: { cpu: 0.2, uptime: 20 } } }; } if (url === `/nodes/${nodeNameBad}/storage`) { return { data: { data: [] } }; } if (url === `/nodes/${nodeNameBad}/qemu`) { // Simulate API error for this specific call throw new Error('Simulated API Error Fetching Guests'); } if (url === `/nodes/${nodeNameBad}/lxc`) { return { data: { data: [] } }; // Successful but empty } // Default fallback for unexpected calls throw new Error(`Unexpected API call in mock: ${url}`); }); // Act: Uses the default mock clients from beforeEach const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert // Check that the correct number of nodes is returned expect(result.nodes).toHaveLength(2); // Find the nodes in the result const goodNodeResult = result.nodes.find(n => n.node === nodeNameGood); const badNodeResult = result.nodes.find(n => n.node === nodeNameBad); expect(goodNodeResult).toBeDefined(); expect(badNodeResult).toBeDefined(); // Assertions for the node where all calls succeeded (nodeNameGood) expect(goodNodeResult.cpu).toBe(0.1); // Should have CPU data from successful /status call expect(goodNodeResult.status).toBe('online'); // Status updated by uptime > 0 expect(goodNodeResult.vms).toBeUndefined(); // VMs/CTs are in the top-level result.vms/result.containers // Assertions for the node where /qemu failed (nodeNameBad) // It should still have basic info from /nodes and status info from its successful /status call expect(badNodeResult.cpu).toBe(0.2); // CPU data from its OWN successful /status call expect(badNodeResult.status).toBe('online'); // Status updated by uptime > 0 // It should not have contributed VMs/CTs because fetchDataForNode rejected // Assert overall VMs/Containers (only from the successful node) expect(result.vms).toHaveLength(1); expect(result.vms[0].vmid).toBe(100); // VM from nodeNameGood expect(result.containers).toHaveLength(0); // Assert PBS is empty expect(result.pbs).toEqual([]); }); test('should handle missing or invalid data.data for a node resource', async () => { // Arrange: Use default mock client and a specific node name const nodeName = 'node-missing-data-data'; // Can be the same or different, impact is on the mock const endpointId = 'primary'; const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); mockPveClientInstance.get // Endpoint level calls .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'test-cluster' }] } }) // 1. /cluster/status .mockResolvedValueOnce({ data: { data: [{ node: nodeName }] } }) // 2. /nodes (for standaloneNodeName) .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online' }] } }) // 3. /nodes (main node list for endpoint) // Node level calls for nodeName .mockResolvedValueOnce({ data: { data: null } }) // 4. /nodes/${nodeName}/status (INVALID data.data) .mockResolvedValueOnce({ data: { data: [] } }) // 5. /nodes/${nodeName}/storage .mockResolvedValueOnce({ data: { data: [] } }) // 6. /nodes/${nodeName}/qemu .mockResolvedValueOnce({ data: { data: [] } }); // 7. /nodes/${nodeName}/lxc // Act const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert // Verify the warning was logged by fetchNodeResource expect(consoleWarnSpy).toHaveBeenCalledWith( `[DataFetcher - ${endpointId}-${nodeName}] Node status data missing or invalid format.` ); // Verify that the node was still processed but status data is default (null/0) expect(result.nodes).toHaveLength(1); expect(result.nodes[0].node).toBe(nodeName); expect(result.nodes[0].cpu).toBeNull(); expect(result.nodes[0].uptime).toBe(0); expect(result.nodes[0].storage).toEqual([]); expect(result.vms).toHaveLength(0); expect(result.containers).toHaveLength(0); consoleWarnSpy.mockRestore(); }); // --- PBS Integration Tests --- test('should fetch PVE and PBS data correctly', async () => { // Arrange PVE (similar to happy path test, simplified) const nodeName = 'pve-node'; const vmId = 200; mockPveClientInstance.get // PVE Endpoint calls .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'test-cluster' }] } }) // 1. /cluster/status .mockResolvedValueOnce({ data: { data: [{ node: nodeName }] } }) // 2. /nodes (standalone) .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online' }] } }) // 3. /nodes (main list) // PVE Node calls for nodeName .mockResolvedValueOnce({ data: { data: { uptime: 1 } } }) // 4. /nodes/${nodeName}/status .mockResolvedValueOnce({ data: { data: [] } }) // 5. /nodes/${nodeName}/storage .mockResolvedValueOnce({ data: { data: [{ vmid: vmId, name: 'pve-vm' }] } }) // 6. /nodes/${nodeName}/qemu .mockResolvedValueOnce({ data: { data: [] } }); // 7. /nodes/${nodeName}/lxc // Arrange PBS (Mock the function to be injected) const mockPbsFunction = jest.fn(); const mockPbsResult = [ { pbsEndpointId: 'pbs1', pbsInstanceName: 'MyPBS', status: 'ok', nodeName: 'pbs-node', datastores: [{ name: 'ds1', total: 1000, used: 500, snapshots: [] }], } ]; mockPbsFunction.mockResolvedValue(mockPbsResult); // Act (Inject the mock function) const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient, mockPbsFunction); // Assert PVE expect(result.nodes).toHaveLength(1); expect(result.vms).toHaveLength(1); expect(result.containers).toHaveLength(0); // Assert PBS expect(mockPbsFunction).toHaveBeenCalledWith(mockPbsApiClient); expect(result.pbs).toEqual(mockPbsResult); }); test('should handle errors from fetchPbsData gracefully', async () => { // Arrange PVE (same simple mock as above) const nodeName = 'pve-node'; const vmId = 200; mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online' }] } }) // /nodes .mockResolvedValueOnce({ data: { data: { uptime: 1 } } }) // status .mockResolvedValueOnce({ data: { data: [] } }) // storage .mockResolvedValueOnce({ data: { data: [{ vmid: vmId, name: 'pve-vm' }] } }) // qemu .mockResolvedValueOnce({ data: { data: [] } }); // lxc // Arrange PBS (Mock the function to be injected) const mockPbsFunction = jest.fn(); const pbsError = new Error('PBS Connection Failed'); // Revert to mockRejectedValue mockPbsFunction.mockRejectedValue(pbsError); // Mock console.error to suppress expected error message during test const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); // Act (Inject the mock function) const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient, mockPbsFunction); // Assert PVE (should be empty due to Promise.all.catch) expect(result.nodes).toHaveLength(0); expect(result.vms).toHaveLength(0); expect(result.containers).toHaveLength(0); // Assert PBS (should be empty due to Promise.all.catch) expect(mockPbsFunction).toHaveBeenCalledWith(mockPbsApiClient); expect(result.pbs).toEqual([]); // Check that the catch block in fetchDiscoveryData logged the error expect(consoleErrorSpy).toHaveBeenCalledWith("[DataFetcher] Error during discovery cycle Promise.all:", pbsError); // Restore console.error consoleErrorSpy.mockRestore(); }); test('should return empty PBS array when no PBS clients configured', async () => { // Arrange PVE (same simple mock as above) const nodeName = 'pve-node'; const vmId = 200; mockPveClientInstance.get // PVE Endpoint calls .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'test-cluster' }] } }) // 1. /cluster/status .mockResolvedValueOnce({ data: { data: [{ node: nodeName }] } }) // 2. /nodes (standalone) .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online' }] } }) // 3. /nodes (main list) // PVE Node calls for nodeName .mockResolvedValueOnce({ data: { data: { uptime: 1 } } }) // 4. /nodes/${nodeName}/status .mockResolvedValueOnce({ data: { data: [] } }) // 5. /nodes/${nodeName}/storage .mockResolvedValueOnce({ data: { data: [{ vmid: vmId, name: 'pve-vm' }] } }) // 6. /nodes/${nodeName}/qemu .mockResolvedValueOnce({ data: { data: [] } }); // 7. /nodes/${nodeName}/lxc // Arrange PBS: Pass an empty object for PBS clients, mock injected function const emptyPbsClients = {}; const mockPbsFunction = jest.fn(); mockPbsFunction.mockResolvedValue([]); // Should resolve with empty when called with empty clients // Act (Inject the mock function) const result = await fetchDiscoveryData(mockPveApiClient, emptyPbsClients, mockPbsFunction); // Assert PVE expect(result.nodes).toHaveLength(1); expect(result.vms).toHaveLength(1); // Assert PBS expect(mockPbsFunction).toHaveBeenCalledWith(emptyPbsClients); expect(result.pbs).toEqual([]); }); test('should handle error fetching Containers (lxc)', async () => { // Arrange: Uses the default mock clients from beforeEach const nodeNameGood = 'node-good'; const nodeNameBad = 'node-bad-guests'; // This node will have the LXC fetch error const endpointId = 'primary'; // Default endpointId from mockPveApiClient setup const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); mockPveClientInstance.get.mockImplementation(async (url) => { // Endpoint level calls - needed for the overall fetchDiscoveryData structure if (url === '/cluster/status') { return { data: { data: [{ type: 'cluster', nodes: 2, name: 'test-cluster' }] } }; // Simulate a cluster with 2 nodes } if (url === '/nodes') { // This is the main /nodes call for the endpoint return { data: { data: [ { node: nodeNameGood, status: 'online', id: `node/${nodeNameGood}` }, { node: nodeNameBad, status: 'online', id: `node/${nodeNameBad}` } ]}}; } // Calls for nodeNameGood (all succeed) if (url === `/nodes/${nodeNameGood}/status`) return { data: { data: { cpu: 0.1, uptime: 10 } } }; if (url === `/nodes/${nodeNameGood}/storage`) return { data: { data: [] } }; if (url === `/nodes/${nodeNameGood}/qemu`) return { data: { data: [ { vmid: 100, name: 'vm-good', status: 'running' } ] } }; if (url === `/nodes/${nodeNameGood}/lxc`) return { data: { data: [] } }; // Calls for nodeNameBad if (url === `/nodes/${nodeNameBad}/status`) return { data: { data: { cpu: 0.2, uptime: 20 } } }; if (url === `/nodes/${nodeNameBad}/storage`) return { data: { data: [] } }; if (url === `/nodes/${nodeNameBad}/qemu`) return { data: { data: [] } }; // QEMU succeeds if (url === `/nodes/${nodeNameBad}/lxc`) { // LXC fetch fails throw new Error('Simulated LXC Fetch Error'); } // Fallback for unexpected calls console.warn(`Unexpected API call in mockImplementation: ${url}`); throw new Error(`Unexpected API call in mock: ${url}`); }); // Act: Uses the default mock clients from beforeEach const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`[DataFetcher - ${endpointId}-${nodeNameBad}] Error fetching Containers (lxc): Simulated LXC Fetch Error`) ); // Check that the correct number of nodes is returned expect(result.nodes).toHaveLength(2); const goodNodeResult = result.nodes.find(n => n.node === nodeNameGood); const badNodeResult = result.nodes.find(n => n.node === nodeNameBad); expect(goodNodeResult).toBeDefined(); expect(badNodeResult).toBeDefined(); // VMs/CTs from the good node should be present expect(result.vms).toHaveLength(1); // From nodeNameGood expect(result.vms[0].vmid).toBe(100); // Containers from nodeNameGood are [], and from nodeNameBad failed, so overall should be [] expect(result.containers).toHaveLength(0); consoleErrorSpy.mockRestore(); }); test('should handle errors fetching node status and storage', async () => { // Arrange const nodeName = 'node-bad-status-storage'; // This won't be used as errors occur at endpoint/node list level const endpointId = 'primary'; const endpointConfig = mockPveApiClient[endpointId].config; // Get the config for name const statusError = new Error('Status fetch failed'); const storageError = new Error('Storage fetch failed'); // This will be for the /nodes call const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); mockPveClientInstance.get.mockImplementation(async (url) => { if (url === '/cluster/status') { // console.log(`Mock: ${url} throwing statusError`); throw statusError; } // This will be called by the catch block of /cluster/status, and potentially the main /nodes call if (url === '/nodes') { // console.log(`Mock: ${url} throwing storageError`); throw storageError; } // console.warn(`Mock: Unexpected call to ${url}`); throw new Error(`Unexpected API call in mock for failing status/storage: ${url}`); }); // Act const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert // Check the first call to console.error specifically for the /cluster/status failure const firstCallArgs = consoleErrorSpy.mock.calls[0]; const expectedLogMessagePart = `[DataFetcher - primary] Error fetching /cluster/status: ${statusError.message}`; expect(firstCallArgs[0]).toContain(expectedLogMessagePart); expect(firstCallArgs[1]).toBeInstanceOf(Error); expect(firstCallArgs[1].message).toBe(statusError.message); // Check the second call for the /nodes failure after /cluster/status expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`[DataFetcher - primary] Also failed to fetch /nodes after /cluster/status error: ${storageError.message}`) ); expect(consoleErrorSpy).toHaveBeenCalledTimes(2); // Third error log from the main catch block in fetchDataForPveEndpoint // expect(consoleErrorSpy).toHaveBeenCalledWith( // expect.stringContaining(`[DataFetcher - primary] Error fetching PVE discovery data: ${storageError.message}`) // ); // expect(consoleErrorSpy).toHaveBeenCalledTimes(3); // Data should be empty for this endpoint due to critical failures expect(result.nodes).toEqual([]); expect(result.vms).toEqual([]); expect(result.containers).toEqual([]); consoleErrorSpy.mockRestore(); }); test('should handle invalid format for node status response (invalid data.data)', async () => { const nodeName = 'node-bad-status-data'; // A unique name for this test case const endpointId = 'primary'; const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); mockPveClientInstance.get // Endpoint level calls .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'test-cluster' }] } }) // 1. /cluster/status .mockResolvedValueOnce({ data: { data: [{ node: nodeName }] } }) // 2. /nodes (for standaloneNodeName) .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online' }] } }) // 3. /nodes (main node list for endpoint) // Node level calls for nodeName .mockResolvedValueOnce({ data: { data: null } }) // <--- Invalid: data.data is null for /status .mockResolvedValueOnce({ data: { data: [] } }) // storage .mockResolvedValueOnce({ data: { data: [] } }) // qemu .mockResolvedValueOnce({ data: { data: [] } }); // lxc const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining(`[DataFetcher - ${endpointId}-${nodeName}] Node status data missing or invalid format.`) ); // Node should still exist, but status fields should be default/null expect(result.nodes).toHaveLength(1); expect(result.nodes[0].node).toBe(nodeName); expect(result.nodes[0].cpu).toBeNull(); expect(result.nodes[0].mem).toBeNull(); // or existing value if not overwritten by status expect(result.nodes[0].uptime).toBe(0); consoleWarnSpy.mockRestore(); }); test('should handle invalid format for node status response', async () => { const nodeName = 'node-bad-status-format'; // Test a node where its /status call fails const endpointId = 'primary'; const endpointConfig = mockPveApiClient[endpointId].config; const statusFetchError = new Error('Node Status Network Error'); const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ type: 'cluster', nodes: 1, name: 'test-cluster' }] } }) // 1. /cluster/status .mockResolvedValueOnce({ data: { data: [{ node: nodeName }] } }) // 2. /nodes (for standaloneNodeName) .mockResolvedValueOnce({ data: { data: [{ node: nodeName, status: 'online' }] } }) // 3. /nodes (main node list for endpoint) .mockRejectedValueOnce(statusFetchError) // 4. /nodes/${nodeName}/status << THIS FAILS .mockResolvedValueOnce({ data: { data: [] } }) // 5. /nodes/${nodeName}/storage (subsequent calls should still be mocked) .mockResolvedValueOnce({ data: { data: [] } }) // 6. /nodes/${nodeName}/qemu .mockResolvedValueOnce({ data: { data: [] } }); // 7. /nodes/${nodeName}/lxc const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert error logging // Only one log from fetchNodeResource, as it catches the error and returns null, // so the promise in fetchDataForPveEndpoint for this node is fulfilled. expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`[DataFetcher - ${endpointId}-${nodeName}] Error fetching Node status: ${statusFetchError.message}`) ); expect(consoleErrorSpy).toHaveBeenCalledTimes(1); // Assert node data (node should exist, but with default/error state for status) expect(result.nodes).toHaveLength(1); expect(result.nodes[0].node).toBe(nodeName); expect(result.nodes[0].cpu).toBeNull(); expect(result.nodes[0].mem).toBeNull(); expect(result.nodes[0].uptime).toBe(0); // Other parts like storage should be processed if their mocks are fine (empty array here) expect(result.nodes[0].storage).toEqual([]); consoleErrorSpy.mockRestore(); }); test('should handle invalid /nodes response format', async () => { // Arrange const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); // Mock /nodes to return non-array data mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { not_an_array: true } } }); // Act const result = await fetchDiscoveryData(mockPveApiClient, mockPbsApiClient); // Assert // expect(consoleWarnSpy).toHaveBeenCalledWith( // expect.stringContaining(`[DataFetcher - primary] No nodes found or unexpected format.`) // ); // Should return empty arrays as if no nodes were found expect(result.nodes).toEqual([]); expect(result.vms).toEqual([]); expect(result.containers).toEqual([]); consoleWarnSpy.mockRestore(); }); test('should handle outer promise rejection for one PVE endpoint', async () => { // Arrange: Setup one valid client and one config that will cause an error const mockPveClientInstance1 = { get: jest.fn() }; const mockClients = { primary: { client: mockPveClientInstance1, config: { name: 'pve-good' } }, bad_endpoint: null, // This will cause TypeError when accessing .client }; const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); // Mock the valid client to succeed mockPveClientInstance1.get .mockResolvedValueOnce({ data: { data: [{ node: 'node-good', status: 'online' }] } }) .mockResolvedValueOnce({ data: { data: { uptime: 1 } } }) // status .mockResolvedValueOnce({ data: { data: [] } }) // storage .mockResolvedValueOnce({ data: { data: [] } }) // qemu .mockResolvedValueOnce({ data: { data: [] } }); // lxc // Act const result = await fetchDiscoveryData(mockClients, {}); // Assert // The error is now caught by the Promise.all().catch() in fetchDiscoveryData expect(consoleErrorSpy).toHaveBeenCalledWith( "[DataFetcher] Error during discovery cycle Promise.all:", expect.any(TypeError) // Check for the TypeError ); // Check that the overall result is empty because the Promise.all catch was triggered expect(result.nodes).toHaveLength(0); expect(result.vms).toHaveLength(0); expect(result.containers).toHaveLength(0); consoleErrorSpy.mockRestore(); }); test('should handle synchronous error within fetchPveDiscoveryData/fetchPbsData', async () => { // Arrange const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const validPveClients = { primary: { client: { get: jest.fn() }, config: { id: 'primary' } } }; const invalidPbsClients = null; // Trigger TypeError in fetchPbsData // Act // Pass valid PVE clients but invalid PBS clients to trigger sync error const result = await fetchDiscoveryData(validPveClients, invalidPbsClients); // Assert // Check that the Promise.all catch block logged the error expect(consoleErrorSpy).toHaveBeenCalledWith( "[DataFetcher] Error during discovery cycle Promise.all:", expect.any(TypeError) // Should be TypeError from Object.keys(null) ); // Check that the function returned the default empty structure expect(result).toEqual({ nodes: [], vms: [], containers: [], pbs: [] }); consoleErrorSpy.mockRestore(); }); }); // --- NEW: describe block for fetchMetricsData --- describe('fetchMetricsData', () => { // Note: This block now relies on mockPveApiClient and mockPveClientInstance // set up in the main beforeEach of the outer describe block. let mockCurrentApiClients; // Keep this structure locally if tests modify it beforeEach(() => { // Reset only the client's get method, as the client itself is setup outside mockPveClientInstance.get.mockClear(); // Use the mock PVE client setup in the outer scope. // Tests within this block might add more clients (e.g., pve2) to this object. mockCurrentApiClients = { ...mockPveApiClient }; }); test('should return empty array when no running guests are provided', async () => { const result = await fetchMetricsData([], [], mockCurrentApiClients); expect(result).toEqual([]); expect(mockPveClientInstance.get).not.toHaveBeenCalled(); // Use outer mock instance }); test('should fetch metrics for a single running VM', async () => { const runningVms = [ // Use the endpointId matching the outer mock setup (e.g., 'primary') { endpointId: 'primary', node: 'node1', vmid: 100, type: 'qemu', name: 'vm-test' } ]; // Mock RRD data response on the outer instance mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }); // Mock current status response on the outer instance mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: 1024, disk: 2048 } } }); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); // Retrieve endpointName from the config of the passed client const endpointName = mockCurrentApiClients.primary.config.name || 'primary'; // Fallback needed? expect(result[0]).toEqual({ id: 100, guestName: 'vm-test', node: 'node1', type: 'qemu', endpointId: 'primary', // Matches input endpointName: endpointName, // Use retrieved name data: [{ time: 1, cpu: 0.5 }], // RRD data current: { cpu: 0.5, mem: 1024, disk: 2048 } // Current status }); // Verify API calls on the outer instance expect(mockPveClientInstance.get).toHaveBeenCalledTimes(2); expect(mockPveClientInstance.get).toHaveBeenCalledWith('/nodes/node1/qemu/100/rrddata', expect.any(Object)); expect(mockPveClientInstance.get).toHaveBeenCalledWith('/nodes/node1/qemu/100/status/current'); }); test('should fetch metrics for a single running container', async () => { const runningContainers = [ { endpointId: 'primary', node: 'node2', vmid: 101, type: 'lxc', name: 'ct-test' } ]; mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ time: 2, cpu: 0.2 }] } }); // RRD mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.2, mem: 512, disk: 1024 } } }); // Current const result = await fetchMetricsData([], runningContainers, mockCurrentApiClients); const endpointName = mockCurrentApiClients.primary.config.name || 'primary'; expect(result).toHaveLength(1); expect(result[0]).toEqual({ id: 101, guestName: 'ct-test', node: 'node2', type: 'lxc', endpointId: 'primary', endpointName: endpointName, data: [{ time: 2, cpu: 0.2 }], current: { cpu: 0.2, mem: 512, disk: 1024 } }); expect(mockPveClientInstance.get).toHaveBeenCalledTimes(2); expect(mockPveClientInstance.get).toHaveBeenCalledWith('/nodes/node2/lxc/101/rrddata', expect.any(Object)); expect(mockPveClientInstance.get).toHaveBeenCalledWith('/nodes/node2/lxc/101/status/current'); }); test('should fetch metrics for multiple guests (VMs and CTs) across nodes/endpoints', async () => { // Add another endpoint client const mockApiClient2 = { get: jest.fn(), config: { name: 'PVE-2' } }; // Add config here // Add to the mockCurrentApiClients object used in this test scope mockCurrentApiClients.pve2 = { client: mockApiClient2, config: mockApiClient2.config }; const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 100, type: 'qemu', name: 'vm1' }, { endpointId: 'pve2', node: 'node3', vmid: 300, type: 'qemu', name: 'vm3' } ]; const runningContainers = [ { endpointId: 'primary', node: 'node1', vmid: 101, type: 'lxc', name: 'ct1' }, { endpointId: 'primary', node: 'node2', vmid: 200, type: 'lxc', name: 'ct2' } ]; // Mock responses for PVE-1 / node1 / vm1 (qemu 100) - Use mockPveClientInstance mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ cpu: 0.1 }] } }); // rrd mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.11 } } }); // current // Mock responses for PVE-1 / node1 / ct1 (lxc 101) - Use mockPveClientInstance mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ cpu: 0.2 }] } }); // rrd mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.22 } } }); // current // Mock responses for PVE-1 / node2 / ct2 (lxc 200) - Use mockPveClientInstance mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ cpu: 0.3 }] } }); // rrd mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.33 } } }); // current // Mock responses for PVE-2 / node3 / vm3 (qemu 300) - Use mockApiClient2 mockApiClient2.get.mockResolvedValueOnce({ data: { data: [{ cpu: 0.4 }] } }); // rrd mockApiClient2.get.mockResolvedValueOnce({ data: { data: { cpu: 0.44 } } }); // current const result = await fetchMetricsData(runningVms, runningContainers, mockCurrentApiClients); expect(result).toHaveLength(4); // Expect results for all 4 guests const primaryName = mockCurrentApiClients.primary.config.name || 'primary'; const pve2Name = mockCurrentApiClients.pve2.config.name || 'pve2'; // Check a couple of results expect(result.find(m => m.id === 100 && m.endpointId === 'primary')).toMatchObject({ guestName: 'vm1', endpointName: primaryName, current: { cpu: 0.11 } }); expect(result.find(m => m.id === 101 && m.endpointId === 'primary')).toMatchObject({ guestName: 'ct1', endpointName: primaryName, current: { cpu: 0.22 } }); expect(result.find(m => m.id === 200 && m.endpointId === 'primary')).toMatchObject({ guestName: 'ct2', endpointName: primaryName, current: { cpu: 0.33 } }); expect(result.find(m => m.id === 300 && m.endpointId === 'pve2')).toMatchObject({ guestName: 'vm3', endpointName: pve2Name, current: { cpu: 0.44 } }); expect(mockPveClientInstance.get).toHaveBeenCalledTimes(6); // 3 guests on pve1 * 2 calls each expect(mockApiClient2.get).toHaveBeenCalledTimes(2); // 1 guest on pve2 * 2 calls each }); test('should handle missing API client for an endpoint gracefully', async () => { const runningVms = [ // Reference the endpointId from the outer setup { endpointId: 'primary', node: 'node1', vmid: 100, type: 'qemu', name: 'vm-good' }, { endpointId: 'pve_missing', node: 'nodeX', vmid: 999, type: 'qemu', name: 'vm-bad-client' } ]; // Mock success for the valid guest (on the primary client) mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ cpu: 0.5 }] } }); // RRD mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.5 } } }); // Current // Spy on console.warn before the call const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); // Only the guest with a valid client should return data expect(result[0].id).toBe(100); // Check that the warning was called for the missing endpoint expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining('No API client found for endpoint: pve_missing')); // Ensure the valid client's API was called, but no attempt for the missing one expect(mockPveClientInstance.get).toHaveBeenCalledTimes(2); // Only calls for vm-good consoleWarnSpy.mockRestore(); // Clean up the spy }); test('should handle API error when fetching RRD data for one guest', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 100, type: 'qemu', name: 'vm-ok' }, { endpointId: 'primary', node: 'node1', vmid: 101, type: 'qemu', name: 'vm-fail-rrd' } ]; const error = new Error('RRD Fetch Failed'); const endpointName = mockCurrentApiClients.primary.config.name || 'primary'; // Mock success for vm-ok mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ cpu: 0.1 }] } }); // rrd ok mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.11 } } }); // current ok // Mock failure for vm-fail-rrd (RRD call fails, current call succeeds) mockPveClientInstance.get.mockRejectedValueOnce(error); // rrd fails mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.22 } } }); // current ok (Promise.all proceeds) // Spy on console.error const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); // Only vm-ok should be in results expect(result[0].id).toBe(100); // Verify API calls (Promise.all means both sets of calls were attempted) expect(mockPveClientInstance.get).toHaveBeenCalledTimes(4); // 2 calls * 2 guests // Verify error log for the failed guest expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`[Metrics Cycle - ${endpointName}] Failed to get metrics for qemu 101 (vm-fail-rrd) on node node1: RRD Fetch Failed`) ); consoleErrorSpy.mockRestore(); }); test('should handle API error when fetching current status for one guest', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 100, type: 'qemu', name: 'vm-ok' }, { endpointId: 'primary', node: 'node1', vmid: 101, type: 'qemu', name: 'vm-fail-current' } ]; const error = new Error('Current Status Fetch Failed'); const endpointName = mockCurrentApiClients.primary.config.name || 'primary'; // Mock success for vm-ok mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ cpu: 0.1 }] } }); // rrd ok mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.11 } } }); // current ok // Mock failure for vm-fail-current (RRD call succeeds, current call fails) mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ cpu: 0.2 }] } }); // rrd ok mockPveClientInstance.get.mockRejectedValueOnce(error); // current fails const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); // Only vm-ok should be in results expect(result[0].id).toBe(100); expect(mockPveClientInstance.get).toHaveBeenCalledTimes(4); expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`[Metrics Cycle - ${endpointName}] Failed to get metrics for qemu 101 (vm-fail-current) on node node1: Current Status Fetch Failed`) ); consoleErrorSpy.mockRestore(); }); test('should handle API 400 error gracefully (guest likely stopped)', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 100, type: 'qemu', name: 'vm-ok' }, { endpointId: 'primary', node: 'node1', vmid: 101, type: 'qemu', name: 'vm-stopped' } ]; const endpointName = mockCurrentApiClients.primary.config.name || 'primary'; // Simulate a 400 error response const error400 = new Error('Bad Request'); error400.response = { status: 400 }; // Mock success for vm-ok mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ cpu: 0.1 }] } }); // rrd ok mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.11 } } }); // current ok // Mock 400 failure for vm-stopped (assume RRD call fails first) mockPveClientInstance.get.mockRejectedValueOnce(error400); // rrd fails with 400 // The current status call for the failing guest might not even happen if RRD fails hard, // but mock it just in case the error handling changes. Let's assume it would succeed if called. mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.22 } } }); const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); // Only vm-ok should be in results expect(result[0].id).toBe(100); // Should attempt 2 calls for vm-ok, and potentially 2 calls for vm-stopped (RRD and current status) // even if RRD fails with 400, Promise.allSettled allows the current status call to proceed. expect(mockPveClientInstance.get).toHaveBeenCalledTimes(4); // Verify the specific warning log for the 400 error expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining(`[Metrics Cycle - ${endpointName}] Guest qemu 101 (vm-stopped) on node node1 might be stopped or inaccessible (Status: 400). Skipping metrics.`) ); // Verify the specific warning log for the 400 error // expect(consoleWarnSpy).toHaveBeenCalledWith( // expect.stringContaining(`[Metrics Cycle - ${endpointName}] Guest qemu 101 (vm-stopped) on node node1 might be stopped or inaccessible (Status: 400). Skipping metrics.`) // ); expect(consoleErrorSpy).not.toHaveBeenCalled(); // Should not log as a generic error consoleWarnSpy.mockRestore(); consoleErrorSpy.mockRestore(); }); test('should handle empty RRD data array gracefully', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 100, type: 'qemu', name: 'vm-no-rrd-data' } ]; // Mock RRD data response with empty data array mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [] } }); // Mock current status response mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: 1024 } } }); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); const endpointName = mockCurrentApiClients.primary.config.name || 'primary'; expect(result).toHaveLength(1); expect(result[0]).toMatchObject({ id: 100, endpointName: endpointName, data: [], // RRD data should be an empty array current: { cpu: 0.5, mem: 1024 } }); expect(mockPveClientInstance.get).toHaveBeenCalledTimes(2); }); test('should handle null current status data gracefully', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 100, type: 'qemu', name: 'vm-no-current-data' } ]; // Mock RRD data response mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }); // Mock current status response with null data mockPveClientInstance.get.mockResolvedValueOnce({ data: { data: null } }); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); const endpointName = mockCurrentApiClients.primary.config.name || 'primary'; expect(result).toHaveLength(1); expect(result[0]).toMatchObject({ id: 100, endpointName: endpointName, data: [{ time: 1, cpu: 0.5 }], current: null // Current data should be null }); expect(mockPveClientInstance.get).toHaveBeenCalledTimes(2); }); // --- Tests for QEMU Guest Agent Memory Fetching --- test('should fetch QEMU guest agent memory info when agent is enabled and responsive', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 100, type: 'qemu', name: 'vm-agent-ok', agent: '1' } ]; mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }) // RRD .mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: 2048*1024*1024, disk: 2048, agent: 1 } } }); // Current status (agent enabled) // Mock the POST call for guest agent mockPveClientInstance.post = jest.fn().mockResolvedValueOnce({ data: { data: { result: { total: 2048*1024*1024, free: 1024*1024*1024, available: 1536*1024*1024 } } } }); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); expect(result[0].current).toBeDefined(); expect(result[0].current.guest_mem_total_bytes).toBe(2048*1024*1024); expect(result[0].current.guest_mem_free_bytes).toBe(1024*1024*1024); expect(result[0].current.guest_mem_available_bytes).toBe(1536*1024*1024); expect(result[0].current.guest_mem_actual_used_bytes).toBe((2048-1536)*1024*1024); expect(mockPveClientInstance.post).toHaveBeenCalledWith('/nodes/node1/qemu/100/agent/get-memory-block-info', {}); }); test('should not attempt QEMU guest agent memory fetch if agent is not enabled in current status', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 101, type: 'qemu', name: 'vm-agent-off', agent: '1'} // Configured as on, but status says off ]; mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }) // RRD .mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: 1024, disk: 2048, agent: 0 } } }); // Current status (agent OFF) mockPveClientInstance.post = jest.fn(); // Ensure post is a mock const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); expect(result[0].current.guest_mem_total_bytes).toBeUndefined(); expect(mockPveClientInstance.post).not.toHaveBeenCalled(); }); test('should not attempt QEMU guest agent memory fetch if guest agent config is missing/off', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 102, type: 'qemu', name: 'vm-agent-not-configured' } // No agent field ]; mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }) // RRD .mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: 1024, disk: 2048, agent: 1 } } }); // Current status (agent ON) mockPveClientInstance.post = jest.fn(); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); expect(result[0].current.guest_mem_total_bytes).toBeUndefined(); expect(mockPveClientInstance.post).not.toHaveBeenCalled(); }); test('should handle QEMU guest agent error (e.g., agent not responsive)', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 103, type: 'qemu', name: 'vm-agent-error', agent: 'enabled=1' } ]; mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }) // RRD .mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: 1024, disk: 2048, agent: 1 } } }); // Current status const agentError = new Error('Agent not responsive'); agentError.response = { status: 500, data: { data: { exitcode: -2 } } }; mockPveClientInstance.post = jest.fn().mockRejectedValueOnce(agentError); const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); expect(result[0].current.guest_mem_total_bytes).toBeUndefined(); expect(mockPveClientInstance.post).toHaveBeenCalledTimes(1); expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining("QEMU Guest Agent not responsive or command 'get-memory-block-info' not available/supported")); consoleLogSpy.mockRestore(); }); test('should handle unexpected QEMU guest agent response format', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 104, type: 'qemu', name: 'vm-agent-bad-format', agent: '1' } ]; mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }) // RRD .mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: 1024, disk: 2048, agent: 1 } } }); // Current status mockPveClientInstance.post = jest.fn().mockResolvedValueOnce({ data: { data: { result: { unexpected: "data" } } } }); // Bad format const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); expect(result[0].current.guest_mem_total_bytes).toBeUndefined(); expect(mockPveClientInstance.post).toHaveBeenCalledTimes(1); // Adjust expectation to match the actual log which includes the data object expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining("Guest agent memory command 'get-memory-block-info' response format not as expected"), expect.objectContaining({ result: { unexpected: "data" } }) // Check for the logged object too ); consoleWarnSpy.mockRestore(); }); test('should handle generic error fetching QEMU guest agent memory info', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 105, type: 'qemu', name: 'vm-agent-generic-error', agent: '1' } ]; mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }) // RRD .mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: 1024, disk: 2048, agent: 1 } } }); // Current status const genericAgentError = new Error('Network Failure'); genericAgentError.response = { status: 503 }; // Simulate a non-500 error mockPveClientInstance.post = jest.fn().mockRejectedValueOnce(genericAgentError); const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); expect(result[0].current.guest_mem_total_bytes).toBeUndefined(); expect(mockPveClientInstance.post).toHaveBeenCalledTimes(1); expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining("Error fetching guest agent memory info: Network Failure. Status: 503") ); consoleWarnSpy.mockRestore(); }); test('should handle generic error fetching RRD/status data', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 106, type: 'qemu', name: 'vm-generic-rrd-error' } ]; const genericError = new Error('Server Unavailable'); genericError.response = { status: 503 }; // Simulate non-400 error // Mock RRD call to fail with generic error, current status call to succeed mockPveClientInstance.get .mockRejectedValueOnce(genericError) // RRD fails .mockResolvedValueOnce({ data: { data: { cpu: 0.1 } } }); // Current status succeeds const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); // Spy on warn too const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(0); // Guest data should be skipped due to the error expect(mockPveClientInstance.get).toHaveBeenCalledTimes(2); // Both RRD and current status were attempted expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`Failed to get metrics for qemu 106 (vm-generic-rrd-error) on node node1 (Status: 503): Server Unavailable`) ); expect(consoleWarnSpy).not.toHaveBeenCalledWith( // Ensure the 400-specific warning wasn't called expect.stringContaining("might be stopped or inaccessible") ); consoleErrorSpy.mockRestore(); consoleWarnSpy.mockRestore(); }); test('should calculate actual used memory using fallback when "available" is missing', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 107, type: 'qemu', name: 'vm-agent-fallback-mem', agent: '1' } ]; const totalMem = 4096 * 1024 * 1024; const freeMem = 1024 * 1024 * 1024; const cachedMem = 512 * 1024 * 1024; const buffersMem = 256 * 1024 * 1024; const expectedUsed = totalMem - freeMem - cachedMem - buffersMem; mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }) // RRD .mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: totalMem, disk: 2048, agent: 1 } } }); // Current status mockPveClientInstance.post = jest.fn().mockResolvedValueOnce({ data: { data: { // Agent response *without* 'available' field result: { total: totalMem, free: freeMem, cached: cachedMem, buffers: buffersMem } } } }); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); expect(result[0].current).toBeDefined(); expect(result[0].current.guest_mem_total_bytes).toBe(totalMem); expect(result[0].current.guest_mem_free_bytes).toBe(freeMem); expect(result[0].current.guest_mem_cached_bytes).toBe(cachedMem); expect(result[0].current.guest_mem_buffers_bytes).toBe(buffersMem); expect(result[0].current.guest_mem_available_bytes).toBeUndefined(); // Ensure 'available' was indeed missing expect(result[0].current.guest_mem_actual_used_bytes).toBe(expectedUsed); // Check fallback calculation expect(mockPveClientInstance.post).toHaveBeenCalledTimes(1); }); test('should calculate actual used memory using final fallback (total - free) when other fields missing', async () => { const runningVms = [ { endpointId: 'primary', node: 'node1', vmid: 108, type: 'qemu', name: 'vm-agent-final-fallback', agent: '1' } ]; const totalMem = 2048 * 1024 * 1024; const freeMem = 512 * 1024 * 1024; const expectedUsed = totalMem - freeMem; mockPveClientInstance.get .mockResolvedValueOnce({ data: { data: [{ time: 1, cpu: 0.5 }] } }) // RRD .mockResolvedValueOnce({ data: { data: { cpu: 0.5, mem: totalMem, disk: 2048, agent: 1 } } }); // Current status mockPveClientInstance.post = jest.fn().mockResolvedValueOnce({ data: { data: { // Agent response *only* with total and free result: { total: totalMem, free: freeMem } } } }); const result = await fetchMetricsData(runningVms, [], mockCurrentApiClients); expect(result).toHaveLength(1); expect(result[0].current).toBeDefined(); expect(result[0].current.guest_mem_total_bytes).toBe(totalMem); expect(result[0].current.guest_mem_free_bytes).toBe(freeMem); expect(result[0].current.guest_mem_available_bytes).toBeUndefined(); expect(result[0].current.guest_mem_cached_bytes).toBeUndefined(); expect(result[0].current.guest_mem_buffers_bytes).toBeUndefined(); expect(result[0].current.guest_mem_actual_used_bytes).toBe(expectedUsed); // Check final fallback calculation expect(mockPveClientInstance.post).toHaveBeenCalledTimes(1); }); }); // End describe fetchMetricsData // --- NEW: describe block for fetchPbsData --- describe('fetchPbsData', () => { // Relies on mockPbsApiClient and mockPbsClientInstance from outer describe beforeEach(() => { // Ensure the default mocks are reset/cleared if needed for PBS specific tests // Typically mockPbsClientInstance.get.mockClear() is sufficient if reusing the instance mockPbsClientInstance.get.mockClear(); // Reset the default PBS mock to an empty object for clarity mockPbsApiClient = {}; // Override the initializer mock if tests need specific PBS clients setup via initializeApiClients // Otherwise, tests will construct and pass mock PBS clients directly }); test('should return empty array when no PBS clients are provided', async () => { const result = await fetchPbsData({}); expect(result).toEqual([]); }); test('should fetch data correctly for one PBS instance (happy path)', async () => { // Arrange const pbsId = 'pbs-happy'; const pbsName = 'PBS Happy Path'; const pbsNodeName = 'pbs-node1'; const datastoreName = 'datastore1'; const mockPbsClient = { get: jest.fn() }; const mockClients = { [pbsId]: { client: mockPbsClient, config: { name: pbsName } } }; mockPbsClient.get .mockResolvedValueOnce({ data: { data: [{ node: pbsNodeName }] } }) // /nodes .mockResolvedValueOnce({ data: { data: [{ store: datastoreName, total: 1, used: 0 }] } }) // /status/datastore-usage .mockResolvedValueOnce({ data: { data: [{ 'backup-id': 'snap1' }] } }) // /admin/datastore/{store}/snapshots .mockResolvedValueOnce({ data: { data: [{ store: datastoreName, 'deduplication-factor': 1.0 }] } }) // /status/datastore-usage in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [{ name: datastoreName }] } }) // /config/datastore (for fetchAllPbsTasksForProcessing) .mockResolvedValueOnce({ data: { data: [{ 'backup-type': 'vm', 'backup-id': '100' }] } }) // /admin/datastore/{store}/groups .mockResolvedValueOnce({ data: { data: [{ 'backup-time': Math.floor(Date.now() / 1000) - 86400 }] } }) // /admin/datastore/{store}/snapshots for group (1 day ago) .mockResolvedValueOnce({ data: { data: [] } }); // /nodes/{node}/tasks - empty to simplify test // Act const result = await fetchPbsData(mockClients); // Assert expect(result).toHaveLength(1); expect(result[0]).toMatchObject({ pbsEndpointId: pbsId, pbsInstanceName: pbsName, nodeName: pbsNodeName, status: 'ok', datastores: expect.any(Array) }); // Since nodeStatus and versionInfo are skipped in test env, total calls are reduced expect(mockPbsClient.get).toHaveBeenCalledTimes(8); // nodes, usage, snapshots, usage(dedup), config/datastore, groups, snapshots(group), tasks expect(mockPbsClient.get).toHaveBeenCalledWith('/nodes'); expect(mockPbsClient.get).toHaveBeenCalledWith('/status/datastore-usage'); expect(mockPbsClient.get).toHaveBeenCalledWith(`/admin/datastore/${datastoreName}/snapshots`); expect(mockPbsClient.get).toHaveBeenCalledWith(`/nodes/${pbsNodeName}/tasks`, expect.any(Object)); // Check that basic task structure is present - tasks processing has been verified in pbsUtils tests expect(result[0]).toHaveProperty('backupTasks'); expect(result[0]).toHaveProperty('verifyTasks'); expect(result[0]).toHaveProperty('gcTasks'); }); test('should handle error fetching PBS node name (and skip subsequent calls)', async () => { const pbsId = 'pbs-err-node'; const mockPbsClient = { get: jest.fn() }; const mockClients = { [pbsId]: { client: mockPbsClient, config: { name: 'PBS Node Err' } } }; const nodeError = new Error('Node fetch failed'); // Mock /nodes to fail mockPbsClient.get.mockRejectedValueOnce(nodeError); const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const result = await fetchPbsData(mockClients); expect(result).toHaveLength(1); expect(result[0].status).toBe('error'); // Should be error because node detection failed // Expect empty arrays/undefined for other fields as fetching is skipped expect(result[0].datastores).toBeUndefined(); expect(result[0].backupTasks).toBeUndefined(); // ... other task types ... // Verify API calls (only /nodes should have been attempted) expect(mockPbsClient.get).toHaveBeenCalledTimes(1); expect(mockPbsClient.get).toHaveBeenCalledWith('/nodes'); // Verify error logging // There are two error logs now: one from fetchPbsNodeName catch and one from fetchPbsData outer catch expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`ERROR: [DataFetcher] Failed to fetch PBS nodes list for ${mockClients[pbsId].config.name}: ${nodeError.message}`) ); expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`ERROR: [DataFetcher - ${mockClients[pbsId].config.name}] PBS fetch failed: Could not determine node name for PBS instance ${mockClients[pbsId].config.name}`) ); expect(consoleErrorSpy).toHaveBeenCalledTimes(2); consoleErrorSpy.mockRestore(); }); test('should handle error fetching PBS datastores', async () => { // Arrange const pbsId = 'pbs-err-ds'; const pbsNodeName = 'pbs-node-ds-err'; const mockPbsClient = { get: jest.fn() }; const mockClients = { [pbsId]: { client: mockPbsClient, config: { name: 'PBS DS Err' } } }; const dsError = new Error('Datastore fetch failed'); mockPbsClient.get .mockResolvedValueOnce({ data: { data: [{ node: pbsNodeName }] } }) // /nodes (succeeds) .mockRejectedValueOnce(dsError) // /status/datastore-usage (fails) .mockResolvedValueOnce({ data: { data: [] } }) // Mock fallback /config/datastore call (returns empty) .mockRejectedValueOnce(new Error('Dedup fetch failed')) // /status/datastore-usage in fetchAllPbsTasksForProcessing (fails) .mockResolvedValueOnce({ data: { data: [] } }) // /config/datastore in fetchAllPbsTasksForProcessing (empty) .mockResolvedValueOnce({ data: { data: [] } }); // /nodes/{node}/tasks (empty tasks) const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const result = await fetchPbsData(mockClients); expect(result).toHaveLength(1); expect(result[0]).toMatchObject({ pbsEndpointId: pbsId, status: 'ok', // Status remains 'ok' because node name succeeded, datastore fetch fell back nodeName: pbsNodeName, datastores: [], // Should be empty }); // Expect calls for /nodes, /status/datastore-usage, the fallback /config/datastore // No snapshots because datastores is empty. Task processing calls /status/datastore-usage (fails), /config/datastore again and /tasks expect(mockPbsClient.get).toHaveBeenCalledTimes(6); // Expect a warning about the fallback, not an error in the result object itself expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining(`Failed to get datastore usage for ${mockClients[pbsId].config.name}, falling back to /config/datastore. Error: ${dsError.message}`)); consoleErrorSpy.mockRestore(); consoleWarnSpy.mockRestore(); }); test('should handle error fetching snapshots for one datastore but succeed for others', async () => { const pbsId = 'pbs-err-snap'; const pbsNodeName = 'pbs-node-snap-err'; const dsGood = 'datastore-good'; const dsBad = 'datastore-bad'; const mockPbsClient = { get: jest.fn() }; const mockClients = { [pbsId]: { client: mockPbsClient, config: { name: 'PBS Snap Err' } } }; const snapError = new Error('Snapshot fetch failed'); mockPbsClient.get .mockResolvedValueOnce({ data: { data: [{ node: pbsNodeName }] } }) // /nodes .mockResolvedValueOnce({ data: { data: [{ store: dsGood, total: 1, used: 0 }, { store: dsBad, total: 1, used: 0 }] } }) // /status/datastore-usage // Mock snapshot fetches: .mockResolvedValueOnce({ data: { data: [{ 'backup-id': 'snap-good-1' }] } }) // snapshots for dsGood (success) .mockRejectedValueOnce(snapError) // snapshots for dsBad (fails) .mockResolvedValueOnce({ data: { data: [{ store: dsGood, 'deduplication-factor': 1.0 }] } }) // /status/datastore-usage in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [{ name: dsGood }, { name: dsBad }] } }) // /config/datastore in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [{ 'backup-type': 'vm', 'backup-id': '100' }] } }) // /admin/datastore/{dsGood}/groups .mockResolvedValueOnce({ data: { data: [{ 'backup-time': 1678886400 }] } }) // /admin/datastore/{dsGood}/snapshots for group .mockResolvedValueOnce({ data: { data: [{ 'backup-type': 'vm', 'backup-id': '101' }] } }) // /admin/datastore/{dsBad}/groups .mockResolvedValueOnce({ data: { data: [{ 'backup-time': 1678886401 }] } }) // /admin/datastore/{dsBad}/snapshots for group .mockResolvedValueOnce({ data: { data: [{ upid: 'task1'}] } }); // /nodes/{node}/tasks const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); // Act const result = await fetchPbsData(mockClients); // Assert expect(result).toHaveLength(1); expect(result[0].status).toBe('ok'); // Status is still ok expect(result[0].nodeName).toBe(pbsNodeName); expect(result[0].datastores).toHaveLength(2); // Check datastore results (snapshots for dsBad should be empty array) const goodDs = result[0].datastores.find(ds => ds.name === dsGood); const badDs = result[0].datastores.find(ds => ds.name === dsBad); expect(goodDs).toBeDefined(); expect(goodDs.snapshots).toHaveLength(1); expect(badDs).toBeDefined(); expect(badDs.snapshots).toHaveLength(0); // Empty due to error // Verify API calls: nodes, usage, snapshots(good), snapshots(bad), tasks expect(mockPbsClient.get).toHaveBeenCalledTimes(11); // nodes, usage, snapshots(good), snapshots(bad), usage(dedup), config, groups*2, snapshots*2, tasks expect(mockPbsClient.get).toHaveBeenCalledWith('/nodes'); expect(mockPbsClient.get).toHaveBeenCalledWith('/status/datastore-usage'); expect(mockPbsClient.get).toHaveBeenCalledWith(`/admin/datastore/${dsGood}/snapshots`); expect(mockPbsClient.get).toHaveBeenCalledWith(`/admin/datastore/${dsBad}/snapshots`); expect(mockPbsClient.get).toHaveBeenCalledWith(`/nodes/${pbsNodeName}/tasks`, expect.any(Object)); // Verify error log for the failed snapshot fetch expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`ERROR: [DataFetcher] Failed to fetch snapshots for datastore ${dsBad} on ${mockClients[pbsId].config.name}: ${snapError.message}`) ); expect(consoleErrorSpy).toHaveBeenCalledTimes(1); // Only snapshot error, not outer catch consoleErrorSpy.mockRestore(); }); test('should handle error fetching PBS tasks (tasks are null, error is true)', async () => { const pbsId = 'pbs-err-tasks'; const pbsNodeName = 'pbs-node-tasks-err'; const datastoreName = 'store-tasks-err'; const mockPbsClient = { get: jest.fn() }; const mockClients = { [pbsId]: { client: mockPbsClient, config: { name: 'PBS Tasks Err' } } }; const taskError = new Error('Task fetch failed'); mockPbsClient.get .mockResolvedValueOnce({ data: { data: [{ node: pbsNodeName }] } }) // /nodes .mockResolvedValueOnce({ data: { data: [{ store: datastoreName, total: 1, used: 0 }] } }) // /status/datastore-usage // Mock snapshot fetch to also fail to simulate the observed error path from the test output .mockRejectedValueOnce(new Error('Datastore backup history fetch failed')) // Snapshots (fails here) .mockResolvedValueOnce({ data: { data: [{ store: datastoreName, 'deduplication-factor': 1.0 }] } }) // /status/datastore-usage in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [{ name: datastoreName }] } }) // /config/datastore in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [] } }) // /admin/datastore/{store}/groups (empty) .mockRejectedValueOnce(taskError); // /nodes/{node}/tasks (fails) const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const result = await fetchPbsData(mockClients); expect(result).toHaveLength(1); expect(result[0].status).toBe('ok'); // Status is still ok expect(result[0].datastores).toHaveLength(1); // Datastore fetch succeeded (initial usage) // Snapshots should be empty as that fetch failed expect(result[0].datastores[0].snapshots).toHaveLength(0); // Tasks now have default structure when fetchAllPbsTasksForProcessing fails expect(result[0]).toHaveProperty('backupTasks'); expect(result[0]).toHaveProperty('verifyTasks'); expect(result[0]).toHaveProperty('gcTasks'); // Expected calls: nodes, usage, snapshots (fails), then tasks processing continues expect(mockPbsClient.get).toHaveBeenCalledTimes(7); // nodes, usage, snapshots (fail), usage(dedup), config/datastore, groups, tasks // Update expected error message to match actual output from snapshot fetch failure expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`ERROR: [DataFetcher] Failed to fetch snapshots for datastore store-tasks-err on PBS Tasks Err`) ); // Expect warning about tasks not being processed expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining(`No tasks to process or task fetching failed. Error flag: true, Tasks array: null`) ); consoleErrorSpy.mockRestore(); consoleWarnSpy.mockRestore(); }); test('should aggregate data from multiple PBS instances, including partial failures', async () => { const pbsId1 = 'pbs-ok'; const pbsId2 = 'pbs-fail-ds'; const pbsId3 = 'pbs-fail-node'; const mockClient1 = { get: jest.fn() }; const mockClient2 = { get: jest.fn() }; const mockClient3 = { get: jest.fn() }; const mockClients = { [pbsId1]: { client: mockClient1, config: { name: 'PBS OK' } }, [pbsId2]: { client: mockClient2, config: { name: 'PBS DS Fail' } }, [pbsId3]: { client: mockClient3, config: { name: 'PBS Node Fail' } }, }; // Mock Client 1 (Success) mockClient1.get .mockResolvedValueOnce({ data: { data: [{ node: 'node1' }] } }) // nodes .mockResolvedValueOnce({ data: { data: [{ store: 'ds1', total: 1, used: 0 }] } }) // usage .mockResolvedValueOnce({ data: { data: [] } }) // snapshots for ds1 .mockResolvedValueOnce({ data: { data: [{ store: 'ds1', 'deduplication-factor': 1.0 }] } }) // /status/datastore-usage in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [{ name: 'ds1' }] } }) // /config/datastore in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [] } }) // /admin/datastore/ds1/groups (empty) .mockResolvedValueOnce({ data: { data: [{upid: 'task-c1', type: 'backup', status: 'OK', starttime: 1, endtime: 2, worker_id: 'vm/1'}] } }); // tasks for node1 (success) // Mock Client 2 (Fail Datastore, fallback success, tasks success) const node2Name = 'node2'; const fallbackDsName = 'fallback-ds-client2'; mockClient2.get .mockResolvedValueOnce({ data: { data: [{ node: node2Name }] } }) // /nodes (success) .mockRejectedValueOnce(new Error('DS Usage API Error')) // /status/datastore-usage (fail) .mockResolvedValueOnce({ data: { data: [{ name: fallbackDsName, path: '/mnt/fb', store: fallbackDsName }] }}) // /config/datastore (fallback success, 1 store) .mockResolvedValueOnce({ data: { data: [{ 'backup-id': 'snap-fb'}] } }) // /admin/datastore/fallback-ds-client2/snapshots (success) .mockResolvedValueOnce({ data: { data: [{ store: fallbackDsName, 'deduplication-factor': 1.0 }] } }) // /status/datastore-usage in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [{ name: fallbackDsName }] } }) // /config/datastore in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [] } }) // /admin/datastore/{fallback}/groups (empty) .mockResolvedValueOnce({ data: { data: [{ upid: 'task-c2', type: 'verify', status: 'OK', starttime: 3, endtime: 4 }] } }); // /nodes/node2/tasks (success) // Mock Client 3 (Fail Node - tasks won't be called) mockClient3.get.mockRejectedValueOnce(new Error('Node Error')); // nodes fails const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); // Act const result = await fetchPbsData(mockClients); // Assert // Expect all instances in the final result, including the failed one expect(result).toHaveLength(3); const res1 = result.find(r => r.pbsEndpointId === pbsId1); const res2 = result.find(r => r.pbsEndpointId === pbsId2); const res3 = result.find(r => r.pbsEndpointId === pbsId3); expect(res1).toBeDefined(); expect(res1.status).toBe('ok'); expect(res1.datastores).toHaveLength(1); expect(res1).toHaveProperty('backupTasks'); // These fields are now defined but potentially empty expect(res1).toHaveProperty('verifyTasks'); expect(res1).toHaveProperty('gcTasks'); expect(res2).toBeDefined(); expect(res2.status).toBe('ok'); expect(res2.nodeName).toBe(node2Name); expect(res2.datastores).toHaveLength(1); expect(res2.datastores[0].name).toBe(fallbackDsName); expect(res2.datastores[0].snapshots).toHaveLength(1); expect(res2.datastores[0].snapshots[0]['backup-id']).toBe('snap-fb'); expect(res2.backupTasks).toBeDefined(); expect(res2.verifyTasks).toBeDefined(); expect(res2.gcTasks).toBeDefined(); expect(res3).toBeDefined(); expect(res3).toMatchObject({ pbsEndpointId: pbsId3, pbsInstanceName: mockClients[pbsId3].config.name, status: 'error' // error field is not present in the returned object here }); // Data should be empty/undefined for res3 as node fetch failed expect(res3.datastores).toBeUndefined(); expect(res3.backupTasks).toBeUndefined(); // Error spy should be called for the node failure on res3 and the usage error on res2 expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining(`ERROR: [DataFetcher] Failed to fetch PBS nodes list for ${mockClients[pbsId3].config.name}: Node Error`)); // Error log from fetchPbsData outer catch for res3 node failure expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining(`ERROR: [DataFetcher - ${mockClients[pbsId3].config.name}] PBS fetch failed: Could not determine node name for PBS instance`)); expect(consoleErrorSpy).toHaveBeenCalledTimes(2); // Node list error + Outer catch error for res3 // Warn spy should be called for the datastore fallback on res2 expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining(`WARN: [DataFetcher] Failed to get datastore usage for PBS DS Fail, falling back to /config/datastore. Error: DS Usage API Error`)); // Warn log from tasks fetch failure on res2 (if tasks fetch also failed, which it doesn't in this mock) // expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining(`No tasks to process or task fetching failed`)); expect(consoleWarnSpy).toHaveBeenCalledTimes(1); // Only datastore usage fallback warning for res2 // Check calls for each client // The call counts now align with the simplified test environment flow expect(mockClient1.get).toHaveBeenCalledTimes(7); // nodes, usage, snapshots, usage(dedup), config/datastore, groups, tasks expect(mockClient2.get).toHaveBeenCalledTimes(8); // nodes, usage (fail), config, snapshots, usage(dedup), config/datastore, groups, tasks expect(mockClient3.get).toHaveBeenCalledTimes(1); // nodes (fail) consoleErrorSpy.mockRestore(); consoleWarnSpy.mockRestore(); }); test('should return error status and log warnings if /nodes response is invalid (e.g. empty array)', async () => { // Arrange const pbsId = 'pbs-bad-nodes'; const mockPbsBadNodesClient = { get: jest.fn() }; const mockPbsBadNodesApiClients = { [pbsId]: { client: mockPbsBadNodesClient, config: { id: pbsId, name: 'PBS Bad Nodes' } } }; const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); // Mock /nodes to return invalid data (empty array) mockPbsBadNodesClient.get.mockResolvedValueOnce({ data: { data: [] } }); // Invalid - empty array // Act const result = await fetchPbsData(mockPbsBadNodesApiClients); // Assert expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining(`WARN: [DataFetcher] Could not automatically detect PBS node name for ${mockPbsBadNodesApiClients[pbsId].config.name}. Response format unexpected.`) ); // This warning also occurs because nodeName becomes 'localhost' and then the outer catch hits // expect(consoleWarnSpy).toHaveBeenCalledWith( // expect.stringContaining(`WARN: [DataFetcher - PBS Bad Nodes] Node name 'localhost' is invalid or 'localhost'. Throwing error.`) // ); expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`ERROR: [DataFetcher - ${mockPbsBadNodesApiClients[pbsId].config.name}] PBS fetch failed: Could not determine node name for PBS instance ${mockPbsBadNodesApiClients[pbsId].config.name}`) ); expect(consoleErrorSpy).toHaveBeenCalledTimes(1); // Only the outer catch error expect(mockPbsBadNodesClient.get).toHaveBeenCalledTimes(1); // Only /nodes called expect(mockPbsBadNodesClient.get).toHaveBeenCalledWith('/nodes'); expect(result).toHaveLength(1); expect(result[0]).toMatchObject({ pbsEndpointId: pbsId, pbsInstanceName: 'PBS Bad Nodes', status: 'error' // Should be error because node detection failed }); // Task related fields should be undefined as processing is skipped // expect(result[0].backupTasks).toBeUndefined(); consoleWarnSpy.mockRestore(); consoleErrorSpy.mockRestore(); }); test('should handle empty datastore usage response and attempt fallback, tasks should still process', async () => { // Arrange const pbsId = 'pbs-empty-usage'; const pbsNodeName = 'pbs-node-empty-usage'; const mockPbsClient = { get: jest.fn() }; const mockClients = { [pbsId]: { client: mockPbsClient, config: { name: 'PBS Empty Usage' } } }; const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); mockPbsClient.get .mockResolvedValueOnce({ data: { data: [{ node: pbsNodeName }] } }) // /nodes .mockResolvedValueOnce({ data: { data: [] } }) // /status/datastore-usage (empty) .mockResolvedValueOnce({ data: { data: [{ name: 'fallback-store', path: '/mnt/fallback', store: 'fallback-store' }] } }) // /config/datastore (fallback succeeds) .mockResolvedValueOnce({ data: { data: [{ 'backup-id': 'snap1' }] } }) // snapshots for 'fallback-store' .mockResolvedValueOnce({ data: { data: [{ store: 'fallback-store', 'deduplication-factor': 1.0 }] } }) // /status/datastore-usage in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [{ name: 'fallback-store' }] } }) // /config/datastore in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [] } }) // /admin/datastore/fallback-store/groups (empty) .mockResolvedValueOnce({ data: { data: [{ upid: 'task1'}] } }); // tasks for pbsNodeName // Act const result = await fetchPbsData(mockClients); // Assert expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining(`WARN: [DataFetcher] PBS /status/datastore-usage returned empty data for ${mockClients[pbsId].config.name}. Falling back.`) ); expect(mockPbsClient.get).toHaveBeenCalledWith('/config/datastore'); // Check fallback was attempted expect(mockPbsClient.get).toHaveBeenCalledWith('/admin/datastore/fallback-store/snapshots'); // Check snapshot call // Since nodeStatus and versionInfo are skipped, the call count is reduced expect(mockPbsClient.get).toHaveBeenCalledTimes(8); // nodes, usage, config, snapshots, usage(dedup), config/datastore, groups, tasks expect(mockPbsClient.get).toHaveBeenCalledWith(`/nodes/${pbsNodeName}/tasks`, expect.any(Object)); // Check task call expect(result).toHaveLength(1); expect(result[0].status).toBe('ok'); expect(result[0].datastores).toHaveLength(1); expect(result[0].datastores[0].name).toBe('fallback-store'); expect(result[0].datastores[0].total).toBeNull(); // Fallback doesn't have usage stats from /config/datastore expect(result[0].datastores[0].snapshots).toHaveLength(1); // Check snapshots from mock expect(result[0].datastores[0].snapshots[0]['backup-id']).toBe('snap1'); // Check tasks (processed by mocked processPbsTasks) expect(result[0].backupTasks).toEqual([]); expect(result[0].verifyTasks).toEqual([]); expect(result[0].gcTasks).toEqual([]); consoleWarnSpy.mockRestore(); }); test('should handle error fetching datastore usage', async () => { // Arrange const pbsId = 'pbs-err-ds'; const pbsNodeName = 'pbs-node-ds-err'; const mockPbsClient = { get: jest.fn() }; const mockClients = { [pbsId]: { client: mockPbsClient, config: { name: 'PBS DS Err' } } }; const dsError = new Error('Datastore fetch failed'); mockPbsClient.get .mockResolvedValueOnce({ data: { data: [{ node: pbsNodeName }] } }) // /nodes (succeeds) .mockRejectedValueOnce(dsError) // /status/datastore-usage (fails) .mockResolvedValueOnce({ data: { data: [] }}) // Mock fallback /config/datastore call (returns empty) .mockRejectedValueOnce(new Error('Dedup fetch failed')) // /status/datastore-usage in fetchAllPbsTasksForProcessing (fails) .mockResolvedValueOnce({ data: { data: [] }}) // /config/datastore in fetchAllPbsTasksForProcessing (empty) .mockResolvedValueOnce({ data: { data: [] }}); // /nodes/{node}/tasks (empty tasks) const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const result = await fetchPbsData(mockClients); expect(result).toHaveLength(1); expect(result[0]).toMatchObject({ pbsEndpointId: pbsId, status: 'ok', // Status remains 'ok' because node name succeeded, datastore fetch fell back nodeName: pbsNodeName, datastores: [], // Should be empty }); // Expect calls for /nodes, /status/datastore-usage, the fallback /config/datastore // No snapshots because datastores is empty. Task processing calls /status/datastore-usage (fails), /config/datastore again and /tasks expect(mockPbsClient.get).toHaveBeenCalledTimes(6); // Expect a warning about the fallback, not an error in the result object itself expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining(`Failed to get datastore usage for ${mockClients[pbsId].config.name}, falling back to /config/datastore. Error: ${dsError.message}`)); // Task fetch doesn't happen for res2 in this scenario, so no error logged for it. // expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining(`Failed to fetch PBS task list for node ${pbsNodeName} (${mockClients[pbsId].config.name}):`)); consoleErrorSpy.mockRestore(); consoleWarnSpy.mockRestore(); }); test('should handle failure of both datastore usage and config fetch', async () => { // Arrange const pbsId = 'pbs-double-ds-fail'; const pbsNodeName = 'pbs-node-double-fail'; const mockPbsClient = { get: jest.fn() }; const mockClients = { [pbsId]: { client: mockPbsClient, config: { name: 'PBS Double DS Fail' } } }; const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const usageError = new Error('Usage API Failed'); const configError = new Error('Config API Failed'); mockPbsClient.get .mockResolvedValueOnce({ data: { data: [{ node: pbsNodeName }] } }) // /nodes .mockRejectedValueOnce(usageError) // /status/datastore-usage (fails) .mockRejectedValueOnce(configError) // /config/datastore (fallback also fails) .mockRejectedValueOnce(new Error('Dedup fetch failed')) // /status/datastore-usage in fetchAllPbsTasksForProcessing (fails) .mockResolvedValueOnce({ data: { data: [] } }) // /config/datastore in fetchAllPbsTasksForProcessing (empty due to previous failure) .mockResolvedValueOnce({ data: { data: [{ upid: 'task1' }] } }); // tasks for pbsNodeName (still called) // Act const result = await fetchPbsData(mockClients); // Assert expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining(`WARN: [DataFetcher] Failed to get datastore usage for ${mockClients[pbsId].config.name}, falling back to /config/datastore. Error: ${usageError.message}`) ); expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`ERROR: [DataFetcher] Fallback fetch of PBS datastore config failed for ${mockClients[pbsId].config.name}: ${configError.message}`) ); expect(mockPbsClient.get).toHaveBeenCalledWith('/status/datastore-usage'); expect(mockPbsClient.get).toHaveBeenCalledWith('/config/datastore'); // expect(mockPbsClient.get).toHaveBeenCalledWith(`/nodes/${pbsNodeName}/tasks`); // This was the original failing line expect(mockPbsClient.get).toHaveBeenCalledTimes(6); // nodes, usage (fail), config (fail), usage(dedup fail), config/datastore, tasks const callsDoubleFailTest = mockPbsClient.get.mock.calls; expect(callsDoubleFailTest[5][0]).toBe(`/nodes/${pbsNodeName}/tasks`); // Check 6th call path expect(result).toHaveLength(1); expect(result[0].status).toBe('ok'); // Status is still ok as node name succeeded expect(result[0].datastores).toEqual([]); // Datastores should be empty consoleWarnSpy.mockRestore(); consoleErrorSpy.mockRestore(); }); test('should handle API error when fetching PBS tasks', async () => { const pbsId = 'pbs-task-fetch-error'; const pbsNodeName = 'pbs-node-task-error'; const datastoreName = 'store-task-error'; const mockPbsClient = { get: jest.fn() }; const mockClients = { [pbsId]: { client: mockPbsClient, config: { name: 'PBS Task Fetch Error' } } }; const taskError = new Error('Simulated task fetch error'); mockPbsClient.get .mockResolvedValueOnce({ data: { data: [{ node: pbsNodeName }] } }) // /nodes (succeeds) .mockResolvedValueOnce({ data: { data: [{ store: datastoreName, total: 1, used: 0 }] } }) // /status/datastore-usage (succeeds) .mockResolvedValueOnce({ data: { data: [] } }) // Snapshots (succeeds empty) .mockResolvedValueOnce({ data: { data: [{ name: datastoreName }] } }) // /config/datastore in fetchAllPbsTasksForProcessing .mockResolvedValueOnce({ data: { data: [] } }) // /admin/datastore/{store}/groups (empty) .mockRejectedValueOnce(taskError); // /nodes/{node}/tasks (FAILS) const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); const result = await fetchPbsData(mockClients); expect(result).toHaveLength(1); expect(result[0]).toMatchObject({ pbsEndpointId: pbsId, pbsInstanceName: 'PBS Task Fetch Error', status: 'ok', // Status remains 'ok' as node and datastores (even if empty) fetched nodeName: pbsNodeName, datastores: [{ name: datastoreName, total: 1, used: 0, available: undefined, gcStatus: 'unknown' , snapshots: []}], // Datastore fetch succeeded }); // Check that task-related properties have default structure due to fetch error expect(result[0]).toHaveProperty('backupTasks'); expect(result[0]).toHaveProperty('verifyTasks'); expect(result[0]).toHaveProperty('gcTasks'); // Verify API calls expect(mockPbsClient.get).toHaveBeenCalledTimes(6); // nodes, usage, snapshots, usage(dedup), config/datastore, tasks (fails) - no groups because datastores are empty expect(mockPbsClient.get).toHaveBeenCalledWith(`/nodes/${pbsNodeName}/tasks`, expect.any(Object)); // Check tasks call was attempted // Verify error logging from fetchAllPbsTasksForProcessing's catch block (covers lines 264-265) expect(consoleErrorSpy).toHaveBeenCalledWith( expect.stringContaining(`Failed to fetch PBS task list for node ${pbsNodeName} (PBS Task Fetch Error): ${taskError.message}`) ); // Verify the warning logged in fetchPbsData when tasks cannot be processed (covers line 341) expect(consoleWarnSpy).toHaveBeenCalledWith( expect.stringContaining('No tasks to process or task fetching failed. Error flag: true, Tasks array: null') ); consoleErrorSpy.mockRestore(); consoleWarnSpy.mockRestore(); }); }); // End describe fetchPbsData }); // End describe Data Fetcher