fix: keep fleet prune estimate bytes after a target timeout (#1783)

* fix: keep fleet prune estimate bytes after a target timeout

Accumulate successful per-target reclaimable bytes on both the local
serial loop and the remote fan-out, mark partial nodes, and surface that
state in the Fleet prune card instead of zeroing the whole node.

* test: clarify zero-byte partial estimate card case

Rename the FleetPruneCard assertion so it matches fold semantics:
partial with zero bytes means a successful zero-byte target plus a failure,
not an all-failed node.

* fix: keep fleet prune estimate resilient on init and partial errors

Guard DockerController init so a deleted-node race cannot 500 the whole
fleet estimate, surface partial-node failure text in the card row title,
and pin the generic-rejection partial path with a fast route test.

* test: destroy reverse-route connect-ack sockets on teardown

Unguarded accepted sockets could emit a late ECONNRESET after assertions passed, failing the Backend Vitest job with zero assertion failures.

backend/src/__tests__/fleet-action-card-endpoints.test.ts

@@ -558,6 +558,98 @@ describe('POST /api/fleet/prune/estimate', () => {
       db.deleteNode(remoteId);
     }
   });
+
+  it('keeps successful remote bytes when one per-target estimate fails', async () => {
+    const remoteId = db.addNode({
+      name: 'remote-partial',
+      type: 'remote',
+      api_url: 'http://remote-partial.example:1852',
+      api_token: 'tok',
+      compose_dir: '/app/compose',
+      is_default: false,
+    });
+    try {
+      vi.spyOn(globalThis, 'fetch').mockImplementation(async (_url, init) => {
+        const body = JSON.parse(String(init?.body ?? '{}')) as { target?: string };
+        if (body.target === 'images') {
+          return new Response(JSON.stringify({ reclaimableBytes: 42 }), {
+            status: 200,
+            headers: { 'Content-Type': 'application/json' },
+          });
+        }
+        return new Response(JSON.stringify({ error: 'Docker daemon is busy', code: 'docker_df_slow' }), {
+          status: 503,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      });
+      const res = await request(app)
+        .post('/api/fleet/prune/estimate')
+        .set('Authorization', authHeader)
+        .send({ targets: ['images', 'volumes'], scope: 'managed' });
+      expect(res.status).toBe(200);
+      const remote = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
+      expect(remote.reachable).toBe(true);
+      expect(remote.partial).toBe(true);
+      expect(remote.reclaimableBytes).toBe(42);
+      expect(remote.error).toMatch(/Docker daemon is busy|503/);
+      expect(res.body.totalBytes).toBeGreaterThanOrEqual(42);
+    } finally {
+      db.deleteNode(remoteId);
+    }
+  });
+
+  it('keeps successful local bytes when a later target rejects without timing out', async () => {
+    estimateManagedReclaim.mockImplementation(async (target: string) => {
+      if (target === 'images') return { reclaimableBytes: 4096 };
+      throw new Error('enumeration failed');
+    });
+    const res = await request(app)
+      .post('/api/fleet/prune/estimate')
+      .set('Authorization', authHeader)
+      .send({ targets: ['images', 'volumes'], scope: 'managed' });
+    expect(res.status).toBe(200);
+    expect(res.body.perNode[0].reachable).toBe(true);
+    expect(res.body.perNode[0].partial).toBe(true);
+    expect(res.body.perNode[0].reclaimableBytes).toBe(4096);
+    expect(res.body.perNode[0].error).toMatch(/enumeration failed/);
+    expect(res.body.totalBytes).toBe(4096);
+  });
+
+  it('marks the local node unreachable when DockerController init fails without 500ing the fleet', async () => {
+    const remoteId = db.addNode({
+      name: 'remote-survives-init',
+      type: 'remote',
+      api_url: 'http://remote-survives.example:1852',
+      api_token: 'tok',
+      compose_dir: '/app/compose',
+      is_default: false,
+    });
+    try {
+      const DockerController = (await import('../services/DockerController')).default;
+      vi.mocked(DockerController.getInstance).mockImplementationOnce(() => {
+        throw new Error('Node with id 1 not found');
+      });
+      vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(
+        JSON.stringify({ reclaimableBytes: 99 }),
+        { status: 200, headers: { 'Content-Type': 'application/json' } },
+      ));
+      const res = await request(app)
+        .post('/api/fleet/prune/estimate')
+        .set('Authorization', authHeader)
+        .send({ targets: ['images'], scope: 'managed' });
+      expect(res.status).toBe(200);
+      const local = res.body.perNode.find((n: { nodeId: number; nodeName: string }) => n.nodeName !== 'remote-survives-init');
+      expect(local.reachable).toBe(false);
+      expect(local.reclaimableBytes).toBe(0);
+      expect(local.error).toMatch(/not found|Failed to estimate locally/);
+      const remote = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
+      expect(remote.reachable).toBe(true);
+      expect(remote.reclaimableBytes).toBe(99);
+      expect(res.body.totalBytes).toBe(99);
+    } finally {
+      db.deleteNode(remoteId);
+    }
+  });
 });
 
 describe('POST /api/fleet/labels/fleet-stop remote leg', () => {

backend/src/__tests__/fleet-estimate.test.ts

@@ -0,0 +1,58 @@
+import { describe, it, expect } from 'vitest';
+import { foldNodeEstimate } from '../helpers/fleetEstimate';
+
+const node = { nodeId: 1, nodeName: 'local' };
+
+describe('foldNodeEstimate', () => {
+  it('returns the sum when every target succeeds', () => {
+    expect(foldNodeEstimate(node, [
+      { bytes: 500 },
+      { bytes: 100 },
+    ])).toEqual({
+      nodeId: 1,
+      nodeName: 'local',
+      reclaimableBytes: 600,
+      reachable: true,
+    });
+  });
+
+  it('keeps successful bytes and marks partial when some targets fail', () => {
+    expect(foldNodeEstimate(node, [
+      { bytes: 500 },
+      { bytes: 0, error: 'Docker daemon is busy. Please try again in a moment.' },
+      { bytes: 100 },
+    ])).toEqual({
+      nodeId: 1,
+      nodeName: 'local',
+      reclaimableBytes: 600,
+      reachable: true,
+      partial: true,
+      error: 'Docker daemon is busy. Please try again in a moment.',
+    });
+  });
+
+  it('marks the node unreachable when every target fails', () => {
+    expect(foldNodeEstimate(node, [
+      { bytes: 0, error: 'first failure' },
+      { bytes: 0, error: 'second failure' },
+    ])).toEqual({
+      nodeId: 1,
+      nodeName: 'local',
+      reclaimableBytes: 0,
+      reachable: false,
+      error: 'first failure',
+    });
+  });
+
+  it('surfaces the single-target failure message without partial', () => {
+    expect(foldNodeEstimate(node, [
+      { bytes: 0, error: 'Docker daemon is busy. Please try again in a moment.' },
+    ])).toEqual({
+      nodeId: 1,
+      nodeName: 'local',
+      reclaimableBytes: 0,
+      reachable: false,
+      error: 'Docker daemon is busy. Please try again in a moment.',
+    });
+  });
+});

backend/src/__tests__/fleet-prune-df-timeout.test.ts

@@ -9,7 +9,8 @@
  *
  * Uses real timers because supertest dispatches lazily and the in-route
  * `withTimeout` setTimeout cannot be advanced via vi.useFakeTimers from
- * outside the request lifecycle. Three timeout tests add ~25s to the file.
+ * outside the request lifecycle. Five timeout tests add ~49s to the file
+ * (three single-target ~12s each, plus two multi-target partial ~12s each).
  */
 import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
 import request from 'supertest';
@@ -29,7 +30,7 @@ beforeAll(async () => {
   ({ default: DockerController } = await import('../services/DockerController'));
   ({ FileSystemService } = await import('../services/FileSystemService'));
   ({ activeBulkActions } = await import('../routes/labels'));
-  // 10-minute expiry survives the file even with three ~12s timeout tests.
+  // 10-minute expiry survives the file even with five ~12s timeout tests.
   const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '10m' });
   authHeader = `Bearer ${token}`;
 });
@@ -115,6 +116,66 @@ describe('Fleet prune routes bound docker df at 12s on local nodes (F-6)', () =>
     expect(local.error).toMatch(/Docker daemon is busy/);
   }, 20_000);
 
+  it('POST /api/fleet/prune/estimate keeps successful all-scope bytes when a later target times out', async () => {
+    const estimateSystemReclaim = vi.fn().mockImplementation(async (target: string) => {
+      if (target === 'images') return { reclaimableBytes: 500 };
+      if (target === 'volumes') return new Promise(() => { /* never resolves */ });
+      return { reclaimableBytes: 100 };
+    });
+    vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue([]);
+    vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+      estimateSystemReclaim,
+      estimateManagedReclaim: vi.fn().mockResolvedValue({ reclaimableBytes: 0 }),
+    } as unknown as ReturnType<typeof DockerController.getInstance>);
+
+    const t0 = Date.now();
+    const res = await request(app)
+      .post('/api/fleet/prune/estimate')
+      .set('Authorization', authHeader)
+      .send({ targets: ['images', 'volumes', 'networks'], scope: 'all' });
+    const elapsed = Date.now() - t0;
+
+    expect(res.status).toBe(200);
+    const local = res.body.perNode[0];
+    expect(local.reachable).toBe(true);
+    expect(local.partial).toBe(true);
+    expect(local.reclaimableBytes).toBe(600);
+    expect(local.error).toMatch(/Docker daemon is busy/);
+    expect(estimateSystemReclaim).toHaveBeenCalledTimes(3);
+    expect(elapsed).toBeGreaterThanOrEqual(11_500);
+    expect(elapsed).toBeLessThan(24_000);
+  }, 30_000);
+
+  it('POST /api/fleet/prune/estimate keeps successful managed-scope bytes when a later target times out', async () => {
+    const estimateManagedReclaim = vi.fn().mockImplementation(async (target: string) => {
+      if (target === 'images') return { reclaimableBytes: 500 };
+      if (target === 'volumes') return new Promise(() => { /* never resolves */ });
+      return { reclaimableBytes: 100 };
+    });
+    vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue([]);
+    vi.spyOn(DockerController, 'getInstance').mockReturnValue({
+      estimateManagedReclaim,
+      estimateSystemReclaim: vi.fn().mockResolvedValue({ reclaimableBytes: 0 }),
+    } as unknown as ReturnType<typeof DockerController.getInstance>);
+
+    const t0 = Date.now();
+    const res = await request(app)
+      .post('/api/fleet/prune/estimate')
+      .set('Authorization', authHeader)
+      .send({ targets: ['images', 'volumes', 'networks'], scope: 'managed' });
+    const elapsed = Date.now() - t0;
+
+    expect(res.status).toBe(200);
+    const local = res.body.perNode[0];
+    expect(local.reachable).toBe(true);
+    expect(local.partial).toBe(true);
+    expect(local.reclaimableBytes).toBe(600);
+    expect(local.error).toMatch(/Docker daemon is busy/);
+    expect(estimateManagedReclaim).toHaveBeenCalledTimes(3);
+    expect(elapsed).toBeGreaterThanOrEqual(11_500);
+    expect(elapsed).toBeLessThan(24_000);
+  }, 30_000);
+
   it('fleet-prune dry-run succeeds normally when estimateSystemReclaim resolves quickly', async () => {
     stubLocalEstimate(
       () => Promise.resolve({ reclaimableBytes: 256 }),

backend/src/__tests__/pilot-tunnel-bridge-reverse-route-events.test.ts

@@ -189,7 +189,13 @@ describe('R1-B: PilotTunnelBridge.acceptReverseLocal route events', () => {
         await bridge.start();
 
         // Real local server so the dial succeeds and 'connect' fires.
+        // Track accepted sockets and swallow teardown noise: closing the
+        // bridge while a peer still holds the connection can emit a late
+        // ECONNRESET with no listener, which Vitest treats as a suite failure.
+        const accepted: net.Socket[] = [];
         const upstream = net.createServer((socket) => {
+            accepted.push(socket);
+            socket.on('error', () => { /* expected post-handshake teardown */ });
             socket.write('hello-upstream');
         });
         await new Promise<void>((resolve) => upstream.listen(0, '127.0.0.1', () => resolve()));
@@ -237,7 +243,10 @@ describe('R1-B: PilotTunnelBridge.acceptReverseLocal route events', () => {
             targetPort: upstreamPort,
         });
 
-        upstream.close();
+        for (const socket of accepted) {
+            try { socket.destroy(); } catch { /* ignore */ }
+        }
+        await new Promise<void>((resolve) => upstream.close(() => resolve()));
         bridge.close();
     });