fix: keep fleet prune estimate bytes after a target timeout (#1783)

* fix: keep fleet prune estimate bytes after a target timeout

Accumulate successful per-target reclaimable bytes on both the local
serial loop and the remote fan-out, mark partial nodes, and surface that
state in the Fleet prune card instead of zeroing the whole node.

* test: clarify zero-byte partial estimate card case

Rename the FleetPruneCard assertion so it matches fold semantics:
partial with zero bytes means a successful zero-byte target plus a failure,
not an all-failed node.

* fix: keep fleet prune estimate resilient on init and partial errors

Guard DockerController init so a deleted-node race cannot 500 the whole
fleet estimate, surface partial-node failure text in the card row title,
and pin the generic-rejection partial path with a fast route test.

* test: destroy reverse-route connect-ack sockets on teardown

Unguarded accepted sockets could emit a late ECONNRESET after assertions passed, failing the Backend Vitest job with zero assertion failures.
This commit is contained in:
Anso
2026-08-05 22:11:03 -04:00
committed by GitHub
parent 4e54a9272d
commit 575848e017
8 changed files with 386 additions and 37 deletions
@@ -558,6 +558,98 @@ describe('POST /api/fleet/prune/estimate', () => {
db.deleteNode(remoteId);
}
});
it('keeps successful remote bytes when one per-target estimate fails', async () => {
const remoteId = db.addNode({
name: 'remote-partial',
type: 'remote',
api_url: 'http://remote-partial.example:1852',
api_token: 'tok',
compose_dir: '/app/compose',
is_default: false,
});
try {
vi.spyOn(globalThis, 'fetch').mockImplementation(async (_url, init) => {
const body = JSON.parse(String(init?.body ?? '{}')) as { target?: string };
if (body.target === 'images') {
return new Response(JSON.stringify({ reclaimableBytes: 42 }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
});
}
return new Response(JSON.stringify({ error: 'Docker daemon is busy', code: 'docker_df_slow' }), {
status: 503,
headers: { 'Content-Type': 'application/json' },
});
});
const res = await request(app)
.post('/api/fleet/prune/estimate')
.set('Authorization', authHeader)
.send({ targets: ['images', 'volumes'], scope: 'managed' });
expect(res.status).toBe(200);
const remote = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
expect(remote.reachable).toBe(true);
expect(remote.partial).toBe(true);
expect(remote.reclaimableBytes).toBe(42);
expect(remote.error).toMatch(/Docker daemon is busy|503/);
expect(res.body.totalBytes).toBeGreaterThanOrEqual(42);
} finally {
db.deleteNode(remoteId);
}
});
it('keeps successful local bytes when a later target rejects without timing out', async () => {
estimateManagedReclaim.mockImplementation(async (target: string) => {
if (target === 'images') return { reclaimableBytes: 4096 };
throw new Error('enumeration failed');
});
const res = await request(app)
.post('/api/fleet/prune/estimate')
.set('Authorization', authHeader)
.send({ targets: ['images', 'volumes'], scope: 'managed' });
expect(res.status).toBe(200);
expect(res.body.perNode[0].reachable).toBe(true);
expect(res.body.perNode[0].partial).toBe(true);
expect(res.body.perNode[0].reclaimableBytes).toBe(4096);
expect(res.body.perNode[0].error).toMatch(/enumeration failed/);
expect(res.body.totalBytes).toBe(4096);
});
it('marks the local node unreachable when DockerController init fails without 500ing the fleet', async () => {
const remoteId = db.addNode({
name: 'remote-survives-init',
type: 'remote',
api_url: 'http://remote-survives.example:1852',
api_token: 'tok',
compose_dir: '/app/compose',
is_default: false,
});
try {
const DockerController = (await import('../services/DockerController')).default;
vi.mocked(DockerController.getInstance).mockImplementationOnce(() => {
throw new Error('Node with id 1 not found');
});
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(
JSON.stringify({ reclaimableBytes: 99 }),
{ status: 200, headers: { 'Content-Type': 'application/json' } },
));
const res = await request(app)
.post('/api/fleet/prune/estimate')
.set('Authorization', authHeader)
.send({ targets: ['images'], scope: 'managed' });
expect(res.status).toBe(200);
const local = res.body.perNode.find((n: { nodeId: number; nodeName: string }) => n.nodeName !== 'remote-survives-init');
expect(local.reachable).toBe(false);
expect(local.reclaimableBytes).toBe(0);
expect(local.error).toMatch(/not found|Failed to estimate locally/);
const remote = res.body.perNode.find((n: { nodeId: number }) => n.nodeId === remoteId);
expect(remote.reachable).toBe(true);
expect(remote.reclaimableBytes).toBe(99);
expect(res.body.totalBytes).toBe(99);
} finally {
db.deleteNode(remoteId);
}
});
});
describe('POST /api/fleet/labels/fleet-stop remote leg', () => {
@@ -0,0 +1,58 @@
import { describe, it, expect } from 'vitest';
import { foldNodeEstimate } from '../helpers/fleetEstimate';
const node = { nodeId: 1, nodeName: 'local' };
describe('foldNodeEstimate', () => {
it('returns the sum when every target succeeds', () => {
expect(foldNodeEstimate(node, [
{ bytes: 500 },
{ bytes: 100 },
])).toEqual({
nodeId: 1,
nodeName: 'local',
reclaimableBytes: 600,
reachable: true,
});
});
it('keeps successful bytes and marks partial when some targets fail', () => {
expect(foldNodeEstimate(node, [
{ bytes: 500 },
{ bytes: 0, error: 'Docker daemon is busy. Please try again in a moment.' },
{ bytes: 100 },
])).toEqual({
nodeId: 1,
nodeName: 'local',
reclaimableBytes: 600,
reachable: true,
partial: true,
error: 'Docker daemon is busy. Please try again in a moment.',
});
});
it('marks the node unreachable when every target fails', () => {
expect(foldNodeEstimate(node, [
{ bytes: 0, error: 'first failure' },
{ bytes: 0, error: 'second failure' },
])).toEqual({
nodeId: 1,
nodeName: 'local',
reclaimableBytes: 0,
reachable: false,
error: 'first failure',
});
});
it('surfaces the single-target failure message without partial', () => {
expect(foldNodeEstimate(node, [
{ bytes: 0, error: 'Docker daemon is busy. Please try again in a moment.' },
])).toEqual({
nodeId: 1,
nodeName: 'local',
reclaimableBytes: 0,
reachable: false,
error: 'Docker daemon is busy. Please try again in a moment.',
});
});
});
@@ -9,7 +9,8 @@
*
* Uses real timers because supertest dispatches lazily and the in-route
* `withTimeout` setTimeout cannot be advanced via vi.useFakeTimers from
* outside the request lifecycle. Three timeout tests add ~25s to the file.
* outside the request lifecycle. Five timeout tests add ~49s to the file
* (three single-target ~12s each, plus two multi-target partial ~12s each).
*/
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
import request from 'supertest';
@@ -29,7 +30,7 @@ beforeAll(async () => {
({ default: DockerController } = await import('../services/DockerController'));
({ FileSystemService } = await import('../services/FileSystemService'));
({ activeBulkActions } = await import('../routes/labels'));
// 10-minute expiry survives the file even with three ~12s timeout tests.
// 10-minute expiry survives the file even with five ~12s timeout tests.
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '10m' });
authHeader = `Bearer ${token}`;
});
@@ -115,6 +116,66 @@ describe('Fleet prune routes bound docker df at 12s on local nodes (F-6)', () =>
expect(local.error).toMatch(/Docker daemon is busy/);
}, 20_000);
it('POST /api/fleet/prune/estimate keeps successful all-scope bytes when a later target times out', async () => {
const estimateSystemReclaim = vi.fn().mockImplementation(async (target: string) => {
if (target === 'images') return { reclaimableBytes: 500 };
if (target === 'volumes') return new Promise(() => { /* never resolves */ });
return { reclaimableBytes: 100 };
});
vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue([]);
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
estimateSystemReclaim,
estimateManagedReclaim: vi.fn().mockResolvedValue({ reclaimableBytes: 0 }),
} as unknown as ReturnType<typeof DockerController.getInstance>);
const t0 = Date.now();
const res = await request(app)
.post('/api/fleet/prune/estimate')
.set('Authorization', authHeader)
.send({ targets: ['images', 'volumes', 'networks'], scope: 'all' });
const elapsed = Date.now() - t0;
expect(res.status).toBe(200);
const local = res.body.perNode[0];
expect(local.reachable).toBe(true);
expect(local.partial).toBe(true);
expect(local.reclaimableBytes).toBe(600);
expect(local.error).toMatch(/Docker daemon is busy/);
expect(estimateSystemReclaim).toHaveBeenCalledTimes(3);
expect(elapsed).toBeGreaterThanOrEqual(11_500);
expect(elapsed).toBeLessThan(24_000);
}, 30_000);
it('POST /api/fleet/prune/estimate keeps successful managed-scope bytes when a later target times out', async () => {
const estimateManagedReclaim = vi.fn().mockImplementation(async (target: string) => {
if (target === 'images') return { reclaimableBytes: 500 };
if (target === 'volumes') return new Promise(() => { /* never resolves */ });
return { reclaimableBytes: 100 };
});
vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue([]);
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
estimateManagedReclaim,
estimateSystemReclaim: vi.fn().mockResolvedValue({ reclaimableBytes: 0 }),
} as unknown as ReturnType<typeof DockerController.getInstance>);
const t0 = Date.now();
const res = await request(app)
.post('/api/fleet/prune/estimate')
.set('Authorization', authHeader)
.send({ targets: ['images', 'volumes', 'networks'], scope: 'managed' });
const elapsed = Date.now() - t0;
expect(res.status).toBe(200);
const local = res.body.perNode[0];
expect(local.reachable).toBe(true);
expect(local.partial).toBe(true);
expect(local.reclaimableBytes).toBe(600);
expect(local.error).toMatch(/Docker daemon is busy/);
expect(estimateManagedReclaim).toHaveBeenCalledTimes(3);
expect(elapsed).toBeGreaterThanOrEqual(11_500);
expect(elapsed).toBeLessThan(24_000);
}, 30_000);
it('fleet-prune dry-run succeeds normally when estimateSystemReclaim resolves quickly', async () => {
stubLocalEstimate(
() => Promise.resolve({ reclaimableBytes: 256 }),
@@ -189,7 +189,13 @@ describe('R1-B: PilotTunnelBridge.acceptReverseLocal route events', () => {
await bridge.start();
// Real local server so the dial succeeds and 'connect' fires.
// Track accepted sockets and swallow teardown noise: closing the
// bridge while a peer still holds the connection can emit a late
// ECONNRESET with no listener, which Vitest treats as a suite failure.
const accepted: net.Socket[] = [];
const upstream = net.createServer((socket) => {
accepted.push(socket);
socket.on('error', () => { /* expected post-handshake teardown */ });
socket.write('hello-upstream');
});
await new Promise<void>((resolve) => upstream.listen(0, '127.0.0.1', () => resolve()));
@@ -237,7 +243,10 @@ describe('R1-B: PilotTunnelBridge.acceptReverseLocal route events', () => {
targetPort: upstreamPort,
});
upstream.close();
for (const socket of accepted) {
try { socket.destroy(); } catch { /* ignore */ }
}
await new Promise<void>((resolve) => upstream.close(() => resolve()));
bridge.close();
});
+61
View File
@@ -0,0 +1,61 @@
/**
* Live fleet prune-estimate folding: accumulate per-target byte results into
* one per-node estimate without discarding successes when a later target fails.
*/
export type FleetEstimateTargetResult = {
bytes: number;
error?: string;
};
export type FleetNodeEstimate = {
nodeId: number;
nodeName: string;
reclaimableBytes: number;
reachable: boolean;
error?: string;
partial?: true;
};
export function foldNodeEstimate(
node: { nodeId: number; nodeName: string },
perTarget: ReadonlyArray<FleetEstimateTargetResult>,
): FleetNodeEstimate {
let reclaimableBytes = 0;
let successCount = 0;
let firstError: string | undefined;
for (const entry of perTarget) {
if (entry.error) {
if (firstError === undefined) firstError = entry.error;
continue;
}
reclaimableBytes += entry.bytes;
successCount += 1;
}
const failCount = perTarget.length - successCount;
if (successCount === 0) {
return {
nodeId: node.nodeId,
nodeName: node.nodeName,
reclaimableBytes: 0,
reachable: false,
error: firstError,
};
}
if (failCount > 0) {
return {
nodeId: node.nodeId,
nodeName: node.nodeName,
reclaimableBytes,
reachable: true,
partial: true,
error: firstError,
};
}
return {
nodeId: node.nodeId,
nodeName: node.nodeName,
reclaimableBytes,
reachable: true,
};
}
+35 -29
View File
@@ -36,6 +36,7 @@ import { parseIntParam } from '../utils/parseIntParam';
import { parseRequestedTargetVersion, pickCompareTarget } from '../utils/targetVersion';
import { buildTargetImageRef, isRepinBlocked, type ImagePinKind } from '../helpers/selfUpdateCompose';
import { withTimeout, TimeoutError } from '../utils/withTimeout';
import { foldNodeEstimate, type FleetEstimateTargetResult, type FleetNodeEstimate } from '../helpers/fleetEstimate';
// Mirror the system-maintenance route timeout so fleet's local-node prune
// paths cap the slow `docker system df` call at the same 12 s budget (F-6).
@@ -2331,7 +2332,8 @@ fleetRouter.get('/labels/suggestions', authMiddleware, async (req: Request, res:
// Fleet-wide prune size estimate. Local node uses the controller estimate
// helper; remote nodes hit `/api/system/prune/estimate` per target. Same
// fan-out shape as `/labels/fleet-prune` minus the locks (estimation is read
// only).
// only). Per-target failures keep successful targets' bytes via foldNodeEstimate
// rather than zeroing the whole node.
fleetRouter.post('/prune/estimate', authMiddleware, async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
@@ -2356,38 +2358,47 @@ fleetRouter.post('/prune/estimate', authMiddleware, async (req: Request, res: Re
const targets: FleetPruneTarget[] = Array.from(dedup);
const scope: 'managed' | 'all' = body.scope === 'all' ? 'all' : 'managed';
type NodeEstimate = {
nodeId: number; nodeName: string; reclaimableBytes: number; reachable: boolean; error?: string;
};
try {
const db = DatabaseService.getInstance();
const nodes = db.getNodes();
const perNode: NodeEstimate[] = await Promise.all(nodes.map(async (node): Promise<NodeEstimate> => {
const perNode: FleetNodeEstimate[] = await Promise.all(nodes.map(async (node): Promise<FleetNodeEstimate> => {
if (node.type === 'local') {
let knownStacks: string[];
let dockerController: ReturnType<typeof DockerController.getInstance>;
try {
const knownStacks = scope === 'managed' ? await FileSystemService.getInstance(node.id).getStacks() : [];
const dockerController = DockerController.getInstance(node.id);
let nodeBytes = 0;
for (const target of targets) {
// Bound so a slow local daemon does not hang the fleet
// estimate (F-6). Both managed and all scopes bound each
// per-target call at the same 12 s.
knownStacks = scope === 'managed' ? await FileSystemService.getInstance(node.id).getStacks() : [];
// Init can throw if the node row disappears mid-request; keep that
// failure per-node so other fleet estimates still return.
dockerController = DockerController.getInstance(node.id);
} catch (err) {
return {
nodeId: node.id,
nodeName: node.name,
reclaimableBytes: 0,
reachable: false,
error: getErrorMessage(err, 'Failed to estimate locally'),
};
}
const perTarget: FleetEstimateTargetResult[] = [];
for (const target of targets) {
// Bound so a slow local daemon does not hang the fleet
// estimate (F-6). Both managed and all scopes bound each
// per-target call at the same 12 s. Failures stay per-target so
// earlier successes are not discarded.
try {
const estimate = scope === 'managed'
? dockerController.estimateManagedReclaim(target, knownStacks)
: dockerController.estimateSystemReclaim(target, knownStacks);
const result = await withTimeout(estimate, FLEET_DF_TIMEOUT_MS, 'docker disk usage');
nodeBytes += result.reclaimableBytes;
perTarget.push({ bytes: result.reclaimableBytes });
} catch (err) {
const error = err instanceof TimeoutError
? 'Docker daemon is busy. Please try again in a moment.'
: getErrorMessage(err, 'Failed to estimate locally');
perTarget.push({ bytes: 0, error });
}
return { nodeId: node.id, nodeName: node.name, reclaimableBytes: nodeBytes, reachable: true };
} catch (err) {
const error = err instanceof TimeoutError
? 'Docker daemon is busy. Please try again in a moment.'
: getErrorMessage(err, 'Failed to estimate locally');
return {
nodeId: node.id, nodeName: node.name, reclaimableBytes: 0, reachable: false, error,
};
}
return foldNodeEstimate({ nodeId: node.id, nodeName: node.name }, perTarget);
}
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id);
@@ -2404,7 +2415,7 @@ fleetRouter.post('/prune/estimate', authMiddleware, async (req: Request, res: Re
// so wall time matches the slowest single call rather than the sum.
// (The destructive sibling stays serial because Docker prune is internally
// serialized and one failure should short-circuit later targets there.)
const perTarget = await Promise.all(targets.map(async (target): Promise<{ bytes: number; error?: string }> => {
const perTarget = await Promise.all(targets.map(async (target): Promise<FleetEstimateTargetResult> => {
try {
const response = await fetch(`${baseUrl}/api/system/prune/estimate`, {
method: 'POST',
@@ -2425,12 +2436,7 @@ fleetRouter.post('/prune/estimate', authMiddleware, async (req: Request, res: Re
return { bytes: 0, error: getErrorMessage(err, 'Failed to reach remote node') };
}
}));
const firstError = perTarget.find(t => t.error)?.error;
if (firstError) {
return { nodeId: node.id, nodeName: node.name, reclaimableBytes: 0, reachable: false, error: firstError };
}
const nodeBytes = perTarget.reduce((sum, t) => sum + t.bytes, 0);
return { nodeId: node.id, nodeName: node.name, reclaimableBytes: nodeBytes, reachable: true };
return foldNodeEstimate({ nodeId: node.id, nodeName: node.name }, perTarget);
}));
const totalBytes = perNode.reduce((acc, n) => acc + (n.reachable ? n.reclaimableBytes : 0), 0);
@@ -71,10 +71,61 @@ beforeEach(() => {
it('keeps destructive prune disabled until an itemized dry run is reviewed', async () => {
render(<FleetPruneCard nodes={nodes} />);
await waitFor(() => expect(screen.getByText('~ 4 KB reclaimable')).toBeInTheDocument());
expect(screen.queryByText(/· partial/)).not.toBeInTheDocument();
expect(screen.getByText('OK')).toBeInTheDocument();
expect(screen.queryByText('PART')).not.toBeInTheDocument();
expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeDisabled();
expect(screen.getByText(/Run Dry run to unlock Prune fleet/)).toBeInTheDocument();
});
it('marks partial estimates with a PART pill and blast suffix', async () => {
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/prune/estimate') {
return Promise.resolve(jsonResponse(200, {
totalBytes: 500,
perNode: [{
nodeId: 1,
nodeName: 'central',
reclaimableBytes: 500,
reachable: true,
partial: true,
error: 'Docker daemon is busy. Please try again in a moment.',
}],
}));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<FleetPruneCard nodes={nodes} />);
await waitFor(() => expect(screen.getByText('~ 500 Bytes reclaimable · partial')).toBeInTheDocument());
expect(screen.getByText('PART')).toBeInTheDocument();
expect(screen.queryByText('OK')).not.toBeInTheDocument();
expect(screen.getByTitle('Docker daemon is busy. Please try again in a moment.')).toBeInTheDocument();
});
it('shows 0 reclaimable · partial when successful targets report zero bytes', async () => {
mockedFetch.mockImplementation((url: string) => {
if (url === '/fleet/prune/estimate') {
return Promise.resolve(jsonResponse(200, {
totalBytes: 0,
perNode: [{
nodeId: 1,
nodeName: 'central',
reclaimableBytes: 0,
reachable: true,
partial: true,
error: 'Docker daemon is busy. Please try again in a moment.',
}],
}));
}
return Promise.resolve(jsonResponse(404, {}));
});
render(<FleetPruneCard nodes={nodes} />);
await waitFor(() => expect(screen.getByText('0 reclaimable · partial')).toBeInTheDocument());
expect(screen.getByText('PART')).toBeInTheDocument();
});
it('drops the unlock footer once dry run review is valid', async () => {
const user = userEvent.setup();
mockedFetch.mockImplementation((url: string) => {
@@ -28,6 +28,7 @@ interface PruneEstimateNode {
reclaimableBytes: number;
reachable: boolean;
error?: string;
partial?: boolean;
}
interface PruneEstimateResponse {
@@ -287,8 +288,10 @@ export function FleetPruneCard({ nodes }: Props) {
if (estimate.kind === 'loading') return '~ estimating…';
if (estimate.kind === 'unavailable') return '~ estimate unavailable';
if (estimate.kind === 'ready') {
if (estimate.data.totalBytes === 0) return '0 reclaimable';
return `~ ${formatBytes(estimate.data.totalBytes)} reclaimable`;
const partial = estimate.data.perNode.some((node) => node.partial);
const suffix = partial ? ' · partial' : '';
if (estimate.data.totalBytes === 0) return `0 reclaimable${suffix}`;
return `~ ${formatBytes(estimate.data.totalBytes)} reclaimable${suffix}`;
}
return 'awaiting target';
}, [targets.size, estimate]);
@@ -396,13 +399,21 @@ function EstimateSection({ estimate }: { estimate: EstimateState }) {
<div className="rounded border border-card-border/60 bg-card/40 p-2 shadow-[inset_0_2px_4px_0_oklch(0_0_0_/_0.35)]">
<ul className="space-y-1">
{visible.map((node) => (
<li key={node.nodeId} className="flex items-center gap-2">
<li
key={node.nodeId}
className="flex items-center gap-2"
title={node.error}
>
<span className={cn(
KICKER,
'inline-flex shrink-0 items-center rounded-sm border px-1 py-0.5',
node.reachable ? 'border-success/40 bg-success/10 text-success' : 'border-stat-subtitle/40 bg-card text-stat-subtitle',
node.reachable
? (node.partial
? 'border-warning/40 bg-warning/10 text-warning'
: 'border-success/40 bg-success/10 text-success')
: 'border-stat-subtitle/40 bg-card text-stat-subtitle',
)}>
{node.reachable ? 'OK' : '--'}
{node.reachable ? (node.partial ? 'PART' : 'OK') : '--'}
</span>
<span className="min-w-0 flex-1 truncate font-mono text-[11px] text-stat-value">{node.nodeName}</span>
<span className={cn(KICKER, 'shrink-0 tabular-nums', node.reachable ? 'text-stat-subtitle' : 'text-stat-icon')}>