fix: reduce prune estimate work and add managed-scope timeout (#1768)

* fix: reduce prune estimate work and add managed-scope timeout

estimateSystemReclaim previously called getDiskUsageClassified,
which walks the full classified-resources pipeline (6+ Docker
API calls and filesystem I/O) under the 8 s timeout, but only
reads the three reclaimable* fields that getDiskUsage() (a single
docker.df() call) already provides. Switch to getDiskUsage() so
the timeout actually bounds the work the comment describes.

Additionally, the managed-scope estimateManagedReclaim path had
no timeout on either the remote route or the fleet local path.
Wrap both call sites in withTimeout so a slow daemon surfaces
the actionable 'Docker daemon is busy' message within 8 s instead
of hanging until the hub's 15 s fetch abort fires.

* fix: skip getStacks() for all scope in prune estimate route

The remote handler unconditionally walked the compose directory before
starting the 8 s estimate timer, but for 'all' scope the knownStackNames
parameter is now unused (estimateSystemReclaim uses only docker system df).
Mirror the fleet route's conditional so the walk only happens for managed
scope, where estimateManagedReclaim genuinely needs stack names.

Found during QA: on a Pilot node with real tunnel latency, this unbounded
walk added latency outside the timeout budget.

* fix: raise prune estimate budget to 12s for large image stores

docker.df() cost scales with image-store size: measured ~7.4s on a
34GB / 96-image store, alone nearly exhausting the previous 8s budget
before tunnel transport overhead. A healthy Pilot node could flip to
'Docker daemon is busy' at idle load.

Raise PRUNE_ESTIMATE_TIMEOUT_MS and FLEET_DF_TIMEOUT_MS to 12s, which
sits strictly below the hub's 15s AbortSignal.timeout on the fleet
estimate fetch, keeping the remote 503 the actionable failure. The
MonitorService janitor keeps its own 8s budget for destructive paths.

Found in QA pass 2: single-target estimate failed at ~8.05s on a node
where docker.df() alone takes ~7.4s.
This commit is contained in:
Anso
2026-08-05 13:01:14 -04:00
committed by GitHub
parent 85b841175a
commit e2fc3a58a0
6 changed files with 136 additions and 45 deletions
@@ -382,6 +382,48 @@ describe('DockerController - getDiskUsage', () => {
expect(usage.reclaimableContainers).toBe(1_500);
expect(usage.reclaimableContainerCount).toBe(2);
});
it('estimateSystemReclaim returns same bytes as getDiskUsage for every target type', async () => {
mockDocker.df.mockResolvedValue({
LayersSize: 1_000_000_000,
Images: [
{ Id: 'a', Containers: 0, Size: 500_000_000, SharedSize: 0 },
{ Id: 'b', Containers: 1, Size: 300_000_000, SharedSize: 0 },
],
Containers: [
{ State: 'exited', SizeRw: 200 },
],
Volumes: [
{ UsageData: { RefCount: 0, Size: 400 } },
],
});
const dc = DockerController.getInstance(1);
const usage = await dc.getDiskUsage();
// Each estimateSystemReclaim(target) must equal the corresponding
// getDiskUsage() field. knownStackNames is unused in the fast path
// but retained for call-site symmetry.
const expectations: Array<[target: 'images' | 'containers' | 'volumes' | 'networks', expected: number]> = [
['images', usage.reclaimableImages],
['containers', usage.reclaimableContainers],
['volumes', usage.reclaimableVolumes],
['networks', 0],
];
for (const [target, expected] of expectations) {
const estimate = await dc.estimateSystemReclaim(target, []);
expect(estimate.reclaimableBytes).toBe(expected);
}
// The fast path uses only docker.df(), never the classified-resources
// API calls that estimateSystemReclaim previously incurred via
// getDiskUsageClassified (listImages, listVolumes, etc.).
expect(mockDocker.df).toHaveBeenCalledTimes(5); // 1 getDiskUsage + 4 estimateSystemReclaim
expect(mockDocker.listImages).not.toHaveBeenCalled();
expect(mockDocker.listVolumes).not.toHaveBeenCalled();
expect(mockDocker.listNetworks).not.toHaveBeenCalled();
expect(mockDocker.listContainers).not.toHaveBeenCalled();
});
});
// ── pruneSystem ────────────────────────────────────────────────────────
@@ -1,6 +1,6 @@
/**
* F-6 regression: Fleet itemized plan enumeration and byte estimation both
* bound the slow `docker system df` call (8s) and surface a recognizable
* bound the slow `docker system df` call (12s) and surface a recognizable
* timeout message to the operator.
*
* Covers:
@@ -9,7 +9,7 @@
*
* Uses real timers because supertest dispatches lazily and the in-route
* `withTimeout` setTimeout cannot be advanced via vi.useFakeTimers from
* outside the request lifecycle.
* outside the request lifecycle. Three timeout tests add ~25s to the file.
*/
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
import request from 'supertest';
@@ -29,7 +29,7 @@ beforeAll(async () => {
({ default: DockerController } = await import('../services/DockerController'));
({ FileSystemService } = await import('../services/FileSystemService'));
({ activeBulkActions } = await import('../routes/labels'));
// 10-minute expiry survives the file even with two ~8s timeout tests.
// 10-minute expiry survives the file even with three ~12s timeout tests.
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '10m' });
authHeader = `Bearer ${token}`;
});
@@ -56,7 +56,7 @@ function stubLocalEstimate(
vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue([]);
}
describe('Fleet prune routes bound docker df at 8s on local nodes (F-6)', () => {
describe('Fleet prune routes bound docker df at 12s on local nodes (F-6)', () => {
it('POST /api/fleet/labels/fleet-prune dry-run surfaces a busy-daemon error on local timeout', async () => {
stubLocalEstimate(
() => Promise.resolve({ reclaimableBytes: 0 }),
@@ -94,6 +94,27 @@ describe('Fleet prune routes bound docker df at 8s on local nodes (F-6)', () =>
expect(local.error).toMatch(/Docker daemon is busy/);
}, 20_000);
it('POST /api/fleet/prune/estimate marks the local node unreachable on managed timeout', async () => {
vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue([]);
// estimateManagedReclaim never settles so the managed path hits
// FLEET_DF_TIMEOUT_MS and surfaces the busy-daemon error.
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
estimateManagedReclaim: vi.fn().mockImplementation(() => new Promise(() => { /* never resolves */ })),
estimateSystemReclaim: vi.fn().mockResolvedValue({ reclaimableBytes: 0 }),
} as unknown as ReturnType<typeof DockerController.getInstance>);
const res = await request(app)
.post('/api/fleet/prune/estimate')
.set('Authorization', authHeader)
.send({ targets: ['images'], scope: 'managed' });
expect(res.status).toBe(200);
expect(Array.isArray(res.body.perNode)).toBe(true);
const local = res.body.perNode[0];
expect(local.reachable).toBe(false);
expect(local.error).toMatch(/Docker daemon is busy/);
}, 20_000);
it('fleet-prune dry-run succeeds normally when estimateSystemReclaim resolves quickly', async () => {
stubLocalEstimate(
() => Promise.resolve({ reclaimableBytes: 256 }),
@@ -4,7 +4,7 @@
*
* Uses real timers because supertest dispatches lazily and vi.useFakeTimers
* does not compose cleanly with that pattern. Each timeout test waits the
* full 8s withTimeout budget, so two such tests add ~17s to the file.
* full 12s withTimeout budget, so three such tests add ~36s to the file.
*/
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
import request from 'supertest';
@@ -45,9 +45,12 @@ function stubFsStacks() {
} as unknown as ReturnType<typeof FileSystemService.getInstance>);
}
function stubEstimate(impl: () => Promise<{ reclaimableBytes: number }>) {
function stubEstimate(
impl: () => Promise<{ reclaimableBytes: number }>,
method: 'estimateSystemReclaim' | 'estimateManagedReclaim' = 'estimateSystemReclaim',
) {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
estimateSystemReclaim: vi.fn().mockImplementation(impl),
[method]: vi.fn().mockImplementation(impl),
} as unknown as ReturnType<typeof DockerController.getInstance>);
}
@@ -122,6 +125,27 @@ describe('Prune estimate endpoints return 503 on slow docker df (F-6)', () => {
expect(res.status).toBe(500);
expect(res.body.code).not.toBe('docker_df_slow');
});
it('managed images estimate returns 503 docker_df_slow when estimateManagedReclaim never settles', async () => {
stubFsStacks();
stubEstimate(
() => new Promise(() => { /* never resolves */ }),
'estimateManagedReclaim',
);
const t0 = Date.now();
const res = await request(app)
.post('/api/system/prune/estimate')
.set('Authorization', authHeader)
.send({ target: 'images', scope: 'managed' });
const elapsed = Date.now() - t0;
expect(res.status).toBe(503);
expect(res.body.code).toBe('docker_df_slow');
expect(res.body.error).toMatch(/Docker daemon is busy/);
expect(elapsed).toBeGreaterThanOrEqual(7_500);
expect(elapsed).toBeLessThan(15_000);
}, 20_000);
});
describe('Prune plan routes', () => {