mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-31 04:38:11 +00:00
feat(fleet): show itemized prune plans (#1734)
* feat(fleet): itemize prune review plans Build and display fingerprint-bound prune candidates for every reviewed fleet node. Preflight all node plans before mutation and preserve detailed removed, skipped, failed, and partial outcomes. Add safe resource metadata projection, managed ownership attribution, runtime contract validation, transport parity coverage, and operator docs. Closes #1724 * fix(security): harden stack path lookup Use a Map for Compose working-directory ownership resolution so untrusted path strings cannot become object property writes. * fix(fleet): harden prune execution safeguards
This commit is contained in:
Anso
@@ -19,6 +19,8 @@ const pruneManagedOnly = vi.fn();
|
||||
const pruneSystem = vi.fn();
|
||||
const estimateManagedReclaim = vi.fn();
|
||||
const estimateSystemReclaim = vi.fn();
|
||||
const buildPrunePlan = vi.fn();
|
||||
const executePrunePlan = vi.fn();
|
||||
const getContainersByStack = vi.fn();
|
||||
const stopContainer = vi.fn();
|
||||
const restartContainer = vi.fn();
|
||||
@@ -47,6 +49,8 @@ vi.mock('../services/DockerController', () => ({
|
||||
pruneSystem,
|
||||
estimateManagedReclaim,
|
||||
estimateSystemReclaim,
|
||||
buildPrunePlan,
|
||||
executePrunePlan,
|
||||
getContainersByStack,
|
||||
stopContainer,
|
||||
restartContainer,
|
||||
@@ -90,6 +94,11 @@ beforeEach(() => {
|
||||
pruneSystem.mockResolvedValue({ success: true, reclaimedBytes: 0 });
|
||||
estimateManagedReclaim.mockResolvedValue({ reclaimableBytes: 0 });
|
||||
estimateSystemReclaim.mockResolvedValue({ reclaimableBytes: 0 });
|
||||
buildPrunePlan.mockResolvedValue({
|
||||
nodeId: 1, scope: 'managed', targets: ['images'], items: [], reclaimableBytes: 0,
|
||||
fingerprint: 'empty-plan', createdAt: 1,
|
||||
});
|
||||
executePrunePlan.mockResolvedValue({ success: true, reclaimedBytes: 0, outcomes: [] });
|
||||
getContainersByStack.mockResolvedValue([{ Id: 'container-1' }]);
|
||||
stopContainer.mockResolvedValue(undefined);
|
||||
restartContainer.mockResolvedValue(undefined);
|
||||
@@ -1000,8 +1009,19 @@ describe('POST /api/fleet/labels/fleet-stop with dryRun: true', () => {
|
||||
});
|
||||
|
||||
describe('POST /api/fleet/labels/fleet-prune with dryRun: true', () => {
|
||||
it('routes to estimateManagedReclaim and marks each target dryRun: true', async () => {
|
||||
estimateManagedReclaim.mockResolvedValue({ reclaimableBytes: 2048 });
|
||||
it('returns one multi-target itemized plan without calling prune methods', async () => {
|
||||
buildPrunePlan.mockResolvedValue({
|
||||
nodeId: 1,
|
||||
scope: 'managed',
|
||||
targets: ['volumes', 'images'],
|
||||
items: [
|
||||
{ target: 'volumes', id: 'data', name: 'data', sizeBytes: 512, managed: true, reason: 'unused' },
|
||||
{ target: 'images', id: 'image', name: 'app:latest', sizeBytes: 1536, managed: true, reason: 'unused' },
|
||||
],
|
||||
reclaimableBytes: 2048,
|
||||
fingerprint: 'itemized-plan',
|
||||
createdAt: 1,
|
||||
});
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
@@ -1009,41 +1029,105 @@ describe('POST /api/fleet/labels/fleet-prune with dryRun: true', () => {
|
||||
expect(res.status).toBe(200);
|
||||
const node = res.body.results[0];
|
||||
expect(node.reachable).toBe(true);
|
||||
expect(node.fingerprint).toBe('itemized-plan');
|
||||
expect(node.items).toHaveLength(2);
|
||||
expect(node.targets).toHaveLength(2);
|
||||
for (const t of node.targets) {
|
||||
expect(t.success).toBe(true);
|
||||
expect(t.reclaimedBytes).toBe(2048);
|
||||
expect(t.dryRun).toBe(true);
|
||||
}
|
||||
expect(node.targets).toEqual([
|
||||
{ target: 'images', success: true, reclaimedBytes: 1536, dryRun: true },
|
||||
{ target: 'volumes', success: true, reclaimedBytes: 512, dryRun: true },
|
||||
]);
|
||||
expect(pruneManagedOnly).not.toHaveBeenCalled();
|
||||
expect(pruneSystem).not.toHaveBeenCalled();
|
||||
expect(estimateManagedReclaim).toHaveBeenCalledTimes(2);
|
||||
expect(buildPrunePlan).toHaveBeenCalledTimes(1);
|
||||
expect(invalidateNodeCaches).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('routes to estimateSystemReclaim when scope is "all"', async () => {
|
||||
estimateSystemReclaim.mockResolvedValue({ reclaimableBytes: 8192 });
|
||||
it('loads known stacks and builds attribution even when scope is "all"', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images'], scope: 'all', dryRun: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(estimateManagedReclaim).not.toHaveBeenCalled();
|
||||
expect(estimateSystemReclaim).toHaveBeenCalled();
|
||||
expect(estimateSystemReclaim).not.toHaveBeenCalled();
|
||||
expect(buildPrunePlan).toHaveBeenCalledWith(['images'], 'all', ['alpha', 'beta'], expect.any(Number), expect.any(Function));
|
||||
expect(pruneManagedOnly).not.toHaveBeenCalled();
|
||||
expect(pruneSystem).not.toHaveBeenCalled();
|
||||
expect(res.body.results[0].targets[0].reclaimedBytes).toBe(8192);
|
||||
});
|
||||
|
||||
it('still invokes pruneManagedOnly when dryRun is omitted', async () => {
|
||||
pruneManagedOnly.mockResolvedValue({ success: true, reclaimedBytes: 512 });
|
||||
it('requires reviewed roster and fingerprints when dryRun is omitted', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(pruneManagedOnly).toHaveBeenCalled();
|
||||
expect(res.status).toBe(400);
|
||||
expect(pruneManagedOnly).not.toHaveBeenCalled();
|
||||
expect(executePrunePlan).not.toHaveBeenCalled();
|
||||
expect(estimateManagedReclaim).not.toHaveBeenCalled();
|
||||
expect(res.body.results[0].targets[0].dryRun).toBeUndefined();
|
||||
});
|
||||
|
||||
it('invalidates local caches only after an item is removed', async () => {
|
||||
const local = db.getNodes().find((node) => node.type === 'local')!;
|
||||
buildPrunePlan.mockResolvedValue({
|
||||
nodeId: local.id,
|
||||
scope: 'managed',
|
||||
targets: ['images'],
|
||||
items: [{ target: 'images', id: 'image', name: 'app:latest', managed: true, reason: 'unused' }],
|
||||
reclaimableBytes: 0,
|
||||
fingerprint: 'reviewed-plan',
|
||||
createdAt: 1,
|
||||
});
|
||||
executePrunePlan.mockResolvedValue({
|
||||
success: true,
|
||||
reclaimedBytes: 0,
|
||||
outcomes: [{ target: 'images', id: 'image', status: 'removed' }],
|
||||
});
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'],
|
||||
scope: 'managed',
|
||||
dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }],
|
||||
plans: [{ nodeId: local.id, fingerprint: 'reviewed-plan' }],
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
expect(executePrunePlan).toHaveBeenCalledTimes(1);
|
||||
expect(invalidateNodeCaches).toHaveBeenCalledWith(local.id);
|
||||
});
|
||||
|
||||
it('does not invalidate local caches for empty, skipped, or failed outcomes', async () => {
|
||||
const local = db.getNodes().find((node) => node.type === 'local')!;
|
||||
const cases = [
|
||||
{ items: [], outcomes: [] },
|
||||
{
|
||||
items: [{ target: 'images', id: 'image', name: 'app:latest', managed: true, reason: 'unused', image: { references: ['app:latest'] } }],
|
||||
outcomes: [{ target: 'images', id: 'image', status: 'skipped', reason: 'became active' }],
|
||||
},
|
||||
{
|
||||
items: [{ target: 'images', id: 'image', name: 'app:latest', managed: true, reason: 'unused', image: { references: ['app:latest'] } }],
|
||||
outcomes: [{ target: 'images', id: 'image', status: 'failed', error: 'remove failed' }],
|
||||
},
|
||||
] as const;
|
||||
for (const [index, testCase] of cases.entries()) {
|
||||
const fingerprint = `reviewed-plan-${index}`;
|
||||
buildPrunePlan.mockResolvedValue({
|
||||
nodeId: local.id, scope: 'managed', targets: ['images'], items: [...testCase.items],
|
||||
reclaimableBytes: 0, fingerprint, createdAt: 1,
|
||||
});
|
||||
executePrunePlan.mockResolvedValue({ success: true, reclaimedBytes: 0, outcomes: [...testCase.outcomes] });
|
||||
invalidateNodeCaches.mockClear();
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }],
|
||||
plans: [{ nodeId: local.id, fingerprint }],
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
expect(invalidateNodeCaches).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -40,6 +40,8 @@ let proxyNodeId: number;
|
||||
let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let LicenseService: typeof import('../services/LicenseService').LicenseService;
|
||||
let DockerController: typeof import('../services/DockerController').default;
|
||||
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
@@ -47,6 +49,8 @@ beforeAll(async () => {
|
||||
({ NodeRegistry } = await import('../services/NodeRegistry'));
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ LicenseService } = await import('../services/LicenseService'));
|
||||
({ default: DockerController } = await import('../services/DockerController'));
|
||||
({ FileSystemService } = await import('../services/FileSystemService'));
|
||||
|
||||
const db = DatabaseService.getInstance();
|
||||
pilotNodeId = db.addNode({
|
||||
@@ -259,7 +263,19 @@ describe('POST /api/fleet/labels/fleet-prune (pilot-agent dispatch)', () => {
|
||||
mockFetch((url, init) => {
|
||||
const headers = (init?.headers as Record<string, string>) ?? {};
|
||||
calls.push({ url, auth: headers.Authorization });
|
||||
return new Response(JSON.stringify({ success: true, reclaimedBytes: 999, dryRun: true }), {
|
||||
return new Response(JSON.stringify({
|
||||
nodeId: 1,
|
||||
scope: 'managed',
|
||||
targets: ['images'],
|
||||
items: [{
|
||||
target: 'images', id: 'sha256:pilot', name: 'pilot/app:latest', sizeBytes: 999,
|
||||
managed: true, reason: 'Image is not used by any container', stackName: 'app',
|
||||
image: { references: ['pilot/app:latest'] },
|
||||
}],
|
||||
reclaimableBytes: 999,
|
||||
fingerprint: 'pilot-plan',
|
||||
createdAt: 1,
|
||||
}), {
|
||||
status: 200, headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
});
|
||||
@@ -272,11 +288,75 @@ describe('POST /api/fleet/labels/fleet-prune (pilot-agent dispatch)', () => {
|
||||
expect(res.status).toBe(200);
|
||||
const pilotCall = calls.find(c => c.url.startsWith(PILOT_LOOPBACK));
|
||||
expect(pilotCall).toBeDefined();
|
||||
expect(pilotCall?.url).toBe(`${PILOT_LOOPBACK}/api/system/prune/plan`);
|
||||
expect(pilotCall?.auth).toBeUndefined();
|
||||
const pilotResult = res.body.results.find((r: { nodeId: number }) => r.nodeId === pilotNodeId);
|
||||
expect(pilotResult.reachable).toBe(true);
|
||||
expect(pilotResult.targets[0].reclaimedBytes).toBe(999);
|
||||
});
|
||||
|
||||
it('uses one plan request and one fingerprint-bound execute request per remote', async () => {
|
||||
mockPaidTier();
|
||||
mockTargets();
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
const localPlan = {
|
||||
nodeId: local.id, scope: 'managed' as const, targets: ['images' as const], items: [],
|
||||
reclaimableBytes: 0, fingerprint: 'local-plan', createdAt: 1,
|
||||
};
|
||||
const executePrunePlan = vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0, outcomes: [] });
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
buildPrunePlan: vi.fn().mockResolvedValue(localPlan),
|
||||
executePrunePlan,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue([]);
|
||||
|
||||
const calls: Array<{ url: string; auth: string | undefined; body: Record<string, unknown> }> = [];
|
||||
mockFetch((url, init) => {
|
||||
const headers = (init?.headers as Record<string, string>) ?? {};
|
||||
const body = JSON.parse(String(init?.body)) as Record<string, unknown>;
|
||||
calls.push({ url, auth: headers.Authorization, body });
|
||||
const pilot = url.startsWith(PILOT_LOOPBACK);
|
||||
if (url.endsWith('/api/system/prune/plan')) {
|
||||
return new Response(JSON.stringify({
|
||||
nodeId: 1, scope: 'managed', targets: ['images'], items: [], reclaimableBytes: 0,
|
||||
fingerprint: pilot ? 'pilot-plan' : 'proxy-plan', createdAt: 1,
|
||||
}), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
return new Response(JSON.stringify({ success: true, reclaimedBytes: 0, outcomes: [] }), {
|
||||
status: 200, headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [
|
||||
{ nodeId: local.id, reachable: true },
|
||||
{ nodeId: pilotNodeId, reachable: true },
|
||||
{ nodeId: proxyNodeId, reachable: true },
|
||||
],
|
||||
plans: [
|
||||
{ nodeId: local.id, fingerprint: 'local-plan' },
|
||||
{ nodeId: pilotNodeId, fingerprint: 'pilot-plan' },
|
||||
{ nodeId: proxyNodeId, fingerprint: 'proxy-plan' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(executePrunePlan).toHaveBeenCalledTimes(1);
|
||||
const pilotCalls = calls.filter((call) => call.url.startsWith(PILOT_LOOPBACK));
|
||||
expect(pilotCalls.map((call) => call.url)).toEqual([
|
||||
`${PILOT_LOOPBACK}/api/system/prune/plan`,
|
||||
`${PILOT_LOOPBACK}/api/system/prune/system`,
|
||||
]);
|
||||
expect(pilotCalls.every((call) => call.auth === undefined)).toBe(true);
|
||||
expect(pilotCalls[1].body).toMatchObject({ targets: ['images'], planFingerprint: 'pilot-plan' });
|
||||
const proxyCalls = calls.filter((call) => call.url.startsWith(PROXY_URL));
|
||||
expect(proxyCalls).toHaveLength(2);
|
||||
expect(proxyCalls.every((call) => call.auth === `Bearer ${PROXY_TOKEN}`)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/image-updates/fleet (pilot inclusion)', () => {
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
/**
|
||||
* F-6 regression: fleet routes that call estimateSystemReclaim on local
|
||||
* nodes must also bound the slow `docker system df` call (8s) and surface
|
||||
* a recognizable timeout message to the operator, matching the
|
||||
* /api/system/prune/estimate behavior.
|
||||
* F-6 regression: Fleet itemized plan enumeration and byte estimation both
|
||||
* bound the slow `docker system df` call (8s) and surface a recognizable
|
||||
* timeout message to the operator.
|
||||
*
|
||||
* Covers:
|
||||
* - POST /api/fleet/labels/fleet-prune with dryRun: true
|
||||
@@ -42,17 +41,27 @@ afterEach(() => {
|
||||
activeBulkActions.clear();
|
||||
});
|
||||
|
||||
function stubLocalEstimate(impl: () => Promise<{ reclaimableBytes: number }>) {
|
||||
function stubLocalEstimate(
|
||||
estimateImpl: () => Promise<{ reclaimableBytes: number }>,
|
||||
planImpl: () => Promise<unknown> = async () => ({
|
||||
nodeId: 1, scope: 'all', targets: ['volumes'], items: [], reclaimableBytes: 0,
|
||||
fingerprint: 'empty', createdAt: 1,
|
||||
}),
|
||||
) {
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
estimateSystemReclaim: vi.fn().mockImplementation(impl),
|
||||
estimateSystemReclaim: vi.fn().mockImplementation(estimateImpl),
|
||||
estimateManagedReclaim: vi.fn().mockResolvedValue({ reclaimableBytes: 0 }),
|
||||
buildPrunePlan: vi.fn().mockImplementation(planImpl),
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue([]);
|
||||
}
|
||||
|
||||
describe('Fleet prune routes bound docker df at 8s on local nodes (F-6)', () => {
|
||||
it('POST /api/fleet/labels/fleet-prune dry-run surfaces a busy-daemon error on local timeout', async () => {
|
||||
stubLocalEstimate(() => new Promise(() => { /* never resolves */ }));
|
||||
stubLocalEstimate(
|
||||
() => Promise.resolve({ reclaimableBytes: 0 }),
|
||||
() => new Promise(() => { /* never resolves */ }),
|
||||
);
|
||||
|
||||
const t0 = Date.now();
|
||||
const res = await request(app)
|
||||
@@ -86,7 +95,21 @@ describe('Fleet prune routes bound docker df at 8s on local nodes (F-6)', () =>
|
||||
}, 20_000);
|
||||
|
||||
it('fleet-prune dry-run succeeds normally when estimateSystemReclaim resolves quickly', async () => {
|
||||
stubLocalEstimate(() => Promise.resolve({ reclaimableBytes: 256 }));
|
||||
stubLocalEstimate(
|
||||
() => Promise.resolve({ reclaimableBytes: 256 }),
|
||||
async () => ({
|
||||
nodeId: 1,
|
||||
scope: 'all',
|
||||
targets: ['volumes'],
|
||||
items: [{
|
||||
target: 'volumes', id: 'volume-a', name: 'volume-a', sizeBytes: 256,
|
||||
managed: false, reason: 'Volume is not referenced by any container',
|
||||
}],
|
||||
reclaimableBytes: 256,
|
||||
fingerprint: 'volume-plan',
|
||||
createdAt: 1,
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
/**
|
||||
* Tests for the fleet-wide Docker prune endpoint. Covers auth, tier gating,
|
||||
* input validation, local node orchestration with mocked DockerController,
|
||||
* remote-node fan-out with mocked fetch, lock contention, and partial failures.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
|
||||
import { cleanupTestDb, setupTestDb, TEST_JWT_SECRET, TEST_USERNAME } from './helpers/setupTestDb';
|
||||
import type { PruneItemOutcome, PrunePlan, PrunePlanItem } from '../services/prunePlan';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let authHeader: string;
|
||||
let LicenseService: typeof import('../services/LicenseService').LicenseService;
|
||||
let DockerController: typeof import('../services/DockerController').default;
|
||||
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
@@ -20,13 +15,11 @@ let activeBulkActions: typeof import('../routes/labels').activeBulkActions;
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
({ LicenseService } = await import('../services/LicenseService'));
|
||||
({ default: DockerController } = await import('../services/DockerController'));
|
||||
({ FileSystemService } = await import('../services/FileSystemService'));
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ activeBulkActions } = await import('../routes/labels'));
|
||||
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
|
||||
authHeader = `Bearer ${token}`;
|
||||
authHeader = `Bearer ${jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '10m' })}`;
|
||||
});
|
||||
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
@@ -36,198 +29,448 @@ afterEach(() => {
|
||||
activeBulkActions.clear();
|
||||
});
|
||||
|
||||
function mockTier(tier: 'paid' | 'community') {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue(tier);
|
||||
function item(overrides: Partial<PrunePlanItem> = {}): PrunePlanItem {
|
||||
return {
|
||||
target: 'images',
|
||||
id: 'sha256:image',
|
||||
name: 'example/app:latest',
|
||||
sizeBytes: 256,
|
||||
managed: true,
|
||||
reason: 'Image is not used by any container',
|
||||
stackName: 'app',
|
||||
image: { references: ['example/app:latest'] },
|
||||
...overrides,
|
||||
} as PrunePlanItem;
|
||||
}
|
||||
|
||||
function mockLocalPrune(opts: { managedBytes?: Partial<Record<string, number>>; allBytes?: Partial<Record<string, number>>; throwOn?: string } = {}) {
|
||||
function plan(nodeId: number, fingerprint = `fingerprint-${nodeId}`, items: PrunePlanItem[] = [item()]): PrunePlan {
|
||||
return {
|
||||
nodeId,
|
||||
scope: 'managed',
|
||||
targets: ['images'],
|
||||
items,
|
||||
reclaimableBytes: items.reduce((sum, entry) => sum + (entry.sizeBytes ?? 0), 0),
|
||||
fingerprint,
|
||||
createdAt: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function mockLocal(planFactory: (nodeId: number) => PrunePlan = (nodeId) => plan(nodeId)) {
|
||||
const fake = {
|
||||
pruneManagedOnly: vi.fn(async (target: string) => {
|
||||
if (opts.throwOn === target) throw new Error(`mock pruneManagedOnly threw for ${target}`);
|
||||
return { success: true, reclaimedBytes: opts.managedBytes?.[target] ?? 0 };
|
||||
}),
|
||||
pruneSystem: vi.fn(async (target: string) => {
|
||||
if (opts.throwOn === target) throw new Error(`mock pruneSystem threw for ${target}`);
|
||||
return { success: true, reclaimedBytes: opts.allBytes?.[target] ?? 0 };
|
||||
}),
|
||||
buildPrunePlan: vi.fn(async (_targets, _scope, _stacks, nodeId: number) => planFactory(nodeId)),
|
||||
executePrunePlan: vi.fn(async (reviewedPlan: PrunePlan): Promise<{
|
||||
success: boolean;
|
||||
reclaimedBytes: number;
|
||||
outcomes: PruneItemOutcome[];
|
||||
}> => ({
|
||||
success: true,
|
||||
reclaimedBytes: reviewedPlan.reclaimableBytes,
|
||||
outcomes: reviewedPlan.items.map((entry) => ({
|
||||
id: entry.id,
|
||||
target: entry.target,
|
||||
status: 'removed' as const,
|
||||
sizeBytes: entry.sizeBytes,
|
||||
})),
|
||||
})),
|
||||
};
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue(fake as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
// Spy on the prototype so the mock applies to whichever FileSystemService
|
||||
// instance the route creates for the local node id, not a throwaway one.
|
||||
vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue(['stack-a', 'stack-b']);
|
||||
vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue(['app']);
|
||||
return fake;
|
||||
}
|
||||
|
||||
function localReview(fingerprint: string) {
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
return {
|
||||
local,
|
||||
body: {
|
||||
targets: ['images'],
|
||||
scope: 'managed',
|
||||
dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }],
|
||||
plans: [{ nodeId: local.id, fingerprint }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function addRemote(name: string): number {
|
||||
return DatabaseService.getInstance().addNode({
|
||||
name,
|
||||
type: 'remote',
|
||||
api_url: `http://${name}.example:1852`,
|
||||
api_token: 'token',
|
||||
compose_dir: '/app/compose',
|
||||
is_default: false,
|
||||
});
|
||||
}
|
||||
|
||||
describe('POST /api/fleet/labels/fleet-prune', () => {
|
||||
it('returns 401 without auth', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.send({ targets: ['images'], scope: 'managed' });
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('is reachable on community tier for admins (no PAID_REQUIRED)', async () => {
|
||||
mockTier('community');
|
||||
mockLocalPrune({ managedBytes: { images: 128 } });
|
||||
const res = await request(app)
|
||||
it('requires authentication and validates the request', async () => {
|
||||
expect((await request(app).post('/api/fleet/labels/fleet-prune').send({})).status).toBe(401);
|
||||
const invalid = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.code).not.toBe('PAID_REQUIRED');
|
||||
expect(Array.isArray(res.body.results)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns 400 when body is missing', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send();
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('returns 400 when targets is empty', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: [], scope: 'managed' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/non-empty/);
|
||||
});
|
||||
|
||||
it('returns 400 when a target is unrecognized', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'containers'], scope: 'managed' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/Invalid target/);
|
||||
});
|
||||
|
||||
it('runs pruneManagedOnly per target on the local node and returns aggregated bytes', async () => {
|
||||
const fake = mockLocalPrune({ managedBytes: { images: 1500, volumes: 320 } });
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'volumes'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.results).toHaveLength(1);
|
||||
const node = res.body.results[0];
|
||||
expect(node.reachable).toBe(true);
|
||||
expect(node.targets).toEqual([
|
||||
{ target: 'images', success: true, reclaimedBytes: 1500 },
|
||||
{ target: 'volumes', success: true, reclaimedBytes: 320 },
|
||||
]);
|
||||
expect(fake.pruneManagedOnly).toHaveBeenCalledTimes(2);
|
||||
expect(fake.pruneSystem).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('runs pruneSystem when scope is "all" and dedupes targets', async () => {
|
||||
const fake = mockLocalPrune({ allBytes: { networks: 0, images: 2048 } });
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'networks', 'images'], scope: 'all' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(fake.pruneManagedOnly).not.toHaveBeenCalled();
|
||||
expect(fake.pruneSystem).toHaveBeenCalledTimes(2);
|
||||
const node = res.body.results[0];
|
||||
expect(node.targets.map((t: { target: string }) => t.target).sort()).toEqual(['images', 'networks']);
|
||||
});
|
||||
|
||||
it('records per-target failure when DockerController throws but continues remaining targets', async () => {
|
||||
mockLocalPrune({ managedBytes: { images: 100 }, throwOn: 'volumes' });
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'volumes'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
const node = res.body.results[0];
|
||||
expect(node.targets.find((t: { target: string }) => t.target === 'images').success).toBe(true);
|
||||
const volumes = node.targets.find((t: { target: string }) => t.target === 'volumes');
|
||||
expect(volumes.success).toBe(false);
|
||||
expect(volumes.reclaimedBytes).toBe(0);
|
||||
expect(volumes.error).toMatch(/pruneManagedOnly threw/);
|
||||
});
|
||||
|
||||
it('reports lock contention when bulk-prune lock is already held', async () => {
|
||||
mockLocalPrune();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localId = db.getNodes().find(n => n.type === 'local')!.id;
|
||||
activeBulkActions.add(`bulk-prune:${localId}`);
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
const node = res.body.results.find((n: { nodeId: number }) => n.nodeId === localId);
|
||||
expect(node.targets[0].success).toBe(false);
|
||||
expect(node.targets[0].error).toMatch(/already running/);
|
||||
});
|
||||
|
||||
it('marks a remote node unreachable when fetch throws and short-circuits later targets', async () => {
|
||||
mockLocalPrune();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteId = db.addNode({
|
||||
name: 'remote-test',
|
||||
type: 'remote',
|
||||
api_url: 'http://remote.example:1852',
|
||||
api_token: 'tok',
|
||||
compose_dir: '/app/compose',
|
||||
is_default: false,
|
||||
});
|
||||
try {
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('connect ECONNREFUSED'));
|
||||
const res = await request(app)
|
||||
.send({ targets: ['containers'], dryRun: true });
|
||||
expect(invalid.status).toBe(400);
|
||||
for (const scope of [undefined, 'everything', 1]) {
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'volumes', 'networks'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
const remote = res.body.results.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remote.reachable).toBe(false);
|
||||
expect(remote.error).toMatch(/ECONNREFUSED/);
|
||||
expect(remote.targets).toHaveLength(3);
|
||||
for (const t of remote.targets) expect(t.success).toBe(false);
|
||||
// Only the first target attempts the fetch; the rest short-circuit.
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
.send({ targets: ['images'], scope, dryRun: true });
|
||||
expect(response.status).toBe(400);
|
||||
}
|
||||
});
|
||||
|
||||
it('parses remote node responses into per-target reclaimed bytes', async () => {
|
||||
mockLocalPrune();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteId = db.addNode({
|
||||
name: 'remote-ok',
|
||||
type: 'remote',
|
||||
api_url: 'http://remote-ok.example:1852/',
|
||||
api_token: 'tok',
|
||||
compose_dir: '/app/compose',
|
||||
is_default: false,
|
||||
});
|
||||
it('rejects malformed remote plan contracts', async () => {
|
||||
mockLocal();
|
||||
const remoteId = addRemote('remote-malformed-plan');
|
||||
const base = plan(remoteId);
|
||||
const malformedPlans = [
|
||||
{ ...base, targets: ['images', 'images'] },
|
||||
{ ...base, items: [item(), item()], reclaimableBytes: 512 },
|
||||
{ ...base, reclaimableBytes: -1 },
|
||||
{ ...base, nodeId: 'remote' },
|
||||
{ ...base, createdAt: Number.NaN },
|
||||
];
|
||||
try {
|
||||
const responses = new Map<string, number>([['images', 4096], ['volumes', 512]]);
|
||||
vi.spyOn(globalThis, 'fetch').mockImplementation(async (_url, init) => {
|
||||
const body = JSON.parse((init?.body as string) ?? '{}') as { target: string };
|
||||
const reclaimedBytes = responses.get(body.target) ?? 0;
|
||||
return new Response(JSON.stringify({ message: 'ok', success: true, reclaimedBytes }), {
|
||||
status: 200, headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
});
|
||||
const res = await request(app)
|
||||
for (const malformed of malformedPlans) {
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response(JSON.stringify(malformed), { status: 200 }));
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'volumes'], scope: 'managed', dryRun: true });
|
||||
const remote = response.body.results.find((result: { nodeId: number }) => result.nodeId === remoteId);
|
||||
expect(remote).toMatchObject({ reachable: true, code: 'REMOTE_PLAN_INVALID' });
|
||||
expect(remote.fingerprint).toBeUndefined();
|
||||
vi.restoreAllMocks();
|
||||
mockLocal();
|
||||
}
|
||||
} finally {
|
||||
DatabaseService.getInstance().deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('returns itemized dry-run plans without taking the destructive lock', async () => {
|
||||
const fake = mockLocal();
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images'], scope: 'managed', dryRun: true });
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.results[0]).toMatchObject({
|
||||
reachable: true,
|
||||
fingerprint: expect.stringMatching(/^fingerprint-/),
|
||||
reclaimableBytes: 256,
|
||||
items: [expect.objectContaining({ name: 'example/app:latest', managed: true, stackName: 'app' })],
|
||||
targets: [{ target: 'images', success: true, reclaimedBytes: 256, dryRun: true }],
|
||||
});
|
||||
expect(fake.executePrunePlan).not.toHaveBeenCalled();
|
||||
expect(activeBulkActions.size).toBe(0);
|
||||
});
|
||||
|
||||
it('loads known stacks for All unused attribution', async () => {
|
||||
mockLocal();
|
||||
const stackSpy = vi.spyOn(FileSystemService.prototype, 'getStacks');
|
||||
await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images'], scope: 'all', dryRun: true });
|
||||
expect(stackSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects missing, duplicate, and malformed reviewed entries', async () => {
|
||||
mockLocal();
|
||||
const { local } = localReview(`fingerprint-${DatabaseService.getInstance().getNodes()[0].id}`);
|
||||
const cases = [
|
||||
{ reviewedNodes: [{ nodeId: local.id, reachable: true }], plans: [] },
|
||||
{ reviewedNodes: [{ nodeId: local.id, reachable: true }, { nodeId: local.id, reachable: true }], plans: [] },
|
||||
{ reviewedNodes: [{ nodeId: local.id, reachable: true }], plans: [{ nodeId: local.id, fingerprint: '' }] },
|
||||
];
|
||||
for (const testCase of cases) {
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'volumes'], scope: 'all' });
|
||||
expect(res.status).toBe(200);
|
||||
const remote = res.body.results.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remote.reachable).toBe(true);
|
||||
expect(remote.targets).toEqual([
|
||||
{ target: 'images', success: true, reclaimedBytes: 4096 },
|
||||
{ target: 'volumes', success: true, reclaimedBytes: 512 },
|
||||
]);
|
||||
.send({ targets: ['images'], scope: 'managed', dryRun: false, ...testCase });
|
||||
expect([400, 409]).toContain(response.status);
|
||||
}
|
||||
});
|
||||
|
||||
it('executes a valid empty plan and releases the lock', async () => {
|
||||
const fake = mockLocal((nodeId) => plan(nodeId, `empty-${nodeId}`, []));
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }],
|
||||
plans: [{ nodeId: local.id, fingerprint: `empty-${local.id}` }],
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
expect(fake.executePrunePlan).toHaveBeenCalledTimes(1);
|
||||
expect(response.body.results[0].outcomes).toEqual([]);
|
||||
expect(activeBulkActions.size).toBe(0);
|
||||
});
|
||||
|
||||
it('fails closed when a local prune lock is active', async () => {
|
||||
const fake = mockLocal();
|
||||
const { local, body } = localReview(`fingerprint-${DatabaseService.getInstance().getNodes()[0].id}`);
|
||||
const remoteId = addRemote('remote-lock-check');
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch');
|
||||
const lockKey = `bulk-prune:${local.id}`;
|
||||
activeBulkActions.add(lockKey);
|
||||
try {
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
...body,
|
||||
reviewedNodes: [...body.reviewedNodes, { nodeId: remoteId, reachable: true }],
|
||||
plans: [...body.plans, { nodeId: remoteId, fingerprint: 'remote-plan' }],
|
||||
});
|
||||
expect(response.status).toBe(409);
|
||||
expect(response.body.code).toBe('PRUNE_ALREADY_RUNNING');
|
||||
expect(fake.buildPrunePlan).not.toHaveBeenCalled();
|
||||
expect(fake.executePrunePlan).not.toHaveBeenCalled();
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(activeBulkActions.has(lockKey)).toBe(true);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
activeBulkActions.delete(lockKey);
|
||||
DatabaseService.getInstance().deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('returns a node failure when local execution setup fails after preflight', async () => {
|
||||
const fake = mockLocal();
|
||||
vi.spyOn(FileSystemService.prototype, 'getStacks')
|
||||
.mockResolvedValueOnce(['app'])
|
||||
.mockRejectedValueOnce(new Error('stack inventory unavailable'));
|
||||
const { body } = localReview(`fingerprint-${DatabaseService.getInstance().getNodes()[0].id}`);
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send(body);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.results[0]).toMatchObject({
|
||||
code: 'PRUNE_EXECUTE_FAILED',
|
||||
error: 'stack inventory unavailable',
|
||||
targets: [{ success: false }],
|
||||
});
|
||||
expect(fake.executePrunePlan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('prevents every destructive call when one node plan is stale', async () => {
|
||||
const fake = mockLocal();
|
||||
const remoteId = addRemote('remote-stale');
|
||||
try {
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(
|
||||
JSON.stringify(plan(99, 'remote-new')),
|
||||
{ status: 200, headers: { 'content-type': 'application/json' } },
|
||||
));
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }, { nodeId: remoteId, reachable: true }],
|
||||
plans: [
|
||||
{ nodeId: local.id, fingerprint: `fingerprint-${local.id}` },
|
||||
{ nodeId: remoteId, fingerprint: 'remote-old' },
|
||||
],
|
||||
});
|
||||
expect(response.status).toBe(409);
|
||||
expect(response.body.code).toBe('PRUNE_PLAN_STALE');
|
||||
expect(fake.executePrunePlan).not.toHaveBeenCalled();
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
||||
expect(String(fetchSpy.mock.calls[0][0])).toContain('/api/system/prune/plan');
|
||||
} finally {
|
||||
DatabaseService.getInstance().deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a reviewed-unreachable node that becomes reachable', async () => {
|
||||
const fake = mockLocal();
|
||||
const remoteId = addRemote('remote-newly-reachable');
|
||||
try {
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify(plan(99)), { status: 200 }));
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }, { nodeId: remoteId, reachable: false }],
|
||||
plans: [{ nodeId: local.id, fingerprint: `fingerprint-${local.id}` }],
|
||||
});
|
||||
expect(response.status).toBe(409);
|
||||
expect(response.body.code).toBe('PRUNE_NODE_REACHABILITY_CHANGED');
|
||||
expect(fake.executePrunePlan).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
DatabaseService.getInstance().deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a reviewed-reachable node that becomes unreachable', async () => {
|
||||
const fake = mockLocal();
|
||||
const remoteId = addRemote('remote-now-offline');
|
||||
try {
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('connect ECONNREFUSED'));
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }, { nodeId: remoteId, reachable: true }],
|
||||
plans: [
|
||||
{ nodeId: local.id, fingerprint: `fingerprint-${local.id}` },
|
||||
{ nodeId: remoteId, fingerprint: 'remote-plan' },
|
||||
],
|
||||
});
|
||||
expect(response.status).toBe(409);
|
||||
expect(response.body.code).toBe('PRUNE_NODE_REACHABILITY_CHANGED');
|
||||
expect(fake.executePrunePlan).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
DatabaseService.getInstance().deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a changed configured-node roster before preflight', async () => {
|
||||
const fake = mockLocal();
|
||||
const remoteId = addRemote('remote-added-after-review');
|
||||
try {
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }],
|
||||
plans: [{ nodeId: local.id, fingerprint: `fingerprint-${local.id}` }],
|
||||
});
|
||||
expect(response.status).toBe(409);
|
||||
expect(response.body.code).toBe('PRUNE_NODE_ROSTER_CHANGED');
|
||||
expect(fake.buildPrunePlan).not.toHaveBeenCalled();
|
||||
expect(fake.executePrunePlan).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
DatabaseService.getInstance().deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a present but incomplete remote outcome list', async () => {
|
||||
const fake = mockLocal();
|
||||
const remoteId = addRemote('remote-bad-outcomes');
|
||||
try {
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
vi.spyOn(globalThis, 'fetch')
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify(plan(remoteId, 'remote-reviewed')), { status: 200 }))
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify({
|
||||
success: true,
|
||||
reclaimedBytes: 256,
|
||||
outcomes: [],
|
||||
}), { status: 200 }));
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }, { nodeId: remoteId, reachable: true }],
|
||||
plans: [
|
||||
{ nodeId: local.id, fingerprint: `fingerprint-${local.id}` },
|
||||
{ nodeId: remoteId, fingerprint: 'remote-reviewed' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(fake.executePrunePlan).toHaveBeenCalledTimes(1);
|
||||
expect(response.body.results.find((result: { nodeId: number }) => result.nodeId === remoteId)).toMatchObject({
|
||||
code: 'REMOTE_PRUNE_INVALID',
|
||||
error: 'Remote returned malformed or incomplete prune outcomes',
|
||||
});
|
||||
} finally {
|
||||
DatabaseService.getInstance().deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects malformed numeric and success fields in remote execute results', async () => {
|
||||
mockLocal();
|
||||
const remoteId = addRemote('remote-bad-result-fields');
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
const remotePlan = plan(remoteId, 'remote-fields');
|
||||
const malformedResults = [
|
||||
{ success: 'yes', reclaimedBytes: 256 },
|
||||
{ success: true, reclaimedBytes: -1 },
|
||||
{
|
||||
success: true, reclaimedBytes: 0,
|
||||
outcomes: [{ target: 'images', id: 'sha256:image', status: 'removed', sizeBytes: -1 }],
|
||||
},
|
||||
];
|
||||
try {
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch');
|
||||
for (const malformed of malformedResults) {
|
||||
fetchSpy
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify(remotePlan), { status: 200 }))
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify(malformed), { status: 200 }));
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }, { nodeId: remoteId, reachable: true }],
|
||||
plans: [{ nodeId: local.id, fingerprint: `fingerprint-${local.id}` }, { nodeId: remoteId, fingerprint: 'remote-fields' }],
|
||||
});
|
||||
expect(response.body.results.find((result: { nodeId: number }) => result.nodeId === remoteId)).toMatchObject({
|
||||
code: 'REMOTE_PRUNE_INVALID',
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
DatabaseService.getInstance().deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('projects mixed local outcomes and accepts a legacy remote total', async () => {
|
||||
const items = [
|
||||
item({ id: 'removed', sizeBytes: 100 }),
|
||||
item({ id: 'skipped', sizeBytes: 200 }),
|
||||
item({ id: 'failed', sizeBytes: 300 }),
|
||||
];
|
||||
const fake = mockLocal((nodeId) => plan(nodeId, `mixed-${nodeId}`, items));
|
||||
fake.executePrunePlan.mockResolvedValue({
|
||||
success: false,
|
||||
reclaimedBytes: 100,
|
||||
outcomes: [
|
||||
{ target: 'images', id: 'removed', status: 'removed', sizeBytes: 100 },
|
||||
{ target: 'images', id: 'skipped', status: 'skipped', reason: 'became active' },
|
||||
{ target: 'images', id: 'failed', status: 'failed', error: 'remove failed' },
|
||||
],
|
||||
});
|
||||
const remoteId = addRemote('remote-legacy-total');
|
||||
const remotePlan = plan(remoteId, 'legacy-plan', []);
|
||||
try {
|
||||
vi.spyOn(globalThis, 'fetch')
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify(remotePlan), { status: 200 }))
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify({ success: true, reclaimedBytes: 999 }), { status: 200 }));
|
||||
const local = DatabaseService.getInstance().getNodes().find((node) => node.type === 'local')!;
|
||||
const response = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({
|
||||
targets: ['images'], scope: 'managed', dryRun: false,
|
||||
reviewedNodes: [{ nodeId: local.id, reachable: true }, { nodeId: remoteId, reachable: true }],
|
||||
plans: [{ nodeId: local.id, fingerprint: `mixed-${local.id}` }, { nodeId: remoteId, fingerprint: 'legacy-plan' }],
|
||||
});
|
||||
const localResult = response.body.results.find((result: { nodeId: number }) => result.nodeId === local.id);
|
||||
expect(localResult.targets[0]).toMatchObject({
|
||||
success: false, reclaimedBytes: 100, removed: 1, skipped: 1, failed: 1,
|
||||
});
|
||||
const remoteResult = response.body.results.find((result: { nodeId: number }) => result.nodeId === remoteId);
|
||||
expect(remoteResult.reclaimedBytes).toBe(999);
|
||||
expect(remoteResult.outcomes).toBeUndefined();
|
||||
expect(remoteResult.targets[0]).toMatchObject({ success: true, reclaimedBytes: 0 });
|
||||
} finally {
|
||||
DatabaseService.getInstance().deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -105,6 +105,168 @@ describe('normalizePruneTargets', () => {
|
||||
});
|
||||
|
||||
describe('DockerController.buildPrunePlan', () => {
|
||||
it('projects target metadata and only safe ownership labels in all scope', async () => {
|
||||
mockDocker.listImages.mockResolvedValue([{
|
||||
Id: 'sha256:dangling',
|
||||
RepoTags: ['<none>:<none>'],
|
||||
RepoDigests: ['example/app@sha256:digest'],
|
||||
Created: 1_700_000_000,
|
||||
Size: 100,
|
||||
Containers: 0,
|
||||
Labels: { 'com.docker.compose.project': 'my-stack', secret: 'do-not-return' },
|
||||
}]);
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [{
|
||||
Name: 'my-stack_data',
|
||||
Driver: 'local',
|
||||
Labels: { 'com.docker.compose.project': 'my-stack', secret: 'do-not-return' },
|
||||
}] });
|
||||
mockDocker.listNetworks.mockResolvedValue([{
|
||||
Id: 'network-id',
|
||||
Name: 'my-stack_default',
|
||||
Driver: 'bridge',
|
||||
Scope: 'local',
|
||||
Labels: {
|
||||
'com.docker.compose.project': 'my-stack',
|
||||
'com.docker.compose.network': 'default',
|
||||
secret: 'do-not-return',
|
||||
},
|
||||
}]);
|
||||
mockDocker.getNetwork.mockReturnValue({ inspect: vi.fn().mockResolvedValue({ Containers: {} }) });
|
||||
mockDocker.df.mockResolvedValue({
|
||||
Volumes: [{ Name: 'my-stack_data', UsageData: { RefCount: 0, Size: 42 } }],
|
||||
Images: [{ Id: 'sha256:dangling', SharedSize: 10 }],
|
||||
LayersSize: 0,
|
||||
});
|
||||
|
||||
const plan = await DockerController.getInstance(1).buildPrunePlan(
|
||||
['images', 'volumes', 'networks'], 'all', ['my-stack'], 1,
|
||||
);
|
||||
|
||||
expect(plan.items.find((entry) => entry.target === 'images')).toMatchObject({
|
||||
name: '<none>:<none>',
|
||||
managed: true,
|
||||
stackName: 'my-stack',
|
||||
image: {
|
||||
references: [],
|
||||
digest: 'example/app@sha256:digest',
|
||||
createdAt: 1_700_000_000,
|
||||
},
|
||||
});
|
||||
expect(plan.items.find((entry) => entry.target === 'volumes')).toMatchObject({
|
||||
managed: true,
|
||||
stackName: 'my-stack',
|
||||
volume: {
|
||||
driver: 'local',
|
||||
ownershipLabels: { 'com.docker.compose.project': 'my-stack' },
|
||||
},
|
||||
});
|
||||
expect(plan.items.find((entry) => entry.target === 'networks')).toMatchObject({
|
||||
managed: true,
|
||||
stackName: 'my-stack',
|
||||
network: {
|
||||
driver: 'bridge',
|
||||
scope: 'local',
|
||||
ownershipLabels: {
|
||||
'com.docker.compose.project': 'my-stack',
|
||||
'com.docker.compose.network': 'default',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(plan.items)).not.toContain('do-not-return');
|
||||
expect(plan.reclaimableBytes).toBe(
|
||||
plan.items.reduce((sum, entry) => sum + (entry.sizeBytes ?? 0), 0),
|
||||
);
|
||||
});
|
||||
|
||||
it('uses Compose path ownership fallbacks for non-container resources', async () => {
|
||||
const ownershipLabels = {
|
||||
'com.docker.compose.project.working_dir': '/app/compose/my-stack',
|
||||
'com.docker.compose.project.config_files': '/app/compose/my-stack/compose.yml',
|
||||
};
|
||||
mockDocker.listImages.mockResolvedValue([{
|
||||
Id: 'sha256:path-owned', RepoTags: ['example/path:latest'], Size: 100, Containers: 0, Labels: ownershipLabels,
|
||||
}]);
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [{ Name: 'path_data', Labels: ownershipLabels }] });
|
||||
mockDocker.listNetworks.mockResolvedValue([{ Id: 'path-network', Name: 'path_default', Labels: ownershipLabels }]);
|
||||
mockDocker.getNetwork.mockReturnValue({ inspect: vi.fn().mockResolvedValue({ Containers: {} }) });
|
||||
mockDocker.df.mockResolvedValue({
|
||||
Volumes: [{ Name: 'path_data', UsageData: { RefCount: 0, Size: 42 } }],
|
||||
Images: [{ Id: 'sha256:path-owned', SharedSize: 0 }],
|
||||
LayersSize: 0,
|
||||
});
|
||||
|
||||
const plan = await DockerController.getInstance(1).buildPrunePlan(
|
||||
['images', 'volumes', 'networks'], 'managed', ['my-stack'], 1,
|
||||
);
|
||||
|
||||
expect(plan.items).toHaveLength(3);
|
||||
expect(plan.items.every((entry) => entry.managed && entry.stackName === 'my-stack')).toBe(true);
|
||||
});
|
||||
|
||||
it.each(['__proto__', 'constructor', 'toString', 'prototype'])(
|
||||
'does not attribute inherited project key %s to a managed stack',
|
||||
async (project) => {
|
||||
const labels = { 'com.docker.compose.project': project };
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [{ Name: `${project}_data`, Labels: labels }] });
|
||||
mockDocker.df.mockResolvedValue({
|
||||
Volumes: [{ Name: `${project}_data`, UsageData: { RefCount: 0, Size: 42 } }],
|
||||
Images: [],
|
||||
LayersSize: 0,
|
||||
});
|
||||
|
||||
const managedPlan = await DockerController.getInstance(1).buildPrunePlan(
|
||||
['volumes'], 'managed', ['my-stack'], 1,
|
||||
);
|
||||
const allPlan = await DockerController.getInstance(1).buildPrunePlan(
|
||||
['volumes'], 'all', ['my-stack'], 1,
|
||||
);
|
||||
|
||||
expect(managedPlan.items).toEqual([]);
|
||||
expect(allPlan.items).toEqual([
|
||||
expect.objectContaining({ id: `${project}_data`, managed: false }),
|
||||
]);
|
||||
expect(allPlan.items[0].stackName).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it('does not plan an image referenced by a container when Docker reports Containers as unknown', async () => {
|
||||
mockDocker.listContainers.mockResolvedValue([{ Id: 'container', ImageID: 'sha256:in-use' }]);
|
||||
mockDocker.listImages.mockResolvedValue([{
|
||||
Id: 'sha256:in-use', RepoTags: ['example/in-use:latest'], Size: 100, Containers: -1,
|
||||
}]);
|
||||
mockDocker.df.mockResolvedValue({
|
||||
Volumes: [], Images: [{ Id: 'sha256:in-use', SharedSize: 0 }], LayersSize: 0,
|
||||
});
|
||||
|
||||
const plan = await DockerController.getInstance(1).buildPrunePlan(['images'], 'all', [], 1);
|
||||
expect(plan.items).toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves Compose path ownership fallback during volume and network execution', async () => {
|
||||
const labels = { 'com.docker.compose.project.working_dir': '/app/compose/my-stack' };
|
||||
const volumeRemove = vi.fn().mockResolvedValue(undefined);
|
||||
const networkRemove = vi.fn().mockResolvedValue(undefined);
|
||||
const networkInspect = vi.fn().mockResolvedValue({ Name: 'path_default', Labels: labels, Containers: {} });
|
||||
mockDocker.listVolumes.mockResolvedValue({ Volumes: [{ Name: 'path_data', Labels: labels }] });
|
||||
mockDocker.listNetworks.mockResolvedValue([{ Id: 'path-network', Name: 'path_default', Labels: labels }]);
|
||||
mockDocker.df.mockResolvedValue({
|
||||
Volumes: [{ Name: 'path_data', UsageData: { RefCount: 0, Size: 42 } }], Images: [], LayersSize: 0,
|
||||
});
|
||||
mockDocker.getVolume.mockReturnValue({ remove: volumeRemove });
|
||||
mockDocker.getNetwork.mockReturnValue({ inspect: networkInspect, remove: networkRemove });
|
||||
|
||||
const controller = DockerController.getInstance(1);
|
||||
const plan = await controller.buildPrunePlan(['volumes', 'networks'], 'managed', ['my-stack'], 1);
|
||||
const result = await controller.executePrunePlan(plan, ['my-stack']);
|
||||
|
||||
expect(result.outcomes).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ target: 'volumes', status: 'removed' }),
|
||||
expect.objectContaining({ target: 'networks', status: 'removed' }),
|
||||
]));
|
||||
expect(volumeRemove).toHaveBeenCalledWith({ force: false });
|
||||
expect(networkRemove).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('enumerates managed stopped containers and never calls pruneSystem', async () => {
|
||||
mockDocker.listContainers.mockResolvedValue([
|
||||
{
|
||||
|
||||
@@ -16,12 +16,16 @@ let app: import('express').Express;
|
||||
let authHeader: string;
|
||||
let DockerController: typeof import('../services/DockerController').default;
|
||||
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
|
||||
let CacheService: typeof import('../services/CacheService').CacheService;
|
||||
let activeBulkActions: typeof import('../helpers/bulkActionLocks').activeBulkActions;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
({ default: DockerController } = await import('../services/DockerController'));
|
||||
({ FileSystemService } = await import('../services/FileSystemService'));
|
||||
({ CacheService } = await import('../services/CacheService'));
|
||||
({ activeBulkActions } = await import('../helpers/bulkActionLocks'));
|
||||
// 10-minute expiry survives the full file even when two timeout tests
|
||||
// burn ~8.5s each in real-timer mode.
|
||||
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '10m' });
|
||||
@@ -31,6 +35,7 @@ beforeAll(async () => {
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
|
||||
afterEach(() => {
|
||||
activeBulkActions.clear();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
@@ -150,6 +155,7 @@ describe('Prune plan routes', () => {
|
||||
buildPrunePlan: vi.fn().mockResolvedValue(plan),
|
||||
executePrunePlan,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate');
|
||||
|
||||
const res = await request(app)
|
||||
.post('/api/system/prune/system')
|
||||
@@ -161,6 +167,45 @@ describe('Prune plan routes', () => {
|
||||
expect(res.body.reclaimedBytes).toBe(42);
|
||||
expect(res.body.outcomes).toHaveLength(1);
|
||||
expect(executePrunePlan).toHaveBeenCalled();
|
||||
expect(invalidate).toHaveBeenCalledWith('stats:1');
|
||||
expect(invalidate).toHaveBeenCalledWith('stack-statuses:1');
|
||||
});
|
||||
|
||||
it('rejects an overlapping destructive prune on the same node', async () => {
|
||||
stubFsStacks();
|
||||
const plan = samplePlan('fp-lock');
|
||||
let releaseExecution!: () => void;
|
||||
const executionBlocked = new Promise<void>((resolve) => {
|
||||
releaseExecution = resolve;
|
||||
});
|
||||
const executePrunePlan = vi.fn().mockImplementation(async () => {
|
||||
await executionBlocked;
|
||||
return { outcomes: [], reclaimedBytes: 0, success: true };
|
||||
});
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
buildPrunePlan: vi.fn().mockResolvedValue(plan),
|
||||
executePrunePlan,
|
||||
} as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
|
||||
const firstRequest = request(app)
|
||||
.post('/api/system/prune/system')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ target: 'volumes', scope: 'managed', planFingerprint: 'fp-lock' });
|
||||
const firstResponse = firstRequest.then((response) => response);
|
||||
await vi.waitFor(() => expect(executePrunePlan).toHaveBeenCalledTimes(1));
|
||||
|
||||
const overlapping = await request(app)
|
||||
.post('/api/system/prune/system')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ target: 'volumes', scope: 'managed', planFingerprint: 'fp-lock' });
|
||||
|
||||
expect(overlapping.status).toBe(409);
|
||||
expect(overlapping.body.code).toBe('PRUNE_ALREADY_RUNNING');
|
||||
expect(executePrunePlan).toHaveBeenCalledTimes(1);
|
||||
|
||||
releaseExecution();
|
||||
expect((await firstResponse).status).toBe(200);
|
||||
expect(activeBulkActions.size).toBe(0);
|
||||
});
|
||||
|
||||
it('POST /api/system/prune/system returns 409 PRUNE_PLAN_STALE on fingerprint mismatch', async () => {
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
// Process-local mutation locks shared by direct node actions and fleet-wide
|
||||
// orchestration. Callers must use the same operation-specific key format.
|
||||
export const activeBulkActions = new Set<string>();
|
||||
@@ -0,0 +1,597 @@
|
||||
import type { Node } from '../services/DatabaseService';
|
||||
import DockerController from '../services/DockerController';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import {
|
||||
PrunePlanStaleError,
|
||||
hasOnlyPruneOwnershipLabels,
|
||||
projectPruneOwnershipLabels,
|
||||
type PruneItemOutcome,
|
||||
type PrunePlan,
|
||||
type PrunePlanItem,
|
||||
type PruneScope,
|
||||
} from '../services/prunePlan';
|
||||
import { invalidateNodeCaches } from './cacheInvalidation';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { TimeoutError, withTimeout } from '../utils/withTimeout';
|
||||
import { formatNoTargetError } from '../utils/remoteTarget';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
|
||||
export const FLEET_PRUNE_TARGETS = ['images', 'volumes', 'networks'] as const;
|
||||
export type FleetPruneTarget = (typeof FLEET_PRUNE_TARGETS)[number];
|
||||
|
||||
export interface ReviewedFleetNode {
|
||||
nodeId: number;
|
||||
reachable: boolean;
|
||||
}
|
||||
|
||||
export interface ReviewedFleetPlan {
|
||||
nodeId: number;
|
||||
fingerprint: string;
|
||||
}
|
||||
|
||||
export interface FleetPruneTargetResult {
|
||||
target: FleetPruneTarget;
|
||||
success: boolean;
|
||||
reclaimedBytes: number;
|
||||
dryRun: boolean;
|
||||
removed?: number;
|
||||
skipped?: number;
|
||||
failed?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface FleetPruneNodeResult {
|
||||
nodeId: number;
|
||||
nodeName: string;
|
||||
reachable: boolean;
|
||||
code?: string;
|
||||
error?: string;
|
||||
fingerprint?: string;
|
||||
items?: PrunePlanItem[];
|
||||
reclaimableBytes?: number;
|
||||
reclaimedBytes?: number;
|
||||
outcomes?: PruneItemOutcome[];
|
||||
targets: FleetPruneTargetResult[];
|
||||
}
|
||||
|
||||
export type ParsedFleetPruneRequest = {
|
||||
targets: FleetPruneTarget[];
|
||||
scope: PruneScope;
|
||||
dryRun: boolean;
|
||||
reviewedNodes: ReviewedFleetNode[];
|
||||
plans: ReviewedFleetPlan[];
|
||||
};
|
||||
|
||||
type ParseResult = { request: ParsedFleetPruneRequest } | { error: string };
|
||||
|
||||
type Preflight = {
|
||||
node: Node;
|
||||
reachable: boolean;
|
||||
plan?: PrunePlan;
|
||||
code?: string;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
type FleetPruneResponse = {
|
||||
status: number;
|
||||
body: { error?: string; code?: string; nodeId?: number; results?: FleetPruneNodeResult[] };
|
||||
};
|
||||
|
||||
const PLAN_TIMEOUT_MS = 8_000;
|
||||
const REMOTE_PLAN_TIMEOUT_MS = 120_000;
|
||||
const BUSY_DAEMON_ERROR = 'Docker daemon is busy. Please try again in a moment.';
|
||||
|
||||
function parseTargets(value: unknown): FleetPruneTarget[] | null {
|
||||
if (!Array.isArray(value) || value.length === 0) return null;
|
||||
const targets = new Set<FleetPruneTarget>();
|
||||
for (const target of value) {
|
||||
if (typeof target !== 'string' || !(FLEET_PRUNE_TARGETS as readonly string[]).includes(target)) {
|
||||
return null;
|
||||
}
|
||||
targets.add(target as FleetPruneTarget);
|
||||
}
|
||||
return [...targets];
|
||||
}
|
||||
|
||||
function parseReviewedNodes(value: unknown): ReviewedFleetNode[] | null {
|
||||
if (!Array.isArray(value) || value.length === 0) return null;
|
||||
const parsed: ReviewedFleetNode[] = [];
|
||||
const seen = new Set<number>();
|
||||
for (const entry of value) {
|
||||
if (!entry || typeof entry !== 'object') return null;
|
||||
const { nodeId, reachable } = entry as { nodeId?: unknown; reachable?: unknown };
|
||||
if (!Number.isInteger(nodeId) || typeof reachable !== 'boolean' || seen.has(nodeId as number)) return null;
|
||||
seen.add(nodeId as number);
|
||||
parsed.push({ nodeId: nodeId as number, reachable });
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parsePlans(value: unknown): ReviewedFleetPlan[] | null {
|
||||
if (!Array.isArray(value)) return null;
|
||||
const parsed: ReviewedFleetPlan[] = [];
|
||||
const seen = new Set<number>();
|
||||
for (const entry of value) {
|
||||
if (!entry || typeof entry !== 'object') return null;
|
||||
const { nodeId, fingerprint } = entry as { nodeId?: unknown; fingerprint?: unknown };
|
||||
if (!Number.isInteger(nodeId) || typeof fingerprint !== 'string' || fingerprint.trim() === '' || seen.has(nodeId as number)) {
|
||||
return null;
|
||||
}
|
||||
seen.add(nodeId as number);
|
||||
parsed.push({ nodeId: nodeId as number, fingerprint: fingerprint.trim() });
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function parseFleetPruneRequest(body: unknown): ParseResult {
|
||||
if (!body || typeof body !== 'object') return { error: 'Request body is required' };
|
||||
const input = body as Record<string, unknown>;
|
||||
const targets = parseTargets(input.targets);
|
||||
if (!targets) return { error: 'targets must be a non-empty array of images, volumes, or networks' };
|
||||
if (input.scope !== 'managed' && input.scope !== 'all') return { error: 'scope must be managed or all' };
|
||||
const scope: PruneScope = input.scope;
|
||||
const dryRun = input.dryRun === true;
|
||||
if (dryRun) return { request: { targets, scope, dryRun, reviewedNodes: [], plans: [] } };
|
||||
const reviewedNodes = parseReviewedNodes(input.reviewedNodes);
|
||||
const plans = parsePlans(input.plans);
|
||||
if (!reviewedNodes || !plans) return { error: 'reviewedNodes and plans are required for fleet prune execution' };
|
||||
return { request: { targets, scope, dryRun, reviewedNodes, plans } };
|
||||
}
|
||||
|
||||
function validateReviewedRoster(
|
||||
nodes: Node[],
|
||||
reviewedNodes: ReviewedFleetNode[],
|
||||
plans: ReviewedFleetPlan[],
|
||||
): string | null {
|
||||
const currentIds = nodes.map((node) => node.id).sort((a, b) => a - b);
|
||||
const reviewedIds = reviewedNodes.map((node) => node.nodeId).sort((a, b) => a - b);
|
||||
if (currentIds.length !== reviewedIds.length || currentIds.some((id, index) => id !== reviewedIds[index])) {
|
||||
return 'The fleet node roster changed after the dry run';
|
||||
}
|
||||
const reachableIds = reviewedNodes.filter((node) => node.reachable).map((node) => node.nodeId).sort((a, b) => a - b);
|
||||
const planIds = plans.map((plan) => plan.nodeId).sort((a, b) => a - b);
|
||||
if (reachableIds.length !== planIds.length || reachableIds.some((id, index) => id !== planIds[index])) {
|
||||
return 'Plans must exactly cover the reachable nodes from the dry run';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function targetRowsFromPlan(plan: PrunePlan, targets: FleetPruneTarget[]): FleetPruneTargetResult[] {
|
||||
return targets.map((target) => ({
|
||||
target,
|
||||
success: true,
|
||||
reclaimedBytes: plan.items
|
||||
.filter((item) => item.target === target)
|
||||
.reduce((sum, item) => sum + (item.sizeBytes ?? 0), 0),
|
||||
dryRun: true,
|
||||
}));
|
||||
}
|
||||
|
||||
function failedTargetRows(
|
||||
targets: FleetPruneTarget[],
|
||||
error: string,
|
||||
dryRun: boolean,
|
||||
): FleetPruneTargetResult[] {
|
||||
return targets.map((target) => ({ target, success: false, reclaimedBytes: 0, dryRun, error }));
|
||||
}
|
||||
|
||||
function isPrunePlan(
|
||||
value: unknown,
|
||||
targets: FleetPruneTarget[],
|
||||
scope: PruneScope,
|
||||
): value is PrunePlan {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
const plan = value as Partial<PrunePlan>;
|
||||
const planTargets = Array.isArray(plan.targets) ? plan.targets : [];
|
||||
const requestedTargets = new Set(targets);
|
||||
const uniquePlanTargets = new Set(planTargets);
|
||||
const itemKeys = new Set<string>();
|
||||
const itemBytes = Array.isArray(plan.items)
|
||||
? plan.items.reduce((sum, item) => sum + (typeof item?.sizeBytes === 'number' ? item.sizeBytes : 0), 0)
|
||||
: -1;
|
||||
return plan.scope === scope
|
||||
&& planTargets.length === requestedTargets.size
|
||||
&& uniquePlanTargets.size === requestedTargets.size
|
||||
&& planTargets.every((target) => typeof target === 'string' && requestedTargets.has(target as FleetPruneTarget))
|
||||
&& typeof plan.fingerprint === 'string'
|
||||
&& plan.fingerprint.length > 0
|
||||
&& Number.isInteger(plan.nodeId)
|
||||
&& typeof plan.createdAt === 'number' && Number.isFinite(plan.createdAt) && plan.createdAt >= 0
|
||||
&& typeof plan.reclaimableBytes === 'number' && Number.isFinite(plan.reclaimableBytes) && plan.reclaimableBytes >= 0
|
||||
&& plan.reclaimableBytes === itemBytes
|
||||
&& Array.isArray(plan.items)
|
||||
&& plan.items.every((item) => {
|
||||
if (!isPrunePlanItem(item) || !requestedTargets.has(item.target as FleetPruneTarget)) return false;
|
||||
const key = `${item.target}\0${item.id}`;
|
||||
if (itemKeys.has(key)) return false;
|
||||
itemKeys.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function isPrunePlanItem(value: unknown): value is PrunePlanItem {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
const item = value as Record<string, unknown>;
|
||||
const validTarget = typeof item.target === 'string'
|
||||
&& ['images', 'volumes', 'networks', 'containers'].includes(item.target);
|
||||
const validSize = item.sizeBytes === undefined
|
||||
|| (typeof item.sizeBytes === 'number' && Number.isFinite(item.sizeBytes) && item.sizeBytes >= 0);
|
||||
const targetMetadata = item.target === 'images'
|
||||
? Boolean(item.image && typeof item.image === 'object'
|
||||
&& Array.isArray((item.image as { references?: unknown }).references)
|
||||
&& (item.image as { references: unknown[] }).references.every((ref) => typeof ref === 'string'))
|
||||
: item.target === 'volumes'
|
||||
? Boolean(item.volume && typeof item.volume === 'object')
|
||||
: item.target === 'networks'
|
||||
? Boolean(item.network && typeof item.network === 'object')
|
||||
: true;
|
||||
return validTarget
|
||||
&& typeof item.id === 'string'
|
||||
&& typeof item.name === 'string'
|
||||
&& typeof item.managed === 'boolean'
|
||||
&& typeof item.reason === 'string'
|
||||
&& validSize
|
||||
&& targetMetadata
|
||||
&& hasOnlyPruneOwnershipLabels((item.volume as { ownershipLabels?: unknown } | undefined)?.ownershipLabels)
|
||||
&& hasOnlyPruneOwnershipLabels((item.network as { ownershipLabels?: unknown } | undefined)?.ownershipLabels);
|
||||
}
|
||||
|
||||
function projectRemoteItem(item: PrunePlanItem): PrunePlanItem {
|
||||
const base = {
|
||||
id: item.id,
|
||||
name: item.name,
|
||||
managed: item.managed,
|
||||
reason: item.reason,
|
||||
...(typeof item.sizeBytes === 'number' ? { sizeBytes: item.sizeBytes } : {}),
|
||||
...(typeof item.stackName === 'string' ? { stackName: item.stackName } : {}),
|
||||
};
|
||||
if (item.target === 'images') {
|
||||
return { ...base, target: 'images', image: {
|
||||
references: Array.isArray(item.image.references)
|
||||
? item.image.references.filter((reference): reference is string => typeof reference === 'string')
|
||||
: [],
|
||||
...(typeof item.image.digest === 'string' ? { digest: item.image.digest } : {}),
|
||||
...(typeof item.image.createdAt === 'number' ? { createdAt: item.image.createdAt } : {}),
|
||||
} };
|
||||
}
|
||||
if (item.target === 'volumes') {
|
||||
const ownershipLabels = projectPruneOwnershipLabels(item.volume.ownershipLabels);
|
||||
return { ...base, target: 'volumes', volume: {
|
||||
...(typeof item.volume.driver === 'string' ? { driver: item.volume.driver } : {}),
|
||||
...(ownershipLabels ? { ownershipLabels } : {}),
|
||||
} };
|
||||
}
|
||||
if (item.target === 'networks') {
|
||||
const ownershipLabels = projectPruneOwnershipLabels(item.network.ownershipLabels);
|
||||
return { ...base, target: 'networks', network: {
|
||||
...(typeof item.network.driver === 'string' ? { driver: item.network.driver } : {}),
|
||||
...(typeof item.network.scope === 'string' ? { scope: item.network.scope } : {}),
|
||||
...(ownershipLabels ? { ownershipLabels } : {}),
|
||||
} };
|
||||
}
|
||||
return { ...base, target: 'containers' };
|
||||
}
|
||||
|
||||
function projectRemotePlan(plan: PrunePlan): PrunePlan {
|
||||
return {
|
||||
scope: plan.scope,
|
||||
targets: [...plan.targets],
|
||||
items: plan.items.map(projectRemoteItem),
|
||||
reclaimableBytes: plan.reclaimableBytes,
|
||||
fingerprint: plan.fingerprint,
|
||||
createdAt: plan.createdAt,
|
||||
nodeId: plan.nodeId,
|
||||
};
|
||||
}
|
||||
|
||||
function isPruneItemOutcome(value: unknown): value is PruneItemOutcome {
|
||||
if (!value || typeof value !== 'object') return false;
|
||||
const outcome = value as { id?: unknown; target?: unknown; status?: unknown; sizeBytes?: unknown; reason?: unknown; error?: unknown };
|
||||
if (typeof outcome.id !== 'string' || typeof outcome.target !== 'string') return false;
|
||||
if (outcome.status === 'removed') return outcome.sizeBytes === undefined
|
||||
|| (typeof outcome.sizeBytes === 'number' && Number.isFinite(outcome.sizeBytes) && outcome.sizeBytes >= 0);
|
||||
if (outcome.status === 'skipped') return typeof outcome.reason === 'string';
|
||||
if (outcome.status === 'failed') return typeof outcome.error === 'string';
|
||||
return false;
|
||||
}
|
||||
|
||||
function validateRemoteOutcomes(
|
||||
value: unknown,
|
||||
plan: PrunePlan,
|
||||
targets: FleetPruneTarget[],
|
||||
): PruneItemOutcome[] | null {
|
||||
if (!Array.isArray(value) || value.length !== plan.items.length) return null;
|
||||
const requestedTargets = new Set<string>(targets);
|
||||
const expected = new Set(plan.items.map((item) => `${item.target}\0${item.id}`));
|
||||
const seen = new Set<string>();
|
||||
for (const outcome of value) {
|
||||
if (!isPruneItemOutcome(outcome) || !requestedTargets.has(outcome.target)) return null;
|
||||
const key = `${outcome.target}\0${outcome.id}`;
|
||||
if (!expected.has(key) || seen.has(key)) return null;
|
||||
seen.add(key);
|
||||
}
|
||||
return seen.size === expected.size ? value : null;
|
||||
}
|
||||
|
||||
async function buildLocalPreflight(node: Node, targets: FleetPruneTarget[], scope: PruneScope): Promise<Preflight> {
|
||||
try {
|
||||
const knownStacks = await FileSystemService.getInstance(node.id).getStacks();
|
||||
const controller = DockerController.getInstance(node.id);
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(node.id);
|
||||
const plan = await withTimeout(
|
||||
controller.buildPrunePlan(targets, scope, knownStacks, node.id, isImageHeld),
|
||||
PLAN_TIMEOUT_MS,
|
||||
'docker prune plan',
|
||||
);
|
||||
return { node, reachable: true, plan };
|
||||
} catch (error) {
|
||||
console.error(`[Fleet prune] Plan failed on ${sanitizeForLog(node.name)}: ${sanitizeForLog(getErrorMessage(error, 'Unknown error'))}`);
|
||||
return {
|
||||
node,
|
||||
reachable: true,
|
||||
code: error instanceof TimeoutError ? 'DOCKER_DAEMON_BUSY' : 'PRUNE_PLAN_FAILED',
|
||||
error: error instanceof TimeoutError ? BUSY_DAEMON_ERROR : getErrorMessage(error, 'Failed to build prune plan'),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchRemotePlan(node: Node, targets: FleetPruneTarget[], scope: PruneScope): Promise<Preflight> {
|
||||
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!proxyTarget) return { node, reachable: false, error: formatNoTargetError(node) };
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;
|
||||
try {
|
||||
const response = await fetch(`${proxyTarget.apiUrl.replace(/\/$/, '')}/api/system/prune/plan`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ targets, scope }),
|
||||
signal: AbortSignal.timeout(REMOTE_PLAN_TIMEOUT_MS),
|
||||
});
|
||||
const data: unknown = await response.json().catch(() => null);
|
||||
if (!response.ok) {
|
||||
const message = data && typeof data === 'object' && typeof (data as { error?: unknown }).error === 'string'
|
||||
? (data as { error: string }).error
|
||||
: `Remote returned ${response.status}`;
|
||||
return { node, reachable: true, code: 'REMOTE_PLAN_FAILED', error: message };
|
||||
}
|
||||
if (!isPrunePlan(data, targets, scope)) {
|
||||
return { node, reachable: true, code: 'REMOTE_PLAN_INVALID', error: 'Remote returned a malformed prune plan' };
|
||||
}
|
||||
return { node, reachable: true, plan: projectRemotePlan(data) };
|
||||
} catch (error) {
|
||||
console.error(`[Fleet prune] Remote plan transport failed for ${sanitizeForLog(node.name)}: ${sanitizeForLog(getErrorMessage(error, 'Unknown error'))}`);
|
||||
return { node, reachable: false, error: getErrorMessage(error, 'Failed to reach remote node') };
|
||||
}
|
||||
}
|
||||
|
||||
function buildPreflight(node: Node, targets: FleetPruneTarget[], scope: PruneScope): Promise<Preflight> {
|
||||
return node.type === 'local'
|
||||
? buildLocalPreflight(node, targets, scope)
|
||||
: fetchRemotePlan(node, targets, scope);
|
||||
}
|
||||
|
||||
function preflightResult(entry: Preflight, targets: FleetPruneTarget[]): FleetPruneNodeResult {
|
||||
if (entry.plan) {
|
||||
return {
|
||||
nodeId: entry.node.id,
|
||||
nodeName: entry.node.name,
|
||||
reachable: true,
|
||||
fingerprint: entry.plan.fingerprint,
|
||||
items: entry.plan.items,
|
||||
reclaimableBytes: entry.plan.reclaimableBytes,
|
||||
targets: targetRowsFromPlan(entry.plan, targets),
|
||||
};
|
||||
}
|
||||
const error = entry.error ?? 'Failed to build prune plan';
|
||||
return {
|
||||
nodeId: entry.node.id,
|
||||
nodeName: entry.node.name,
|
||||
reachable: entry.reachable,
|
||||
code: entry.code,
|
||||
error,
|
||||
reclaimableBytes: 0,
|
||||
targets: failedTargetRows(targets, error, true),
|
||||
};
|
||||
}
|
||||
|
||||
function outcomeTargetRows(
|
||||
targets: FleetPruneTarget[],
|
||||
outcomes: PruneItemOutcome[],
|
||||
fallbackSuccess = true,
|
||||
): FleetPruneTargetResult[] {
|
||||
return targets.map((target) => {
|
||||
const targetOutcomes = outcomes.filter((outcome) => outcome.target === target);
|
||||
const failed = targetOutcomes.filter((outcome) => outcome.status === 'failed');
|
||||
const skipped = targetOutcomes.filter((outcome) => outcome.status === 'skipped');
|
||||
const removed = targetOutcomes.filter((outcome) => outcome.status === 'removed');
|
||||
const removedBytes = removed.reduce((sum, outcome) => sum + (outcome.status === 'removed' ? outcome.sizeBytes ?? 0 : 0), 0);
|
||||
return {
|
||||
target,
|
||||
success: outcomes.length === 0 ? fallbackSuccess : failed.length === 0,
|
||||
reclaimedBytes: outcomes.length === 0 ? 0 : removedBytes,
|
||||
dryRun: false,
|
||||
removed: removed.length,
|
||||
skipped: skipped.length,
|
||||
failed: failed.length,
|
||||
error: failed.length > 0 ? failed.map((outcome) => outcome.status === 'failed' ? outcome.error : '').filter(Boolean).join('; ') : undefined,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function executeLocal(entry: Preflight, targets: FleetPruneTarget[]): Promise<FleetPruneNodeResult> {
|
||||
try {
|
||||
const plan = entry.plan;
|
||||
if (!plan) throw new Error('Local prune preflight is missing');
|
||||
const knownStacks = await FileSystemService.getInstance(entry.node.id).getStacks();
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(entry.node.id);
|
||||
const result = await DockerController.getInstance(entry.node.id).executePrunePlan(plan, knownStacks, isImageHeld);
|
||||
if (result.outcomes.some((outcome) => outcome.status === 'removed')) {
|
||||
try {
|
||||
invalidateNodeCaches(entry.node.id);
|
||||
} catch (error) {
|
||||
console.error(`[Fleet prune] Cache invalidation failed on ${sanitizeForLog(entry.node.name)}: ${sanitizeForLog(getErrorMessage(error, 'Unknown error'))}`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
nodeId: entry.node.id,
|
||||
nodeName: entry.node.name,
|
||||
reachable: true,
|
||||
reclaimedBytes: result.reclaimedBytes,
|
||||
outcomes: result.outcomes,
|
||||
targets: outcomeTargetRows(targets, result.outcomes),
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`[Fleet prune] Execution failed on ${sanitizeForLog(entry.node.name)}: ${sanitizeForLog(getErrorMessage(error, 'Unknown error'))}`);
|
||||
const stale = error instanceof PrunePlanStaleError;
|
||||
const message = getErrorMessage(error, stale ? 'Prune plan changed' : 'Prune failed');
|
||||
return {
|
||||
nodeId: entry.node.id,
|
||||
nodeName: entry.node.name,
|
||||
reachable: true,
|
||||
code: stale ? 'PRUNE_PLAN_STALE' : 'PRUNE_EXECUTE_FAILED',
|
||||
error: message,
|
||||
reclaimedBytes: 0,
|
||||
targets: failedTargetRows(targets, message, false),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function executeRemote(entry: Preflight, targets: FleetPruneTarget[], scope: PruneScope): Promise<FleetPruneNodeResult> {
|
||||
const plan = entry.plan;
|
||||
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(entry.node.id);
|
||||
if (!plan || !proxyTarget) {
|
||||
const error = 'Node became unreachable after fleet preflight';
|
||||
return {
|
||||
nodeId: entry.node.id, nodeName: entry.node.name, reachable: false, error,
|
||||
reclaimedBytes: 0, targets: failedTargetRows(targets, error, false),
|
||||
};
|
||||
}
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;
|
||||
try {
|
||||
const response = await fetch(`${proxyTarget.apiUrl.replace(/\/$/, '')}/api/system/prune/system`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ targets, scope, planFingerprint: plan.fingerprint }),
|
||||
signal: AbortSignal.timeout(REMOTE_PLAN_TIMEOUT_MS),
|
||||
});
|
||||
const data: unknown = await response.json().catch(() => null);
|
||||
const record = data && typeof data === 'object' ? data as Record<string, unknown> : null;
|
||||
if (!response.ok) {
|
||||
const message = typeof record?.error === 'string' ? record.error : `Remote returned ${response.status}`;
|
||||
return {
|
||||
nodeId: entry.node.id, nodeName: entry.node.name, reachable: true,
|
||||
code: typeof record?.code === 'string' ? record.code : 'REMOTE_PRUNE_FAILED',
|
||||
error: message, reclaimedBytes: 0, targets: failedTargetRows(targets, message, false),
|
||||
};
|
||||
}
|
||||
if (!record || typeof record.reclaimedBytes !== 'number' || !Number.isFinite(record.reclaimedBytes)
|
||||
|| record.reclaimedBytes < 0 || (record.success !== undefined && typeof record.success !== 'boolean')) {
|
||||
const error = 'Remote returned a malformed prune result';
|
||||
return {
|
||||
nodeId: entry.node.id, nodeName: entry.node.name, reachable: true,
|
||||
code: 'REMOTE_PRUNE_INVALID', error, reclaimedBytes: 0, targets: failedTargetRows(targets, error, false),
|
||||
};
|
||||
}
|
||||
const hasOutcomes = Object.prototype.hasOwnProperty.call(record, 'outcomes');
|
||||
const outcomes = hasOutcomes ? validateRemoteOutcomes(record.outcomes, plan, targets) : undefined;
|
||||
if (hasOutcomes && !outcomes) {
|
||||
const error = 'Remote returned malformed or incomplete prune outcomes';
|
||||
return {
|
||||
nodeId: entry.node.id, nodeName: entry.node.name, reachable: true,
|
||||
code: 'REMOTE_PRUNE_INVALID', error, reclaimedBytes: 0, targets: failedTargetRows(targets, error, false),
|
||||
};
|
||||
}
|
||||
return {
|
||||
nodeId: entry.node.id,
|
||||
nodeName: entry.node.name,
|
||||
reachable: true,
|
||||
reclaimedBytes: record.reclaimedBytes,
|
||||
outcomes: outcomes ?? undefined,
|
||||
targets: outcomeTargetRows(targets, outcomes ?? [], record.success !== false),
|
||||
};
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to reach remote node');
|
||||
console.error(`[Fleet prune] Remote execution transport failed for ${sanitizeForLog(entry.node.name)}: ${sanitizeForLog(message)}`);
|
||||
return {
|
||||
nodeId: entry.node.id, nodeName: entry.node.name, reachable: false,
|
||||
error: message, reclaimedBytes: 0, targets: failedTargetRows(targets, message, false),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function comparePreflight(
|
||||
preflights: Preflight[],
|
||||
reviewedNodes: ReviewedFleetNode[],
|
||||
plans: ReviewedFleetPlan[],
|
||||
): { code: string; error: string; nodeId?: number } | null {
|
||||
const reviewedById = new Map(reviewedNodes.map((node) => [node.nodeId, node]));
|
||||
const plansById = new Map(plans.map((plan) => [plan.nodeId, plan]));
|
||||
for (const entry of preflights) {
|
||||
const reviewed = reviewedById.get(entry.node.id);
|
||||
if (!reviewed) return { code: 'PRUNE_NODE_ROSTER_CHANGED', error: 'The fleet node roster changed after the dry run' };
|
||||
if (entry.reachable !== reviewed.reachable) {
|
||||
return {
|
||||
code: 'PRUNE_NODE_REACHABILITY_CHANGED',
|
||||
nodeId: entry.node.id,
|
||||
error: `Reachability changed for ${entry.node.name} after the dry run`,
|
||||
};
|
||||
}
|
||||
if (!reviewed.reachable) continue;
|
||||
if (!entry.plan) {
|
||||
return {
|
||||
code: entry.code ?? 'PRUNE_PLAN_FAILED',
|
||||
nodeId: entry.node.id,
|
||||
error: entry.error ?? `Failed to rebuild the plan for ${entry.node.name}`,
|
||||
};
|
||||
}
|
||||
if (entry.plan.fingerprint !== plansById.get(entry.node.id)?.fingerprint) {
|
||||
return {
|
||||
code: 'PRUNE_PLAN_STALE',
|
||||
nodeId: entry.node.id,
|
||||
error: `The prune plan changed on ${entry.node.name} after the dry run`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function runFleetPrune(
|
||||
nodes: Node[],
|
||||
request: ParsedFleetPruneRequest,
|
||||
activeLocks: Set<string>,
|
||||
): Promise<FleetPruneResponse> {
|
||||
if (request.dryRun) {
|
||||
const preflights = await Promise.all(nodes.map((node) => buildPreflight(node, request.targets, request.scope)));
|
||||
return { status: 200, body: { results: preflights.map((entry) => preflightResult(entry, request.targets)) } };
|
||||
}
|
||||
|
||||
const rosterError = validateReviewedRoster(nodes, request.reviewedNodes, request.plans);
|
||||
if (rosterError) return { status: 409, body: { code: 'PRUNE_NODE_ROSTER_CHANGED', error: rosterError } };
|
||||
|
||||
const lockKeys = nodes.filter((node) => node.type === 'local').map((node) => `bulk-prune:${node.id}`);
|
||||
const busyKey = lockKeys.find((key) => activeLocks.has(key));
|
||||
if (busyKey) return { status: 409, body: { code: 'PRUNE_ALREADY_RUNNING', error: 'A prune is already running on a reviewed node' } };
|
||||
for (const key of lockKeys) activeLocks.add(key);
|
||||
|
||||
try {
|
||||
const preflights = await Promise.all(nodes.map((node) => buildPreflight(node, request.targets, request.scope)));
|
||||
const conflict = comparePreflight(preflights, request.reviewedNodes, request.plans);
|
||||
if (conflict) {
|
||||
return {
|
||||
status: 409,
|
||||
body: { ...conflict, results: preflights.map((entry) => preflightResult(entry, request.targets)) },
|
||||
};
|
||||
}
|
||||
const reviewedReachable = new Set(request.reviewedNodes.filter((node) => node.reachable).map((node) => node.nodeId));
|
||||
const executable = preflights.filter((entry) => reviewedReachable.has(entry.node.id));
|
||||
const results = await Promise.all(executable.map((entry) => entry.node.type === 'local'
|
||||
? executeLocal(entry, request.targets)
|
||||
: executeRemote(entry, request.targets, request.scope)));
|
||||
return { status: 200, body: { results } };
|
||||
} finally {
|
||||
for (const key of lockKeys) activeLocks.delete(key);
|
||||
}
|
||||
}
|
||||
+19
-158
@@ -10,7 +10,6 @@ import { FleetUpdateTrackerService, type UpdateTracker, type TerminalStatus, UPD
|
||||
import { NodeRegistry } from '../services/NodeRegistry';
|
||||
import { computeNodeNetworkingSummary, type NodeNetworkingSummary } from '../services/network/networkingSummary';
|
||||
import DockerController from '../services/DockerController';
|
||||
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
|
||||
import { getHostMemory } from '../helpers/hostMemory';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { ComposeService } from '../services/ComposeService';
|
||||
@@ -47,8 +46,14 @@ import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
|
||||
import { formatNoTargetError } from '../utils/remoteTarget';
|
||||
import { CloudBackupService } from '../services/CloudBackupService';
|
||||
import { NotificationService } from '../services/NotificationService';
|
||||
import { invalidateNodeCaches, invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
|
||||
import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation';
|
||||
import { activeBulkActions } from './labels';
|
||||
import {
|
||||
FLEET_PRUNE_TARGETS,
|
||||
parseFleetPruneRequest,
|
||||
runFleetPrune,
|
||||
type FleetPruneTarget,
|
||||
} from '../helpers/fleetPrune';
|
||||
import { runLocalLabelStop, isLabelLocalStopResponse, type StackStopResult } from '../helpers/fleetLabelStop';
|
||||
import { collectFleetLabelSummaries } from '../helpers/fleetLabelSummary';
|
||||
import { runLocalLabelAssign, validateLabelTemplate, validateRemoteAssignResults, failAllAssign, type AssignNodeResult } from '../helpers/fleetLabelAssign';
|
||||
@@ -2202,167 +2207,23 @@ fleetRouter.post('/labels/bulk-assign', authMiddleware, async (req: Request, res
|
||||
}
|
||||
});
|
||||
|
||||
// Fleet-wide Docker prune. Fans out to every node, running per-target prune
|
||||
// (images/volumes/networks) under the chosen scope. Local nodes call
|
||||
// DockerController directly under a per-node bulk-prune lock; remote nodes
|
||||
// receive one POST /api/system/prune/system per target via the standard
|
||||
// Bearer-token path. Concurrent execution against the per-node prune route in
|
||||
// systemMaintenance.ts is safe because Docker's prune API is internally
|
||||
// serialized and idempotent (the worst case is a duplicate call returning 0
|
||||
// reclaimed bytes).
|
||||
// Fleet-wide Docker prune. Dry runs collect one itemized plan per node. Execute
|
||||
// validates the reviewed roster, preflights every plan, then starts mutation.
|
||||
// Tier: requireAdmin (admin-only fleet plumbing; available on every license).
|
||||
const FLEET_PRUNE_TARGETS = ['images', 'volumes', 'networks'] as const;
|
||||
type FleetPruneTarget = (typeof FLEET_PRUNE_TARGETS)[number];
|
||||
|
||||
fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
|
||||
const body = req.body as { targets?: unknown; scope?: unknown; dryRun?: unknown } | undefined;
|
||||
if (!body || typeof body !== 'object') {
|
||||
res.status(400).json({ error: 'Request body is required' });
|
||||
return;
|
||||
}
|
||||
const rawTargets = Array.isArray(body.targets) ? body.targets : null;
|
||||
if (!rawTargets || rawTargets.length === 0) {
|
||||
res.status(400).json({ error: 'targets must be a non-empty array' });
|
||||
return;
|
||||
}
|
||||
const dedup = new Set<FleetPruneTarget>();
|
||||
for (const t of rawTargets) {
|
||||
if (typeof t !== 'string' || !(FLEET_PRUNE_TARGETS as readonly string[]).includes(t)) {
|
||||
res.status(400).json({ error: `Invalid target: ${typeof t === 'string' ? t : typeof t}` });
|
||||
try {
|
||||
const parsed = parseFleetPruneRequest(req.body);
|
||||
if ('error' in parsed) {
|
||||
res.status(400).json({ error: parsed.error });
|
||||
return;
|
||||
}
|
||||
dedup.add(t as FleetPruneTarget);
|
||||
}
|
||||
const targets: FleetPruneTarget[] = Array.from(dedup);
|
||||
const scope: 'managed' | 'all' = body.scope === 'all' ? 'all' : 'managed';
|
||||
const isDryRun = body.dryRun === true;
|
||||
|
||||
type TargetResult = { target: FleetPruneTarget; success: boolean; reclaimedBytes: number; error?: string; dryRun?: boolean };
|
||||
type NodeResult = {
|
||||
nodeId: number; nodeName: string; reachable: boolean; error?: string; targets: TargetResult[];
|
||||
};
|
||||
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodes = db.getNodes();
|
||||
if (isDebugEnabled()) console.debug('[Fleet:debug] fleet-prune:', { targets, scope, dryRun: isDryRun, nodes: nodes.length });
|
||||
|
||||
const results: NodeResult[] = await Promise.all(nodes.map(async (node): Promise<NodeResult> => {
|
||||
if (node.type === 'local') {
|
||||
const lockKey = `bulk-prune:${node.id}`;
|
||||
if (activeBulkActions.has(lockKey)) {
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, reachable: true,
|
||||
targets: targets.map(t => ({ target: t, success: false, reclaimedBytes: 0, error: 'A prune is already running on this node' })),
|
||||
};
|
||||
}
|
||||
activeBulkActions.add(lockKey);
|
||||
try {
|
||||
const knownStacks = scope === 'managed' ? await FileSystemService.getInstance(node.id).getStacks() : [];
|
||||
const dockerController = DockerController.getInstance(node.id);
|
||||
const targetResults: TargetResult[] = [];
|
||||
let anySuccess = false;
|
||||
for (const target of targets) {
|
||||
try {
|
||||
if (isDryRun) {
|
||||
// estimateSystemReclaim hits `docker system df`; bound it
|
||||
// so a slow local daemon doesn't hang the fleet admin tab
|
||||
// (F-6). estimateManagedReclaim is fast (no df) and stays
|
||||
// unwrapped.
|
||||
const estimate = scope === 'managed'
|
||||
? await dockerController.estimateManagedReclaim(target, knownStacks)
|
||||
: await withTimeout(
|
||||
dockerController.estimateSystemReclaim(target, knownStacks),
|
||||
FLEET_DF_TIMEOUT_MS,
|
||||
'docker disk usage',
|
||||
);
|
||||
targetResults.push({ target, success: true, reclaimedBytes: estimate.reclaimableBytes, dryRun: true });
|
||||
continue;
|
||||
}
|
||||
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(node.id);
|
||||
const result = scope === 'managed'
|
||||
? await dockerController.pruneManagedOnly(target, knownStacks, isImageHeld)
|
||||
: await dockerController.pruneSystem(target, undefined, isImageHeld);
|
||||
targetResults.push({ target, success: true, reclaimedBytes: result.reclaimedBytes });
|
||||
if (result.reclaimedBytes > 0 || result.success) anySuccess = true;
|
||||
} catch (err) {
|
||||
const error = err instanceof TimeoutError
|
||||
? 'Docker daemon is busy. Please try again in a moment.'
|
||||
: getErrorMessage(err, 'Prune failed');
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error });
|
||||
}
|
||||
}
|
||||
if (anySuccess && !isDryRun) invalidateNodeCaches(node.id);
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: true, targets: targetResults };
|
||||
} finally {
|
||||
activeBulkActions.delete(lockKey);
|
||||
}
|
||||
}
|
||||
|
||||
// Remote node: POST /api/system/prune/system per target, short-circuiting
|
||||
// on the first transport-level failure so we don't hammer a dead node.
|
||||
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!proxyTarget) {
|
||||
const error = formatNoTargetError(node);
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, reachable: false, error,
|
||||
targets: targets.map(t => ({ target: t, success: false, reclaimedBytes: 0, error })),
|
||||
};
|
||||
}
|
||||
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
|
||||
const remoteHeaders: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (proxyTarget.apiToken) remoteHeaders.Authorization = `Bearer ${proxyTarget.apiToken}`;
|
||||
const targetResults: TargetResult[] = [];
|
||||
let nodeUnreachable: string | null = null;
|
||||
for (const target of targets) {
|
||||
if (nodeUnreachable) {
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error: nodeUnreachable });
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/api/system/prune/system`, {
|
||||
method: 'POST',
|
||||
headers: remoteHeaders,
|
||||
body: JSON.stringify({ target, scope, dryRun: isDryRun }),
|
||||
signal: AbortSignal.timeout(120000),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errBody = (await response.json().catch(() => ({}))) as { error?: string };
|
||||
const message = errBody.error || `Remote returned ${response.status}`;
|
||||
nodeUnreachable = message;
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error: message });
|
||||
continue;
|
||||
}
|
||||
const remote = (await response.json().catch(() => null)) as { success?: boolean; reclaimedBytes?: number; dryRun?: boolean } | null;
|
||||
if (!remote || typeof remote.reclaimedBytes !== 'number') {
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error: 'Invalid response from remote node' });
|
||||
continue;
|
||||
}
|
||||
const entry: TargetResult = { target, success: remote.success !== false, reclaimedBytes: remote.reclaimedBytes };
|
||||
if (remote.dryRun) entry.dryRun = true;
|
||||
targetResults.push(entry);
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'Failed to reach remote node');
|
||||
nodeUnreachable = message;
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error: message });
|
||||
}
|
||||
}
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name,
|
||||
reachable: nodeUnreachable === null,
|
||||
error: nodeUnreachable ?? undefined,
|
||||
targets: targetResults,
|
||||
};
|
||||
}));
|
||||
|
||||
if (isDebugEnabled()) {
|
||||
const reachable = results.filter(r => r.reachable).length;
|
||||
const reclaimed = results.reduce((n, r) => n + r.targets.reduce((m, t) => m + t.reclaimedBytes, 0), 0);
|
||||
console.debug('[Fleet:debug] fleet-prune complete:', { reachable, unreachable: results.length - reachable, reclaimedBytes: reclaimed });
|
||||
}
|
||||
res.json({ results });
|
||||
const response = await runFleetPrune(
|
||||
DatabaseService.getInstance().getNodes(),
|
||||
parsed.request,
|
||||
activeBulkActions,
|
||||
);
|
||||
res.status(response.status).json(response.body);
|
||||
} catch (error) {
|
||||
console.error('[Fleet] fleet-prune error:', error);
|
||||
res.status(500).json({ error: getErrorMessage(error, 'Failed to run fleet prune') });
|
||||
|
||||
@@ -16,12 +16,11 @@ import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage, isSqliteUniqueViolation } from '../utils/errors';
|
||||
import { parseIntParam } from '../utils/parseIntParam';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { activeBulkActions } from '../helpers/bulkActionLocks';
|
||||
|
||||
// Module-scope lock shared by `POST /api/labels/:id/action` and the fleet-wide
|
||||
// bulk endpoints in `routes/fleet.ts`. Keyed by `${nodeId}` so concurrent bulk
|
||||
// actions targeting the same node serialize and a fleet-stop cannot race a
|
||||
// per-label action on the same containers.
|
||||
export const activeBulkActions = new Set<string>();
|
||||
// Shared with the fleet-wide bulk endpoints. Label actions use `${nodeId}` so
|
||||
// a fleet stop cannot race a per-label action on the same containers.
|
||||
export { activeBulkActions };
|
||||
|
||||
export const labelsRouter = Router();
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryS
|
||||
import SelfIdentityService from '../services/SelfIdentityService';
|
||||
import { requireAdmin } from '../middleware/tierGates';
|
||||
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
|
||||
import { activeBulkActions } from '../helpers/bulkActionLocks';
|
||||
import { isValidDockerResourceId, isValidCidr, isValidIPv4 } from '../utils/validation';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
@@ -155,6 +156,7 @@ systemMaintenanceRouter.post('/prune/plan', async (req: Request, res: Response)
|
||||
|
||||
systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response) => {
|
||||
if (!requireAdmin(req, res)) return;
|
||||
let pruneLockHeld = false;
|
||||
try {
|
||||
const body = req.body as {
|
||||
target?: unknown;
|
||||
@@ -200,8 +202,17 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
return;
|
||||
}
|
||||
|
||||
// Resources path: fingerprint-bound execute. Fleet still calls without a
|
||||
// fingerprint and keeps the legacy pruneManagedOnly / pruneSystem path.
|
||||
const pruneLockKey = `bulk-prune:${req.nodeId}`;
|
||||
if (activeBulkActions.has(pruneLockKey)) {
|
||||
return res.status(409).json({
|
||||
error: 'A prune is already running on this node',
|
||||
code: 'PRUNE_ALREADY_RUNNING',
|
||||
});
|
||||
}
|
||||
activeBulkActions.add(pruneLockKey);
|
||||
pruneLockHeld = true;
|
||||
|
||||
// Fingerprint-bound execute used by Resources and Fleet.
|
||||
if (planFingerprint) {
|
||||
const built = await withTimeout(
|
||||
dockerController.buildPrunePlan(targets, pruneScope, knownStacks, req.nodeId, isImageHeld),
|
||||
@@ -231,7 +242,7 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
success: result.success,
|
||||
});
|
||||
}
|
||||
if (built.targets.includes('containers')) {
|
||||
if (result.outcomes.some((outcome) => outcome.status === 'removed')) {
|
||||
invalidateNodeCaches(req.nodeId);
|
||||
}
|
||||
res.json({
|
||||
@@ -288,6 +299,8 @@ systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response
|
||||
}
|
||||
console.error('System prune error:', error);
|
||||
res.status(500).json({ error: 'System prune failed' });
|
||||
} finally {
|
||||
if (pruneLockHeld) activeBulkActions.delete(`bulk-prune:${req.nodeId}`);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import SelfIdentityService from './SelfIdentityService';
|
||||
import {
|
||||
fingerprintPrunePlan,
|
||||
normalizePruneTargets,
|
||||
projectPruneOwnershipLabels,
|
||||
PRUNEABLE_CONTAINER_STATES,
|
||||
PrunePlanStaleError,
|
||||
type PruneItemOutcome,
|
||||
@@ -921,17 +922,18 @@ class DockerController {
|
||||
if (selfIdentity.isOwnContainer(c.Id)) continue;
|
||||
const state = String(c.State ?? '').toLowerCase();
|
||||
if (!PRUNEABLE_CONTAINER_STATES.has(state)) continue;
|
||||
if (scope === 'managed') {
|
||||
const stack = DockerController.resolveContainerStack(
|
||||
c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
||||
);
|
||||
if (!stack) continue;
|
||||
}
|
||||
const stack = DockerController.resolveContainerStack(
|
||||
c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
||||
);
|
||||
if (scope === 'managed' && !stack) continue;
|
||||
items.push({
|
||||
target: 'containers',
|
||||
id: c.Id,
|
||||
name: containerName(c),
|
||||
sizeBytes: typeof c.SizeRw === 'number' && c.SizeRw > 0 ? c.SizeRw : undefined,
|
||||
managed: Boolean(stack),
|
||||
reason: `Container is ${state} and no longer running`,
|
||||
stackName: stack ?? undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -946,23 +948,29 @@ class DockerController {
|
||||
const rawVolumeData = await this.docker.listVolumes();
|
||||
const rawVolumes = (this.validateApiData<{ Volumes?: Array<{
|
||||
Name: string;
|
||||
Driver?: string;
|
||||
Labels?: Record<string, string>;
|
||||
}> }>(rawVolumeData)).Volumes || [];
|
||||
for (const vol of rawVolumes) {
|
||||
if (selfIdentity.isOwnVolume(vol.Name)) continue;
|
||||
const usage = volumeUsage.get(vol.Name);
|
||||
if (!usage || usage.refCount !== 0) continue;
|
||||
if (scope === 'managed') {
|
||||
const stack = DockerController.resolveProjectLabel(
|
||||
vol.Labels?.['com.docker.compose.project'], knownSet, projectToStack,
|
||||
);
|
||||
if (!stack) continue;
|
||||
}
|
||||
const stack = DockerController.resolveContainerStack(
|
||||
vol.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
||||
);
|
||||
if (scope === 'managed' && !stack) continue;
|
||||
items.push({
|
||||
target: 'volumes',
|
||||
id: vol.Name,
|
||||
name: vol.Name,
|
||||
sizeBytes: usage.size > 0 ? usage.size : undefined,
|
||||
managed: Boolean(stack),
|
||||
reason: 'Volume is not referenced by any container',
|
||||
stackName: stack ?? undefined,
|
||||
volume: {
|
||||
driver: vol.Driver,
|
||||
ownershipLabels: projectPruneOwnershipLabels(vol.Labels),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -971,6 +979,8 @@ class DockerController {
|
||||
const rawNetworks = await this.docker.listNetworks() as Array<{
|
||||
Id: string;
|
||||
Name: string;
|
||||
Driver?: string;
|
||||
Scope?: string;
|
||||
Labels?: Record<string, string>;
|
||||
}>;
|
||||
const networksInUse = new Set<string>();
|
||||
@@ -998,19 +1008,30 @@ class DockerController {
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (scope === 'managed') {
|
||||
const stack = DockerController.resolveProjectLabel(
|
||||
net.Labels?.['com.docker.compose.project'], knownSet, projectToStack,
|
||||
);
|
||||
if (!stack) continue;
|
||||
}
|
||||
items.push({ target: 'networks', id: net.Id, name: net.Name });
|
||||
const stack = DockerController.resolveContainerStack(
|
||||
net.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
||||
);
|
||||
if (scope === 'managed' && !stack) continue;
|
||||
items.push({
|
||||
target: 'networks',
|
||||
id: net.Id,
|
||||
name: net.Name,
|
||||
managed: Boolean(stack),
|
||||
reason: 'Network has no attached containers',
|
||||
stackName: stack ?? undefined,
|
||||
network: {
|
||||
driver: net.Driver,
|
||||
scope: net.Scope,
|
||||
ownershipLabels: projectPruneOwnershipLabels(net.Labels),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (ordered.includes('images')) {
|
||||
const unmanagedImageIds = new Set<string>();
|
||||
const managedImageIds = new Set<string>();
|
||||
const imageToStack = new Map<string, string>();
|
||||
const imageToContainerIds = new Map<string, string[]>();
|
||||
for (const c of allContainers) {
|
||||
if (!c.ImageID) continue;
|
||||
@@ -1020,7 +1041,10 @@ class DockerController {
|
||||
const stack = DockerController.resolveContainerStack(
|
||||
c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
||||
);
|
||||
if (stack) managedImageIds.add(c.ImageID);
|
||||
if (stack) {
|
||||
managedImageIds.add(c.ImageID);
|
||||
if (!imageToStack.has(c.ImageID)) imageToStack.set(c.ImageID, stack);
|
||||
}
|
||||
else unmanagedImageIds.add(c.ImageID);
|
||||
}
|
||||
const plannedContainerIds = new Set(
|
||||
@@ -1029,10 +1053,12 @@ class DockerController {
|
||||
const rawImages = await this.docker.listImages({ all: false }) as Array<{
|
||||
Id: string;
|
||||
RepoTags?: string[] | null;
|
||||
RepoDigests?: string[] | null;
|
||||
Labels?: Record<string, string>;
|
||||
Size?: number;
|
||||
VirtualSize?: number;
|
||||
Containers?: number;
|
||||
Created?: number;
|
||||
}>;
|
||||
// An image becomes free only when every container that references it is
|
||||
// also in this plan (not merely when any planned container uses it).
|
||||
@@ -1040,31 +1066,39 @@ class DockerController {
|
||||
for (const img of rawImages) {
|
||||
if (selfIdentity.isOwnImage(img.Id)) continue;
|
||||
if (isImageHeld?.(img.Id)) continue;
|
||||
const containers = img.Containers ?? 0;
|
||||
const refs = imageToContainerIds.get(img.Id) ?? [];
|
||||
const becomesFree = freeingImages
|
||||
&& refs.length > 0
|
||||
&& refs.length >= containers
|
||||
&& refs.every((id) => plannedContainerIds.has(id));
|
||||
if (containers > 0 && !becomesFree) continue;
|
||||
if (refs.length > 0 && !becomesFree) continue;
|
||||
const labeled = DockerController.resolveContainerStack(
|
||||
img.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
||||
);
|
||||
const stack = labeled ?? imageToStack.get(img.Id) ?? null;
|
||||
if (scope === 'managed') {
|
||||
if (unmanagedImageIds.has(img.Id)) continue;
|
||||
const labeled = DockerController.resolveProjectLabel(
|
||||
img.Labels?.['com.docker.compose.project'], knownSet, projectToStack,
|
||||
);
|
||||
// Unattributed unused images (no managed container, no compose label)
|
||||
// are not Sencho-managed; keep them out of managed prune.
|
||||
if (!becomesFree && !labeled && !managedImageIds.has(img.Id)) continue;
|
||||
if (!becomesFree && !stack && !managedImageIds.has(img.Id)) continue;
|
||||
}
|
||||
const name = img.RepoTags?.[0] && img.RepoTags[0] !== '<none>:<none>'
|
||||
? img.RepoTags[0]
|
||||
: img.Id.slice(0, 12);
|
||||
const references = (img.RepoTags ?? []).filter((ref) => ref && ref !== '<none>:<none>');
|
||||
const name = references[0] ?? '<none>:<none>';
|
||||
const unique = DockerController.imageUniqueBytes(img, sharedSizes);
|
||||
items.push({
|
||||
target: 'images',
|
||||
id: img.Id,
|
||||
name,
|
||||
sizeBytes: unique > 0 ? unique : undefined,
|
||||
managed: Boolean(stack || managedImageIds.has(img.Id)),
|
||||
reason: becomesFree
|
||||
? 'Image becomes unused after planned container removal'
|
||||
: 'Image is not used by any container',
|
||||
stackName: stack ?? undefined,
|
||||
image: {
|
||||
references,
|
||||
digest: img.RepoDigests?.find((digest) => Boolean(digest)),
|
||||
createdAt: typeof img.Created === 'number' ? img.Created : undefined,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1093,7 +1127,8 @@ class DockerController {
|
||||
|
||||
/**
|
||||
* Rebuild the plan with the same targets/scope. Returns the fresh plan when
|
||||
* the fingerprint still matches, otherwise null (caller maps to 409).
|
||||
* the fingerprint still matches, otherwise null so the caller can report
|
||||
* staleness through its route-specific response contract.
|
||||
*/
|
||||
public async assertPlanFresh(
|
||||
plan: PrunePlan,
|
||||
@@ -1173,14 +1208,18 @@ class DockerController {
|
||||
}
|
||||
|
||||
if (target === 'volumes') {
|
||||
const outcome = await this.executePlannedVolume(item, fresh.scope, knownSet, projectToStack, selfIdentity);
|
||||
const outcome = await this.executePlannedVolume(
|
||||
item, fresh.scope, knownSet, projectToStack, absDirToStack, resolvedBase, selfIdentity,
|
||||
);
|
||||
outcomes.push(outcome);
|
||||
if (outcome.status === 'removed') reclaimedBytes += outcome.sizeBytes ?? item.sizeBytes ?? 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (target === 'networks') {
|
||||
outcomes.push(await this.executePlannedNetwork(item, fresh.scope, knownSet, projectToStack, selfIdentity));
|
||||
outcomes.push(await this.executePlannedNetwork(
|
||||
item, fresh.scope, knownSet, projectToStack, absDirToStack, resolvedBase, selfIdentity,
|
||||
));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1207,7 +1246,7 @@ class DockerController {
|
||||
}
|
||||
}
|
||||
const outcome = await this.executePlannedImage(
|
||||
item, fresh.scope, knownSet, projectToStack, absDirToStack, resolvedBase, selfIdentity,
|
||||
item, selfIdentity,
|
||||
);
|
||||
outcomes.push(outcome);
|
||||
if (outcome.status === 'removed') reclaimedBytes += outcome.sizeBytes ?? item.sizeBytes ?? 0;
|
||||
@@ -1238,9 +1277,10 @@ class DockerController {
|
||||
|
||||
private async imageStillReferenced(imageId: string): Promise<boolean> {
|
||||
try {
|
||||
const images = await this.docker.listImages({ all: false }) as Array<{ Id: string; Containers?: number }>;
|
||||
const match = images.find((img) => img.Id === imageId || img.Id.startsWith(imageId) || imageId.startsWith(img.Id));
|
||||
return (match?.Containers ?? 0) > 0;
|
||||
const containers = await this.docker.listContainers({ all: true }) as Array<{ ImageID?: string }>;
|
||||
return containers.some((container) => container.ImageID === imageId
|
||||
|| Boolean(container.ImageID?.startsWith(imageId))
|
||||
|| imageId.startsWith(container.ImageID ?? ''));
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
@@ -1250,8 +1290,8 @@ class DockerController {
|
||||
item: PrunePlanItem,
|
||||
scope: PruneScope,
|
||||
knownSet: Set<string>,
|
||||
projectToStack: Record<string, string>,
|
||||
absDirToStack: Record<string, string>,
|
||||
projectToStack: Map<string, string>,
|
||||
absDirToStack: Map<string, string>,
|
||||
resolvedBase: string,
|
||||
selfIdentity: SelfIdentityService,
|
||||
): Promise<
|
||||
@@ -1305,7 +1345,9 @@ class DockerController {
|
||||
item: PrunePlanItem,
|
||||
scope: PruneScope,
|
||||
knownSet: Set<string>,
|
||||
projectToStack: Record<string, string>,
|
||||
projectToStack: Map<string, string>,
|
||||
absDirToStack: Map<string, string>,
|
||||
resolvedBase: string,
|
||||
selfIdentity: SelfIdentityService,
|
||||
): Promise<PruneItemOutcome> {
|
||||
if (selfIdentity.isOwnVolume(item.id)) {
|
||||
@@ -1325,8 +1367,8 @@ class DockerController {
|
||||
return { id: item.id, target: 'volumes', status: 'skipped', reason: 'Volume is in use' };
|
||||
}
|
||||
if (scope === 'managed') {
|
||||
const stack = DockerController.resolveProjectLabel(
|
||||
vol.Labels?.['com.docker.compose.project'], knownSet, projectToStack,
|
||||
const stack = DockerController.resolveContainerStack(
|
||||
vol.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
||||
);
|
||||
if (!stack) {
|
||||
return { id: item.id, target: 'volumes', status: 'skipped', reason: 'No longer a managed volume' };
|
||||
@@ -1340,7 +1382,9 @@ class DockerController {
|
||||
item: PrunePlanItem,
|
||||
scope: PruneScope,
|
||||
knownSet: Set<string>,
|
||||
projectToStack: Record<string, string>,
|
||||
projectToStack: Map<string, string>,
|
||||
absDirToStack: Map<string, string>,
|
||||
resolvedBase: string,
|
||||
selfIdentity: SelfIdentityService,
|
||||
): Promise<PruneItemOutcome> {
|
||||
if (selfIdentity.isOwnNetwork(item.id)) {
|
||||
@@ -1363,8 +1407,8 @@ class DockerController {
|
||||
return { id: item.id, target: 'networks', status: 'skipped', reason: 'Network is in use' };
|
||||
}
|
||||
if (scope === 'managed') {
|
||||
const stack = DockerController.resolveProjectLabel(
|
||||
inspected.Labels?.['com.docker.compose.project'], knownSet, projectToStack,
|
||||
const stack = DockerController.resolveContainerStack(
|
||||
inspected.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
||||
);
|
||||
if (!stack) {
|
||||
return { id: item.id, target: 'networks', status: 'skipped', reason: 'No longer a managed network' };
|
||||
@@ -1376,11 +1420,6 @@ class DockerController {
|
||||
|
||||
private async executePlannedImage(
|
||||
item: PrunePlanItem,
|
||||
scope: PruneScope,
|
||||
knownSet: Set<string>,
|
||||
projectToStack: Record<string, string>,
|
||||
absDirToStack: Record<string, string>,
|
||||
resolvedBase: string,
|
||||
selfIdentity: SelfIdentityService,
|
||||
): Promise<PruneItemOutcome> {
|
||||
if (selfIdentity.isOwnImage(item.id)) {
|
||||
@@ -1389,30 +1428,21 @@ class DockerController {
|
||||
const rawImages = await this.docker.listImages({ all: false }) as Array<{
|
||||
Id: string;
|
||||
Size?: number;
|
||||
Containers?: number;
|
||||
}>;
|
||||
const img = rawImages.find((i) => i.Id === item.id || i.Id.startsWith(item.id) || item.id.startsWith(i.Id));
|
||||
if (!img) {
|
||||
return { id: item.id, target: 'images', status: 'skipped', reason: 'Image no longer exists' };
|
||||
}
|
||||
if ((img.Containers ?? 0) > 0) {
|
||||
const allContainers = await this.docker.listContainers({ all: true }) as Array<{
|
||||
ImageID?: string;
|
||||
Labels?: Record<string, string>;
|
||||
}>;
|
||||
const references = allContainers.filter((container) => container.ImageID === img.Id
|
||||
|| Boolean(container.ImageID?.startsWith(img.Id))
|
||||
|| img.Id.startsWith(container.ImageID ?? ''));
|
||||
if (references.length > 0) {
|
||||
return { id: item.id, target: 'images', status: 'skipped', reason: 'Image still has container references' };
|
||||
}
|
||||
if (scope === 'managed') {
|
||||
const allContainers = await this.docker.listContainers({ all: true }) as Array<{
|
||||
ImageID?: string;
|
||||
Labels?: Record<string, string>;
|
||||
}>;
|
||||
for (const c of allContainers) {
|
||||
if (c.ImageID !== img.Id) continue;
|
||||
const stack = DockerController.resolveContainerStack(
|
||||
c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
||||
);
|
||||
if (!stack) {
|
||||
return { id: item.id, target: 'images', status: 'skipped', reason: 'Image referenced by unmanaged container' };
|
||||
}
|
||||
}
|
||||
}
|
||||
await this.docker.getImage(item.id).remove({ force: false });
|
||||
// Prefer plan unique-bytes; do not fall back to full Size (shared layers).
|
||||
return { id: item.id, target: 'images', status: 'removed', sizeBytes: item.sizeBytes ?? 0 };
|
||||
@@ -1925,19 +1955,20 @@ class DockerController {
|
||||
private static resolveProjectLabel(
|
||||
project: string | undefined,
|
||||
knownSet: Set<string>,
|
||||
projectToStack: Record<string, string>,
|
||||
projectToStack: Map<string, string>,
|
||||
): string | null {
|
||||
if (!project) return null;
|
||||
if (knownSet.has(project)) return project;
|
||||
if (projectToStack[project]) return projectToStack[project];
|
||||
return null;
|
||||
return projectToStack.get(project) ?? null;
|
||||
}
|
||||
|
||||
/** Builds a map from absolute stack directory paths to stack names. */
|
||||
private static buildAbsDirMap(stackNames: string[]): Record<string, string> {
|
||||
const map: Record<string, string> = {};
|
||||
private static buildAbsDirMap(stackNames: string[]): Map<string, string> {
|
||||
const map = new Map<string, string>();
|
||||
for (const stackDir of stackNames) {
|
||||
map[path.join(COMPOSE_DIR, stackDir)] = stackDir;
|
||||
const stackPath = path.join(COMPOSE_DIR, stackDir);
|
||||
map.set(stackPath, stackDir);
|
||||
map.set(path.resolve(stackPath), stackDir);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
@@ -1948,21 +1979,24 @@ class DockerController {
|
||||
*/
|
||||
private static resolveContainerStack(
|
||||
containerLabels: Record<string, string> | undefined,
|
||||
projectToStack: Record<string, string>,
|
||||
projectToStack: Map<string, string>,
|
||||
knownStackSet: Set<string>,
|
||||
absDirToStack: Record<string, string>,
|
||||
absDirToStack: Map<string, string>,
|
||||
resolvedBase: string,
|
||||
): string | null {
|
||||
if (!containerLabels) return null;
|
||||
|
||||
// Primary: match by project name (handles name: overrides and standard directory-based names)
|
||||
const project = containerLabels['com.docker.compose.project'];
|
||||
if (project && projectToStack[project]) return projectToStack[project];
|
||||
if (project) {
|
||||
const stack = projectToStack.get(project);
|
||||
if (stack) return stack;
|
||||
}
|
||||
|
||||
// Fallback 1: match by working_dir
|
||||
const workingDir = containerLabels['com.docker.compose.project.working_dir'];
|
||||
if (workingDir) {
|
||||
const match = absDirToStack[workingDir] ?? absDirToStack[path.resolve(workingDir)];
|
||||
const match = absDirToStack.get(workingDir) ?? absDirToStack.get(path.resolve(workingDir));
|
||||
if (match) return match;
|
||||
}
|
||||
|
||||
@@ -1989,15 +2023,15 @@ class DockerController {
|
||||
* Builds (or returns cached) mapping from Docker project name to Sencho stack directory name.
|
||||
* Compose files with a top-level `name:` field override the default project name.
|
||||
*/
|
||||
private static async resolveProjectNameMap(stackNames: string[]): Promise<Record<string, string>> {
|
||||
private static async resolveProjectNameMap(stackNames: string[]): Promise<Map<string, string>> {
|
||||
return CacheService.getInstance().getOrFetch(
|
||||
PROJECT_NAME_CACHE_KEY,
|
||||
PROJECT_NAME_CACHE_TTL_MS,
|
||||
async () => {
|
||||
const map: Record<string, string> = {};
|
||||
const map = new Map<string, string>();
|
||||
|
||||
await Promise.all(stackNames.map(async (stackDir) => {
|
||||
map[stackDir] = stackDir;
|
||||
map.set(stackDir, stackDir);
|
||||
|
||||
for (const fileName of COMPOSE_FILE_NAMES) {
|
||||
const filePath = path.join(COMPOSE_DIR, stackDir, fileName);
|
||||
@@ -2005,7 +2039,7 @@ class DockerController {
|
||||
const content = await fs.readFile(filePath, 'utf-8');
|
||||
const parsed = yaml.parse(content);
|
||||
if (parsed?.name && typeof parsed.name === 'string') {
|
||||
map[parsed.name] = stackDir;
|
||||
map.set(parsed.name, stackDir);
|
||||
}
|
||||
break;
|
||||
} catch (err: unknown) {
|
||||
|
||||
@@ -3,13 +3,47 @@ import { createHash } from 'crypto';
|
||||
export type PruneTarget = 'images' | 'volumes' | 'networks' | 'containers';
|
||||
export type PruneScope = 'managed' | 'all';
|
||||
|
||||
export interface PrunePlanItem {
|
||||
target: PruneTarget;
|
||||
interface PrunePlanItemBase {
|
||||
id: string;
|
||||
name: string;
|
||||
sizeBytes?: number;
|
||||
managed: boolean;
|
||||
reason: string;
|
||||
stackName?: string;
|
||||
}
|
||||
|
||||
export type PrunePlanItem =
|
||||
| (PrunePlanItemBase & { target: 'containers'; image?: never; volume?: never; network?: never })
|
||||
| (PrunePlanItemBase & {
|
||||
target: 'images';
|
||||
image: {
|
||||
references: string[];
|
||||
digest?: string;
|
||||
createdAt?: number;
|
||||
};
|
||||
volume?: never;
|
||||
network?: never;
|
||||
})
|
||||
| (PrunePlanItemBase & {
|
||||
target: 'volumes';
|
||||
volume: {
|
||||
driver?: string;
|
||||
ownershipLabels?: Record<string, string>;
|
||||
};
|
||||
image?: never;
|
||||
network?: never;
|
||||
})
|
||||
| (PrunePlanItemBase & {
|
||||
target: 'networks';
|
||||
network: {
|
||||
driver?: string;
|
||||
scope?: string;
|
||||
ownershipLabels?: Record<string, string>;
|
||||
};
|
||||
image?: never;
|
||||
volume?: never;
|
||||
});
|
||||
|
||||
export interface PrunePlan {
|
||||
scope: PruneScope;
|
||||
/** Ordered execution sequence (dependency-safe when multi-target). */
|
||||
@@ -34,6 +68,36 @@ export const PRUNE_EXECUTION_ORDER: readonly PruneTarget[] = ['volumes', 'contai
|
||||
|
||||
export const PRUNEABLE_CONTAINER_STATES = new Set(['created', 'exited', 'dead']);
|
||||
|
||||
const COMPOSE_OWNERSHIP_LABEL_KEYS = new Set([
|
||||
'com.docker.compose.project',
|
||||
'com.docker.compose.project.working_dir',
|
||||
'com.docker.compose.project.config_files',
|
||||
'com.docker.compose.volume',
|
||||
'com.docker.compose.network',
|
||||
'com.docker.compose.service',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Disclosure allowlist for ownership evidence returned to API clients.
|
||||
* Do not broaden it without reviewing Docker label values for sensitive data.
|
||||
*/
|
||||
export function projectPruneOwnershipLabels(value: unknown): Record<string, string> | undefined {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
|
||||
const projected = Object.entries(value).filter(
|
||||
(entry): entry is [string, string] => COMPOSE_OWNERSHIP_LABEL_KEYS.has(entry[0])
|
||||
&& typeof entry[1] === 'string' && entry[1].length > 0,
|
||||
);
|
||||
return projected.length > 0 ? Object.fromEntries(projected) : undefined;
|
||||
}
|
||||
|
||||
export function hasOnlyPruneOwnershipLabels(value: unknown): boolean {
|
||||
if (value === undefined) return true;
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
return Object.entries(value).every(
|
||||
([key, label]) => COMPOSE_OWNERSHIP_LABEL_KEYS.has(key) && typeof label === 'string' && label.length > 0,
|
||||
);
|
||||
}
|
||||
|
||||
export function isPruneTarget(value: unknown): value is PruneTarget {
|
||||
return typeof value === 'string' && (PRUNE_TARGETS as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user