mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-27 18:57:09 +00:00
feat(fleet): add fleet-wide Docker prune to Fleet Actions (#1104)
Adds a third card to the Fleet Actions tab that fans out Docker prune (images, volumes, networks) across every node in one submit. Local nodes call DockerController under a bulk-prune lock; remote nodes receive one POST /api/system/prune/system per target. Per-node + per-target results with reclaimed bytes are surfaced inline via ResultsList. Tier: Skipper / Admiral (requirePaid + requireAdmin), matching the rest of Fleet Actions. The frontend card is mounted inside the existing isPaid branch at FleetActionsTab; no new frontend gate is required. The card uses an amber accent rail and the Eraser icon so it reads as 'cleanup' rather than 'destructive stop'. Scope toggle defaults to Managed only (Sencho-tagged resources) with an All unused option that escalates the destructive-confirm copy.
This commit is contained in:
Anso
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* Tests for the fleet-wide Docker prune endpoint. Covers auth, tier gating,
|
||||
* input validation, local node orchestration with mocked DockerController,
|
||||
* remote-node fan-out with mocked fetch, lock contention, and partial failures.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let authHeader: string;
|
||||
let LicenseService: typeof import('../services/LicenseService').LicenseService;
|
||||
let DockerController: typeof import('../services/DockerController').default;
|
||||
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let activeBulkActions: typeof import('../routes/labels').activeBulkActions;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
({ LicenseService } = await import('../services/LicenseService'));
|
||||
({ default: DockerController } = await import('../services/DockerController'));
|
||||
({ FileSystemService } = await import('../services/FileSystemService'));
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ activeBulkActions } = await import('../routes/labels'));
|
||||
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
|
||||
authHeader = `Bearer ${token}`;
|
||||
});
|
||||
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
activeBulkActions.clear();
|
||||
});
|
||||
|
||||
function mockTier(tier: 'paid' | 'community') {
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue(tier);
|
||||
}
|
||||
|
||||
function mockLocalPrune(opts: { managedBytes?: Partial<Record<string, number>>; allBytes?: Partial<Record<string, number>>; throwOn?: string } = {}) {
|
||||
const fake = {
|
||||
pruneManagedOnly: vi.fn(async (target: string) => {
|
||||
if (opts.throwOn === target) throw new Error(`mock pruneManagedOnly threw for ${target}`);
|
||||
return { success: true, reclaimedBytes: opts.managedBytes?.[target] ?? 0 };
|
||||
}),
|
||||
pruneSystem: vi.fn(async (target: string) => {
|
||||
if (opts.throwOn === target) throw new Error(`mock pruneSystem threw for ${target}`);
|
||||
return { success: true, reclaimedBytes: opts.allBytes?.[target] ?? 0 };
|
||||
}),
|
||||
};
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue(fake as unknown as ReturnType<typeof DockerController.getInstance>);
|
||||
// Spy on the prototype so the mock applies to whichever FileSystemService
|
||||
// instance the route creates for the local node id, not a throwaway one.
|
||||
vi.spyOn(FileSystemService.prototype, 'getStacks').mockResolvedValue(['stack-a', 'stack-b']);
|
||||
return fake;
|
||||
}
|
||||
|
||||
describe('POST /api/fleet/labels/fleet-prune', () => {
|
||||
it('returns 401 without auth', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.send({ targets: ['images'], scope: 'managed' });
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('returns 403 PAID_REQUIRED on community tier', async () => {
|
||||
mockTier('community');
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images'], scope: 'managed' });
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body.code).toBe('PAID_REQUIRED');
|
||||
});
|
||||
|
||||
it('returns 400 when body is missing', async () => {
|
||||
mockTier('paid');
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send();
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('returns 400 when targets is empty', async () => {
|
||||
mockTier('paid');
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: [], scope: 'managed' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/non-empty/);
|
||||
});
|
||||
|
||||
it('returns 400 when a target is unrecognized', async () => {
|
||||
mockTier('paid');
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'containers'], scope: 'managed' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toMatch(/Invalid target/);
|
||||
});
|
||||
|
||||
it('runs pruneManagedOnly per target on the local node and returns aggregated bytes', async () => {
|
||||
mockTier('paid');
|
||||
const fake = mockLocalPrune({ managedBytes: { images: 1500, volumes: 320 } });
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'volumes'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.results).toHaveLength(1);
|
||||
const node = res.body.results[0];
|
||||
expect(node.reachable).toBe(true);
|
||||
expect(node.targets).toEqual([
|
||||
{ target: 'images', success: true, reclaimedBytes: 1500 },
|
||||
{ target: 'volumes', success: true, reclaimedBytes: 320 },
|
||||
]);
|
||||
expect(fake.pruneManagedOnly).toHaveBeenCalledTimes(2);
|
||||
expect(fake.pruneSystem).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('runs pruneSystem when scope is "all" and dedupes targets', async () => {
|
||||
mockTier('paid');
|
||||
const fake = mockLocalPrune({ allBytes: { networks: 0, images: 2048 } });
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'networks', 'images'], scope: 'all' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(fake.pruneManagedOnly).not.toHaveBeenCalled();
|
||||
expect(fake.pruneSystem).toHaveBeenCalledTimes(2);
|
||||
const node = res.body.results[0];
|
||||
expect(node.targets.map((t: { target: string }) => t.target).sort()).toEqual(['images', 'networks']);
|
||||
});
|
||||
|
||||
it('records per-target failure when DockerController throws but continues remaining targets', async () => {
|
||||
mockTier('paid');
|
||||
mockLocalPrune({ managedBytes: { images: 100 }, throwOn: 'volumes' });
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'volumes'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
const node = res.body.results[0];
|
||||
expect(node.targets.find((t: { target: string }) => t.target === 'images').success).toBe(true);
|
||||
const volumes = node.targets.find((t: { target: string }) => t.target === 'volumes');
|
||||
expect(volumes.success).toBe(false);
|
||||
expect(volumes.reclaimedBytes).toBe(0);
|
||||
expect(volumes.error).toMatch(/pruneManagedOnly threw/);
|
||||
});
|
||||
|
||||
it('reports lock contention when bulk-prune lock is already held', async () => {
|
||||
mockTier('paid');
|
||||
mockLocalPrune();
|
||||
const db = DatabaseService.getInstance();
|
||||
const localId = db.getNodes().find(n => n.type === 'local')!.id;
|
||||
activeBulkActions.add(`bulk-prune:${localId}`);
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
const node = res.body.results.find((n: { nodeId: number }) => n.nodeId === localId);
|
||||
expect(node.targets[0].success).toBe(false);
|
||||
expect(node.targets[0].error).toMatch(/already running/);
|
||||
});
|
||||
|
||||
it('marks a remote node unreachable when fetch throws and short-circuits later targets', async () => {
|
||||
mockTier('paid');
|
||||
mockLocalPrune();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteId = db.addNode({
|
||||
name: 'remote-test',
|
||||
type: 'remote',
|
||||
api_url: 'http://remote.example:1852',
|
||||
api_token: 'tok',
|
||||
compose_dir: '/app/compose',
|
||||
is_default: false,
|
||||
});
|
||||
try {
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('connect ECONNREFUSED'));
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'volumes', 'networks'], scope: 'managed' });
|
||||
expect(res.status).toBe(200);
|
||||
const remote = res.body.results.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remote.reachable).toBe(false);
|
||||
expect(remote.error).toMatch(/ECONNREFUSED/);
|
||||
expect(remote.targets).toHaveLength(3);
|
||||
for (const t of remote.targets) expect(t.success).toBe(false);
|
||||
// Only the first target attempts the fetch; the rest short-circuit.
|
||||
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('parses remote node responses into per-target reclaimed bytes', async () => {
|
||||
mockTier('paid');
|
||||
mockLocalPrune();
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteId = db.addNode({
|
||||
name: 'remote-ok',
|
||||
type: 'remote',
|
||||
api_url: 'http://remote-ok.example:1852/',
|
||||
api_token: 'tok',
|
||||
compose_dir: '/app/compose',
|
||||
is_default: false,
|
||||
});
|
||||
try {
|
||||
const responses = new Map<string, number>([['images', 4096], ['volumes', 512]]);
|
||||
vi.spyOn(globalThis, 'fetch').mockImplementation(async (_url, init) => {
|
||||
const body = JSON.parse((init?.body as string) ?? '{}') as { target: string };
|
||||
const reclaimedBytes = responses.get(body.target) ?? 0;
|
||||
return new Response(JSON.stringify({ message: 'ok', success: true, reclaimedBytes }), {
|
||||
status: 200, headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
});
|
||||
const res = await request(app)
|
||||
.post('/api/fleet/labels/fleet-prune')
|
||||
.set('Authorization', authHeader)
|
||||
.send({ targets: ['images', 'volumes'], scope: 'all' });
|
||||
expect(res.status).toBe(200);
|
||||
const remote = res.body.results.find((n: { nodeId: number }) => n.nodeId === remoteId);
|
||||
expect(remote.reachable).toBe(true);
|
||||
expect(remote.targets).toEqual([
|
||||
{ target: 'images', success: true, reclaimedBytes: 4096 },
|
||||
{ target: 'volumes', success: true, reclaimedBytes: 512 },
|
||||
]);
|
||||
} finally {
|
||||
db.deleteNode(remoteId);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1124,6 +1124,142 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
|
||||
}
|
||||
});
|
||||
|
||||
// Fleet-wide Docker prune. Fans out to every node, running per-target prune
|
||||
// (images/volumes/networks) under the chosen scope. Local nodes call
|
||||
// DockerController directly under a per-node bulk-prune lock; remote nodes
|
||||
// receive one POST /api/system/prune/system per target via the standard
|
||||
// Bearer-token path. Concurrent execution against the per-node prune route in
|
||||
// systemMaintenance.ts is safe because Docker's prune API is internally
|
||||
// serialized and idempotent (the worst case is a duplicate call returning 0
|
||||
// reclaimed bytes).
|
||||
// Tier: requirePaid + requireAdmin.
|
||||
const FLEET_PRUNE_TARGETS = ['images', 'volumes', 'networks'] as const;
|
||||
type FleetPruneTarget = (typeof FLEET_PRUNE_TARGETS)[number];
|
||||
|
||||
fleetRouter.post('/labels/fleet-prune', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
if (!requirePaid(req, res)) return;
|
||||
if (!requireAdmin(req, res)) return;
|
||||
|
||||
const body = req.body as { targets?: unknown; scope?: unknown } | undefined;
|
||||
if (!body || typeof body !== 'object') {
|
||||
res.status(400).json({ error: 'Request body is required' });
|
||||
return;
|
||||
}
|
||||
const rawTargets = Array.isArray(body.targets) ? body.targets : null;
|
||||
if (!rawTargets || rawTargets.length === 0) {
|
||||
res.status(400).json({ error: 'targets must be a non-empty array' });
|
||||
return;
|
||||
}
|
||||
const dedup = new Set<FleetPruneTarget>();
|
||||
for (const t of rawTargets) {
|
||||
if (typeof t !== 'string' || !(FLEET_PRUNE_TARGETS as readonly string[]).includes(t)) {
|
||||
res.status(400).json({ error: `Invalid target: ${typeof t === 'string' ? t : typeof t}` });
|
||||
return;
|
||||
}
|
||||
dedup.add(t as FleetPruneTarget);
|
||||
}
|
||||
const targets: FleetPruneTarget[] = Array.from(dedup);
|
||||
const scope: 'managed' | 'all' = body.scope === 'all' ? 'all' : 'managed';
|
||||
|
||||
type TargetResult = { target: FleetPruneTarget; success: boolean; reclaimedBytes: number; error?: string };
|
||||
type NodeResult = {
|
||||
nodeId: number; nodeName: string; reachable: boolean; error?: string; targets: TargetResult[];
|
||||
};
|
||||
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const nodes = db.getNodes();
|
||||
|
||||
const results: NodeResult[] = await Promise.all(nodes.map(async (node): Promise<NodeResult> => {
|
||||
if (node.type === 'local') {
|
||||
const lockKey = `bulk-prune:${node.id}`;
|
||||
if (activeBulkActions.has(lockKey)) {
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, reachable: true,
|
||||
targets: targets.map(t => ({ target: t, success: false, reclaimedBytes: 0, error: 'A prune is already running on this node' })),
|
||||
};
|
||||
}
|
||||
activeBulkActions.add(lockKey);
|
||||
try {
|
||||
const knownStacks = scope === 'managed' ? await FileSystemService.getInstance(node.id).getStacks() : [];
|
||||
const dockerController = DockerController.getInstance(node.id);
|
||||
const targetResults: TargetResult[] = [];
|
||||
let anySuccess = false;
|
||||
for (const target of targets) {
|
||||
try {
|
||||
const result = scope === 'managed'
|
||||
? await dockerController.pruneManagedOnly(target, knownStacks)
|
||||
: await dockerController.pruneSystem(target);
|
||||
targetResults.push({ target, success: true, reclaimedBytes: result.reclaimedBytes });
|
||||
if (result.reclaimedBytes > 0 || result.success) anySuccess = true;
|
||||
} catch (err) {
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error: getErrorMessage(err, 'Prune failed') });
|
||||
}
|
||||
}
|
||||
if (anySuccess) invalidateNodeCaches(node.id);
|
||||
return { nodeId: node.id, nodeName: node.name, reachable: true, targets: targetResults };
|
||||
} finally {
|
||||
activeBulkActions.delete(lockKey);
|
||||
}
|
||||
}
|
||||
|
||||
// Remote node: POST /api/system/prune/system per target, short-circuiting
|
||||
// on the first transport-level failure so we don't hammer a dead node.
|
||||
if (!node.api_url || !node.api_token) {
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name, reachable: false, error: 'Remote node not configured',
|
||||
targets: targets.map(t => ({ target: t, success: false, reclaimedBytes: 0, error: 'Remote node not configured' })),
|
||||
};
|
||||
}
|
||||
const baseUrl = node.api_url.replace(/\/$/, '');
|
||||
const targetResults: TargetResult[] = [];
|
||||
let nodeUnreachable: string | null = null;
|
||||
for (const target of targets) {
|
||||
if (nodeUnreachable) {
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error: nodeUnreachable });
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/api/system/prune/system`, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: `Bearer ${node.api_token}`, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ target, scope }),
|
||||
signal: AbortSignal.timeout(120000),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errBody = (await response.json().catch(() => ({}))) as { error?: string };
|
||||
const message = errBody.error || `Remote returned ${response.status}`;
|
||||
nodeUnreachable = message;
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error: message });
|
||||
continue;
|
||||
}
|
||||
const remote = (await response.json().catch(() => null)) as { success?: boolean; reclaimedBytes?: number } | null;
|
||||
if (!remote || typeof remote.reclaimedBytes !== 'number') {
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error: 'Invalid response from remote node' });
|
||||
continue;
|
||||
}
|
||||
targetResults.push({ target, success: remote.success !== false, reclaimedBytes: remote.reclaimedBytes });
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err, 'Failed to reach remote node');
|
||||
nodeUnreachable = message;
|
||||
targetResults.push({ target, success: false, reclaimedBytes: 0, error: message });
|
||||
}
|
||||
}
|
||||
return {
|
||||
nodeId: node.id, nodeName: node.name,
|
||||
reachable: nodeUnreachable === null,
|
||||
error: nodeUnreachable ?? undefined,
|
||||
targets: targetResults,
|
||||
};
|
||||
}));
|
||||
|
||||
res.json({ results });
|
||||
} catch (error) {
|
||||
console.error('[Fleet] fleet-prune error:', error);
|
||||
res.status(500).json({ error: getErrorMessage(error, 'Failed to run fleet prune') });
|
||||
}
|
||||
});
|
||||
|
||||
// ─── Fleet Snapshots (manual: Community; scheduled: Skipper+) ───
|
||||
|
||||
fleetRouter.post('/snapshots', authMiddleware, async (req: Request, res: Response): Promise<void> => {
|
||||
|
||||
Reference in New Issue
Block a user