mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-12 11:47:11 +00:00
a51547a158
* fix(monitor): decouple janitor disk-usage check from 30s cycle (F-6) `docker system df` (called by the MonitorService janitor check) can take 30+ seconds on Docker Desktop with many volumes. Running it on the 30s evaluate cycle compounded with the per-container stats fan-out and pushed the cycle to 140s+, blocking subsequent monitoring work. This change: - Moves the janitor disk-usage check into its own 15-minute cycle with a tight 8s timeout. A circuit breaker opens after 3 consecutive timeouts (60-minute cooldown) so a sick daemon stops pinning Dockerode sockets every tick. The first janitor tick is deferred 45 seconds past boot to avoid head-of-line collision with the initial monitor cycle's stats fan-out. - Adds a paired 8s `withTimeout` wrap to the admin prune-estimate routes (`/api/system/prune/estimate` and the dry-run path of `/api/system/prune/system`) so a slow df does not hang the admin tab. Both routes respond 503 with code `docker_df_slow` on timeout. - Factors `withTimeout` and `TimeoutError` into `utils/withTimeout.ts` so the route layer does not have to import from a service module. - Adds 10 unit tests covering the decoupling guardrail, breaker open/close, cooldown, threshold gate, the 100 MB reclaimable floor, re-entrancy, recovery logging, non-timeout error handling, and the full timer-cleanup contract of `stop()`. - Adds 4 integration tests for the prune routes covering the 503 timeout response, the success path, and the non-timeout 5xx path. * fix(fleet,monitor): extend F-6 timeout to fleet prune routes; close breaker-recovery log gap Codex audit findings on PR #1164: Major. The fleet routes that fan out prune-estimate work on local nodes (`POST /api/fleet/labels/fleet-prune` dry-run path and `POST /api/fleet/prune/estimate`) called `estimateSystemReclaim` without a timeout, so a slow local Docker daemon could still hang the fleet admin tab even though the system-maintenance routes were already bounded. Wrap both call sites with the shared `withTimeout(..., 8s)` and surface a "Docker daemon is busy" message via the per-target and per-node error channels the routes already used for other failures. The destructive (non-dry-run) prune path stays unwrapped because it calls `pruneSystem` / `pruneManagedOnly`, not `df`. Minor. The janitor circuit breaker zeroed `janitorConsecutiveTimeouts` when it opened, so a successful call after a full breaker-open cooldown slipped past the `if (counter > 0)` recovery-log branch and never emitted `[Monitor] Janitor disk-usage check recovered`. The operator observability signal was missing exactly when it mattered most. Extend the predicate to also trip on `janitorBreakerUntil > 0` (which stays set to its past timestamp after cooldown until the next success clears it), so recovery logs symmetrically for both partial-failure and post-breaker recovery paths. Added a dedicated test. Three new integration tests cover the fleet routes (timeout, success, and the estimate endpoint's per-node unreachable shape).
406 lines
18 KiB
TypeScript
406 lines
18 KiB
TypeScript
import { Router, type Request, type Response } from 'express';
|
|
import DockerController, { type CreateNetworkOptions, type NetworkDriver } from '../services/DockerController';
|
|
import { FileSystemService } from '../services/FileSystemService';
|
|
import SelfIdentityService from '../services/SelfIdentityService';
|
|
import { requireAdmin } from '../middleware/tierGates';
|
|
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
|
|
import { isValidDockerResourceId, isValidCidr, isValidIPv4 } from '../utils/validation';
|
|
import { isDebugEnabled } from '../utils/debug';
|
|
import { getErrorMessage } from '../utils/errors';
|
|
import { sanitizeForLog } from '../utils/safeLog';
|
|
import { withTimeout, TimeoutError } from '../utils/withTimeout';
|
|
|
|
// `docker system df` (the call backing estimateSystemReclaim) can take 30+
|
|
// seconds on Docker Desktop with many volumes; 8s matches the MonitorService
|
|
// janitor timeout so the daemon never has more than ~16s of concurrent
|
|
// pressure from Sencho's own paths even when prune and janitor collide.
|
|
const PRUNE_ESTIMATE_TIMEOUT_MS = 8_000;
|
|
|
|
function respondDfSlow(res: Response): Response {
|
|
return res.status(503).json({
|
|
error: 'Docker daemon is busy. Please try again in a moment.',
|
|
code: 'docker_df_slow',
|
|
});
|
|
}
|
|
|
|
export const systemMaintenanceRouter = Router();
|
|
|
|
// 423 Locked is sent when the operator targets the running Sencho container's
|
|
// own image / volume / network. The frontend surfaces the `error` string as a
|
|
// toast; `kind` is for diagnostics.
|
|
function rejectIfSelf(kind: 'image' | 'volume' | 'network', id: string, res: Response): boolean {
|
|
const self = SelfIdentityService.getInstance();
|
|
const matched =
|
|
(kind === 'image' && self.isOwnImage(id)) ||
|
|
(kind === 'volume' && self.isOwnVolume(id)) ||
|
|
(kind === 'network' && self.isOwnNetwork(id));
|
|
if (!matched) return false;
|
|
res.status(423).json({
|
|
error: 'Cannot delete the running Sencho instance',
|
|
kind,
|
|
id,
|
|
});
|
|
return true;
|
|
}
|
|
|
|
systemMaintenanceRouter.get('/orphans', async (req: Request, res: Response) => {
|
|
try {
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
const orphans = await dockerController.getOrphanContainers(knownStacks);
|
|
res.json(orphans);
|
|
} catch (error) {
|
|
console.error('Failed to fetch orphan containers:', error);
|
|
res.status(500).json({ error: 'Failed to fetch orphan containers' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.post('/prune/orphans', async (req: Request, res: Response) => {
|
|
if (!requireAdmin(req, res)) return;
|
|
try {
|
|
const { containerIds } = req.body;
|
|
if (!Array.isArray(containerIds)) {
|
|
return res.status(400).json({ error: 'containerIds must be an array' });
|
|
}
|
|
const invalidIds = containerIds.filter((id: unknown) => typeof id !== 'string' || !isValidDockerResourceId(id));
|
|
if (invalidIds.length > 0) {
|
|
return res.status(400).json({ error: 'One or more container IDs have an invalid format' });
|
|
}
|
|
// Silently drop the running Sencho container if a stale client somehow
|
|
// includes it in the prune set. The Unmanaged tab already filters self
|
|
// out, so this is a belt-and-braces guard.
|
|
const self = SelfIdentityService.getInstance();
|
|
const skippedSelf = (containerIds as string[]).some((id) => self.isOwnContainer(id));
|
|
const safeIds: string[] = (containerIds as string[]).filter((id) => !self.isOwnContainer(id));
|
|
console.log(`[Resources] Prune orphans: ${sanitizeForLog(safeIds.length)} container(s) requested${skippedSelf ? ' (self skipped)' : ''}`);
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
const results = await dockerController.removeContainers(safeIds);
|
|
const succeeded = results.filter((r: { success: boolean }) => r.success).length;
|
|
console.log(`[Resources] Prune orphans completed: ${succeeded}/${sanitizeForLog(safeIds.length)} removed`);
|
|
invalidateNodeCaches(req.nodeId);
|
|
res.json(skippedSelf ? { results, skipped: 'self' } : { results });
|
|
} catch (error) {
|
|
console.error('Failed to prune orphan containers:', error);
|
|
res.status(500).json({ error: 'Failed to prune orphan containers' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.post('/prune/system', async (req: Request, res: Response) => {
|
|
if (!requireAdmin(req, res)) return;
|
|
try {
|
|
const { target, scope, dryRun } = req.body as { target: string; scope?: string; dryRun?: boolean };
|
|
if (!['containers', 'images', 'networks', 'volumes'].includes(target)) {
|
|
return res.status(400).json({ error: 'Invalid prune target' });
|
|
}
|
|
|
|
const pruneScope = scope === 'managed' ? 'managed' : 'all';
|
|
const isDryRun = dryRun === true;
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
|
|
if (isDryRun) {
|
|
// Rehearse the destructive path: same scope resolution, same Docker
|
|
// enumeration, no remove calls. Containers have no managed estimate
|
|
// helper because pruneManagedOnly does not handle them.
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
let estimate: { reclaimableBytes: number };
|
|
if (pruneScope === 'managed' && target !== 'containers') {
|
|
estimate = await dockerController.estimateManagedReclaim(
|
|
target as 'images' | 'volumes' | 'networks',
|
|
knownStacks,
|
|
);
|
|
} else {
|
|
// estimateSystemReclaim calls `docker system df`; bound it so a slow
|
|
// daemon doesn't hang the admin's tab (F-6).
|
|
estimate = await withTimeout(
|
|
dockerController.estimateSystemReclaim(
|
|
target as 'containers' | 'images' | 'networks' | 'volumes',
|
|
knownStacks,
|
|
),
|
|
PRUNE_ESTIMATE_TIMEOUT_MS,
|
|
'docker disk usage',
|
|
);
|
|
}
|
|
res.json({ message: 'Dry run', success: true, dryRun: true, reclaimedBytes: estimate.reclaimableBytes });
|
|
return;
|
|
}
|
|
|
|
console.log(`[Resources] System prune: ${target} (scope: ${pruneScope})`);
|
|
let result: { success: boolean; reclaimedBytes: number };
|
|
if (pruneScope === 'managed' && target !== 'containers') {
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
result = await dockerController.pruneManagedOnly(
|
|
target as 'images' | 'volumes' | 'networks',
|
|
knownStacks
|
|
);
|
|
} else {
|
|
result = await dockerController.pruneSystem(target as 'containers' | 'images' | 'networks' | 'volumes');
|
|
}
|
|
|
|
console.log(`[Resources] System prune completed: ${target}, reclaimed ${result.reclaimedBytes} bytes`);
|
|
if (target === 'containers') {
|
|
invalidateNodeCaches(req.nodeId);
|
|
}
|
|
res.json({ message: 'Prune completed', ...result });
|
|
} catch (error: unknown) {
|
|
if (error instanceof TimeoutError) {
|
|
console.warn('System prune: docker disk usage timed out');
|
|
return respondDfSlow(res);
|
|
}
|
|
console.error('System prune error:', error);
|
|
res.status(500).json({ error: 'System prune failed' });
|
|
}
|
|
});
|
|
|
|
// Non-destructive size estimate for a prune target/scope. The Fleet Actions
|
|
// "Prune fleet-wide" card calls this on each remote node to populate its live
|
|
// blast-radius readout before the operator confirms. Reuses the same Docker
|
|
// enumeration as `/prune/system` so the estimate matches what the destructive
|
|
// path would reclaim.
|
|
systemMaintenanceRouter.post('/prune/estimate', async (req: Request, res: Response) => {
|
|
if (!requireAdmin(req, res)) return;
|
|
try {
|
|
const { target, scope } = req.body as { target: string; scope?: string };
|
|
if (!['containers', 'images', 'networks', 'volumes'].includes(target)) {
|
|
return res.status(400).json({ error: 'Invalid prune target' });
|
|
}
|
|
const pruneScope = scope === 'managed' ? 'managed' : 'all';
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
|
|
let result: { reclaimableBytes: number };
|
|
if (pruneScope === 'managed' && target !== 'containers') {
|
|
result = await dockerController.estimateManagedReclaim(
|
|
target as 'images' | 'volumes' | 'networks',
|
|
knownStacks,
|
|
);
|
|
} else {
|
|
// estimateSystemReclaim calls `docker system df`; bound it so a slow
|
|
// daemon doesn't hang the admin's tab (F-6).
|
|
result = await withTimeout(
|
|
dockerController.estimateSystemReclaim(
|
|
target as 'containers' | 'images' | 'networks' | 'volumes',
|
|
knownStacks,
|
|
),
|
|
PRUNE_ESTIMATE_TIMEOUT_MS,
|
|
'docker disk usage',
|
|
);
|
|
}
|
|
res.json({ reclaimableBytes: result.reclaimableBytes });
|
|
} catch (error: unknown) {
|
|
if (error instanceof TimeoutError) {
|
|
console.warn('Prune estimate: docker disk usage timed out');
|
|
return respondDfSlow(res);
|
|
}
|
|
console.error('Prune estimate error:', error);
|
|
res.status(500).json({ error: 'Failed to estimate reclaimable bytes' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.get('/docker-df', async (req: Request, res: Response) => {
|
|
try {
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
const df = await DockerController.getInstance(req.nodeId).getDiskUsageClassified(knownStacks);
|
|
res.json(df);
|
|
} catch (error) {
|
|
console.error('Failed to fetch docker disk usage:', error);
|
|
res.status(500).json({ error: 'Failed to fetch docker disk usage' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.get('/resources', async (req: Request, res: Response) => {
|
|
try {
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
const result = await DockerController.getInstance(req.nodeId).getClassifiedResources(knownStacks);
|
|
res.json(result);
|
|
} catch (error) {
|
|
console.error('Failed to fetch classified resources:', error);
|
|
res.status(500).json({ error: 'Failed to fetch resources' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.get('/images', async (req: Request, res: Response) => {
|
|
try {
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
const { images } = await DockerController.getInstance(req.nodeId).getClassifiedResources(knownStacks);
|
|
res.json(images);
|
|
} catch (error) {
|
|
console.error('Failed to fetch images:', error);
|
|
res.status(500).json({ error: 'Failed to fetch images' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.get('/volumes', async (req: Request, res: Response) => {
|
|
try {
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
const { volumes } = await DockerController.getInstance(req.nodeId).getClassifiedResources(knownStacks);
|
|
res.json(volumes);
|
|
} catch (error) {
|
|
console.error('Failed to fetch volumes:', error);
|
|
res.status(500).json({ error: 'Failed to fetch volumes' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.get('/networks', async (req: Request, res: Response) => {
|
|
try {
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
const { networks } = await DockerController.getInstance(req.nodeId).getClassifiedResources(knownStacks);
|
|
res.json(networks);
|
|
} catch (error) {
|
|
console.error('Failed to fetch networks:', error);
|
|
res.status(500).json({ error: 'Failed to fetch networks' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.get('/images/:id', async (req: Request, res: Response) => {
|
|
try {
|
|
const rawId = req.params.id as string;
|
|
if (!rawId) return res.status(400).json({ error: 'Invalid image ID format' });
|
|
const hexId = rawId.startsWith('sha256:') ? rawId.slice('sha256:'.length) : rawId;
|
|
if (!isValidDockerResourceId(hexId)) {
|
|
return res.status(400).json({ error: 'Invalid image ID format' });
|
|
}
|
|
const result = await DockerController.getInstance(req.nodeId).inspectImage(hexId);
|
|
res.json(result);
|
|
} catch (error: unknown) {
|
|
console.error('Failed to inspect image:', error);
|
|
const err = error as Record<string, unknown>;
|
|
const is404 = (typeof err.statusCode === 'number' && err.statusCode === 404)
|
|
|| (error instanceof Error && error.message.includes('404'));
|
|
res.status(is404 ? 404 : 500).json({ error: is404 ? 'Image not found' : 'Failed to inspect image' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.post('/images/delete', async (req: Request, res: Response) => {
|
|
if (!requireAdmin(req, res)) return;
|
|
try {
|
|
const { id } = req.body;
|
|
if (!id) return res.status(400).json({ error: 'ID is required' });
|
|
if (typeof id !== 'string') {
|
|
return res.status(400).json({ error: 'Invalid image ID format' });
|
|
}
|
|
// Docker image IDs round-trip as `sha256:<hex>` through /system/images,
|
|
// so the UI and any client that forwards the same value sees the prefixed
|
|
// form. Strip before validation, mirroring the inspect route above.
|
|
const hexId = id.startsWith('sha256:') ? id.slice('sha256:'.length) : id;
|
|
if (!isValidDockerResourceId(hexId)) {
|
|
return res.status(400).json({ error: 'Invalid image ID format' });
|
|
}
|
|
if (rejectIfSelf('image', id, res)) return;
|
|
console.log(`[Resources] Delete image: ${hexId.substring(0, 12)}`);
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
await dockerController.removeImage(id);
|
|
invalidateNodeCaches(req.nodeId);
|
|
res.json({ success: true, message: 'Image deleted' });
|
|
} catch (error: unknown) {
|
|
console.error('Failed to delete image:', error);
|
|
res.status(500).json({ error: 'Failed to delete image' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.post('/volumes/delete', async (req: Request, res: Response) => {
|
|
if (!requireAdmin(req, res)) return;
|
|
try {
|
|
const { id } = req.body;
|
|
if (!id || typeof id !== 'string') return res.status(400).json({ error: 'Volume name is required' });
|
|
if (rejectIfSelf('volume', id, res)) return;
|
|
console.log(`[Resources] Delete volume: ${sanitizeForLog(id)}`);
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
await dockerController.removeVolume(id);
|
|
invalidateNodeCaches(req.nodeId);
|
|
res.json({ success: true, message: 'Volume deleted' });
|
|
} catch (error: unknown) {
|
|
console.error('Failed to delete volume:', error);
|
|
res.status(500).json({ error: 'Failed to delete volume' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.post('/networks/delete', async (req: Request, res: Response) => {
|
|
if (!requireAdmin(req, res)) return;
|
|
try {
|
|
const { id } = req.body;
|
|
if (!id) return res.status(400).json({ error: 'ID is required' });
|
|
if (typeof id !== 'string' || !isValidDockerResourceId(id)) {
|
|
return res.status(400).json({ error: 'Invalid network ID format' });
|
|
}
|
|
if (rejectIfSelf('network', id, res)) return;
|
|
console.log(`[Resources] Delete network: ${id.substring(0, 12)}`);
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
await dockerController.removeNetwork(id);
|
|
invalidateNodeCaches(req.nodeId);
|
|
res.json({ success: true, message: 'Network deleted' });
|
|
} catch (error: unknown) {
|
|
console.error('Failed to delete network:', error);
|
|
res.status(500).json({ error: 'Failed to delete network' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.get('/networks/topology', async (req: Request, res: Response) => {
|
|
try {
|
|
const includeSystem = req.query.includeSystem === 'true';
|
|
const knownStacks = await FileSystemService.getInstance(req.nodeId).getStacks();
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
const topology = await dockerController.getTopologyData(knownStacks, includeSystem);
|
|
console.log(`[Resources] Topology fetched: ${topology.length} networks, includeSystem=${includeSystem}`);
|
|
if (isDebugEnabled()) console.debug('[Resources:debug] Topology fetched', { networkCount: topology.length, includeSystem });
|
|
res.json(topology);
|
|
} catch (error: unknown) {
|
|
console.error('Failed to fetch network topology:', error);
|
|
res.status(500).json({ error: 'Failed to fetch network topology' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.get('/networks/:id', async (req: Request, res: Response) => {
|
|
try {
|
|
const id = req.params.id as string;
|
|
if (!id) return res.status(400).json({ error: 'Network ID is required' });
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
const networkInfo = await dockerController.inspectNetwork(id);
|
|
res.json(networkInfo);
|
|
} catch (error: unknown) {
|
|
console.error('Failed to inspect network:', error);
|
|
const err = error as Record<string, unknown>;
|
|
const is404 = (typeof err.statusCode === 'number' && err.statusCode === 404)
|
|
|| (error instanceof Error && error.message.includes('404'));
|
|
res.status(is404 ? 404 : 500).json({ error: is404 ? 'Network not found' : 'Failed to inspect network' });
|
|
}
|
|
});
|
|
|
|
systemMaintenanceRouter.post('/networks', async (req: Request, res: Response) => {
|
|
if (!requireAdmin(req, res)) return;
|
|
try {
|
|
const { name, driver, subnet, gateway, labels, internal, attachable } = req.body;
|
|
if (!name) return res.status(400).json({ error: 'Network name is required' });
|
|
|
|
const options: CreateNetworkOptions = { Name: name };
|
|
|
|
const VALID_DRIVERS: NetworkDriver[] = ['bridge', 'overlay', 'macvlan', 'host', 'none'];
|
|
if (driver) {
|
|
if (!VALID_DRIVERS.includes(driver)) return res.status(400).json({ error: 'Invalid network driver' });
|
|
options.Driver = driver;
|
|
}
|
|
if (subnet || gateway) {
|
|
if (subnet && !isValidCidr(subnet)) return res.status(400).json({ error: 'Invalid subnet CIDR notation (e.g. 172.20.0.0/16)' });
|
|
if (gateway && !isValidIPv4(gateway)) return res.status(400).json({ error: 'Invalid gateway IP address (e.g. 172.20.0.1)' });
|
|
options.IPAM = { Config: [{}] };
|
|
if (subnet) options.IPAM.Config[0].Subnet = subnet;
|
|
if (gateway) options.IPAM.Config[0].Gateway = gateway;
|
|
}
|
|
if (labels && typeof labels === 'object' && !Array.isArray(labels)) options.Labels = labels;
|
|
if (internal) options.Internal = true;
|
|
if (attachable) options.Attachable = true;
|
|
|
|
const dockerController = DockerController.getInstance(req.nodeId);
|
|
const network = await dockerController.createNetwork(options);
|
|
console.log(`[Resources] Network created: ${sanitizeForLog(name)}`);
|
|
invalidateNodeCaches(req.nodeId);
|
|
res.status(201).json({ success: true, message: 'Network created', id: network.id });
|
|
} catch (error: unknown) {
|
|
console.error('Failed to create network:', error);
|
|
const msg = getErrorMessage(error, '');
|
|
const safePatterns = ['already exists', 'name is invalid', 'invalid network name'];
|
|
const lowerMsg = msg.toLowerCase();
|
|
const isSafe = safePatterns.some(p => lowerMsg.includes(p));
|
|
res.status(isSafe ? 409 : 500).json({ error: isSafe ? msg : 'Failed to create network' });
|
|
}
|
|
});
|