mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-05 16:37:46 +00:00
Anso
9dbce9c3c7
Operators previously saw "spawn docker ENOENT" or "spawn /bin/sh ENOENT" when the host was under memory pressure, which sent them down a missing-binary debugging path. Linux libuv's posix_spawn can fail to allocate its argv / path-search arena under low free memory and surface the underlying ENOMEM as ENOENT. Centralizes spawn-error mapping in a new utils/spawnErrors.ts helper: - Explicit ENOMEM is rewritten to "Out of memory while launching <command> (host free memory: X MiB of Y MiB)". - ENOENT under the 128 MiB free-memory floor is rewritten with the same wording plus a "reported as ENOENT under memory pressure" hint. - ENOENT for docker on a healthy host preserves the existing "Docker CLI unavailable on this node" mapping. - Other errors pass through unchanged. Applied at the four named offenders: ComposeService.execute(), ComposeService.captureCompose(), DockerController.getContainersByStack(), and FileSystemService.getStacks() (which gets an ENOMEM-aware log line for the scandir failure). Startup also logs host free/total MiB once and warns when free memory is below the 128 MiB floor, so the diagnostic surfaces before the first spawn attempt rather than after it fails. 37 tests cover the mapping function directly and the ComposeService / FileSystemService integration paths.
1376 lines
50 KiB
TypeScript
1376 lines
50 KiB
TypeScript
import Docker from 'dockerode';
|
|
import WebSocket from 'ws';
|
|
import { exec } from 'child_process';
|
|
import { promisify } from 'util';
|
|
import path from 'path';
|
|
import fs from 'fs/promises';
|
|
import * as yaml from 'yaml';
|
|
|
|
import { NodeRegistry } from './NodeRegistry';
|
|
import { CacheService } from './CacheService';
|
|
import { isPathWithinBase } from '../utils/validation';
|
|
import { isDebugEnabled } from '../utils/debug';
|
|
import { sanitizeForLog } from '../utils/safeLog';
|
|
import { describeSpawnError } from '../utils/spawnErrors';
|
|
|
|
const execAsync = promisify(exec);
|
|
const COMPOSE_DIR = process.env.COMPOSE_DIR || '/app/compose';
|
|
|
|
/** Canonical compose file name variants, checked in priority order. */
|
|
const COMPOSE_FILE_NAMES = ['compose.yaml', 'compose.yml', 'docker-compose.yaml', 'docker-compose.yml'] as const;
|
|
|
|
/** Cached mapping from compose `name:` field to stack directory name. TTL-based to avoid re-parsing YAML on every poll. */
|
|
const PROJECT_NAME_CACHE_TTL_MS = 60_000;
|
|
const PROJECT_NAME_CACHE_KEY = 'project-name-map';
|
|
|
|
/** Common web-UI private ports, checked in priority order when detecting the main app port. */
|
|
const WEB_UI_PORTS = [32400, 8989, 7878, 9696, 5055, 8080, 80, 443, 3000, 9000];
|
|
/** Ports that should never be treated as the main app port. */
|
|
const IGNORE_PORTS = [1900, 53, 22];
|
|
|
|
export interface BulkStackInfo {
|
|
status: 'running' | 'exited' | 'unknown';
|
|
mainPort?: number;
|
|
/** Unix seconds of the oldest running container (approximates stack uptime). */
|
|
runningSince?: number;
|
|
}
|
|
|
|
export interface ClassifiedImage {
|
|
Id: string;
|
|
RepoTags: string[];
|
|
Size: number;
|
|
Containers: number;
|
|
managedBy: string | null;
|
|
managedStatus: 'managed' | 'unmanaged' | 'unused';
|
|
}
|
|
|
|
export interface PortInUseInfo {
|
|
stack: string | null;
|
|
container: string;
|
|
}
|
|
|
|
export interface ClassifiedVolume {
|
|
Name: string;
|
|
Driver: string;
|
|
Mountpoint: string;
|
|
Size: number;
|
|
CreatedAt: string | null;
|
|
managedBy: string | null;
|
|
managedStatus: 'managed' | 'unmanaged';
|
|
}
|
|
|
|
export interface ClassifiedNetwork {
|
|
Id: string;
|
|
Name: string;
|
|
Driver: string;
|
|
Scope: string;
|
|
managedBy: string | null;
|
|
managedStatus: 'managed' | 'unmanaged' | 'system';
|
|
}
|
|
|
|
export interface TopologyContainer {
|
|
id: string;
|
|
name: string;
|
|
ip: string;
|
|
state: string;
|
|
image: string;
|
|
stack: string | null;
|
|
}
|
|
|
|
export interface TopologyNetwork {
|
|
Id: string;
|
|
Name: string;
|
|
Driver: string;
|
|
Scope: string;
|
|
managedBy: string | null;
|
|
managedStatus: 'managed' | 'unmanaged' | 'system';
|
|
containers: TopologyContainer[];
|
|
}
|
|
|
|
export type NetworkDriver = 'bridge' | 'overlay' | 'macvlan' | 'host' | 'none';
|
|
|
|
export interface CreateNetworkOptions {
|
|
Name: string;
|
|
Driver?: NetworkDriver;
|
|
IPAM?: { Config: Array<{ Subnet?: string; Gateway?: string }> };
|
|
Labels?: Record<string, string>;
|
|
Internal?: boolean;
|
|
Attachable?: boolean;
|
|
}
|
|
|
|
class DockerController {
|
|
private static readonly SYSTEM_NETWORKS = new Set(['bridge', 'host', 'none']);
|
|
private docker: Docker;
|
|
private nodeId: number;
|
|
|
|
private constructor(nodeId: number) {
|
|
this.nodeId = nodeId;
|
|
this.docker = NodeRegistry.getInstance().getDocker(nodeId);
|
|
}
|
|
|
|
public static getInstance(nodeId?: number): DockerController {
|
|
const id = nodeId ?? NodeRegistry.getInstance().getDefaultNodeId();
|
|
return new DockerController(id);
|
|
}
|
|
|
|
public getDocker(): Docker {
|
|
return this.docker;
|
|
}
|
|
|
|
private validateApiData<T>(data: any): T {
|
|
// If the daemon port points to a web server (like Sencho UI), Dockerode receives HTML
|
|
if (typeof data === 'string') {
|
|
throw new Error("Invalid response from Docker API. Did you provide a web port instead of the Docker daemon port?");
|
|
}
|
|
return data as T;
|
|
}
|
|
|
|
public async getDiskUsage() {
|
|
const df = await this.docker.df();
|
|
|
|
const reclaimableContainers = (items: any[]) => {
|
|
if (!items || !Array.isArray(items)) return { bytes: 0, count: 0 };
|
|
const reclaimable = items.filter(i => i.State !== 'running');
|
|
const bytes = reclaimable.reduce((acc, item) => {
|
|
let size = item.SizeRw || item.SizeRootFs || 0;
|
|
if (item.UsageData && typeof item.UsageData.Size === 'number') {
|
|
size = item.UsageData.Size;
|
|
}
|
|
return acc + size;
|
|
}, 0);
|
|
return { bytes, count: reclaimable.length };
|
|
};
|
|
|
|
const reclaimableImages = (items: any[]) => {
|
|
if (!items || !Array.isArray(items)) return { bytes: 0, count: 0 };
|
|
const reclaimable = items.filter(i => i.Containers === 0);
|
|
const bytes = reclaimable.reduce((acc, item) => {
|
|
let size = item.VirtualSize || item.Size || item.SharedSize || 0;
|
|
if (item.UsageData && typeof item.UsageData.Size === 'number') {
|
|
size = item.UsageData.Size;
|
|
}
|
|
return acc + size;
|
|
}, 0);
|
|
return { bytes, count: reclaimable.length };
|
|
};
|
|
|
|
const reclaimableVolumes = (items: any[]) => {
|
|
if (!items || !Array.isArray(items)) return { bytes: 0, count: 0 };
|
|
const reclaimable = items.filter(i => i.UsageData?.RefCount === 0);
|
|
const bytes = reclaimable.reduce((acc, item) => {
|
|
const size = item.UsageData?.Size || 0;
|
|
return acc + size;
|
|
}, 0);
|
|
return { bytes, count: reclaimable.length };
|
|
};
|
|
|
|
const reclaimableBuildCache = (items: any[]) => {
|
|
if (!items || !Array.isArray(items)) return { bytes: 0, count: 0 };
|
|
const reclaimable = items.filter(i => i.InUse === false);
|
|
const bytes = reclaimable.reduce((acc, item) => acc + (item.Size || 0), 0);
|
|
return { bytes, count: reclaimable.length };
|
|
};
|
|
|
|
const images = df.Images ? reclaimableImages(df.Images) : { bytes: 0, count: 0 };
|
|
const containers = df.Containers ? reclaimableContainers(df.Containers) : { bytes: 0, count: 0 };
|
|
const volumes = df.Volumes ? reclaimableVolumes(df.Volumes) : { bytes: 0, count: 0 };
|
|
const buildCache = df.BuildCache ? reclaimableBuildCache(df.BuildCache) : { bytes: 0, count: 0 };
|
|
|
|
return {
|
|
reclaimableImages: images.bytes,
|
|
reclaimableContainers: containers.bytes,
|
|
reclaimableVolumes: volumes.bytes,
|
|
reclaimableBuildCache: buildCache.bytes,
|
|
reclaimableImageCount: images.count,
|
|
reclaimableContainerCount: containers.count,
|
|
reclaimableVolumeCount: volumes.count,
|
|
reclaimableBuildCacheCount: buildCache.count,
|
|
};
|
|
}
|
|
|
|
public async pruneSystem(target: 'containers' | 'images' | 'networks' | 'volumes', labelFilter?: string) {
|
|
let spaceReclaimed = 0;
|
|
if (target === 'containers') {
|
|
const filters: Record<string, string[]> = {};
|
|
if (labelFilter) filters.label = [labelFilter];
|
|
const r = await this.docker.pruneContainers({ filters });
|
|
spaceReclaimed = r.SpaceReclaimed || 0;
|
|
} else if (target === 'images') {
|
|
// Remove all unused images, not just dangling ones
|
|
const filters: Record<string, string[] | Record<string, boolean>> = { dangling: { 'false': true } };
|
|
if (labelFilter) filters.label = [labelFilter];
|
|
const r = await this.docker.pruneImages({ filters });
|
|
spaceReclaimed = r.SpaceReclaimed || 0;
|
|
} else if (target === 'networks') {
|
|
const filters: Record<string, string[]> = {};
|
|
if (labelFilter) filters.label = [labelFilter];
|
|
const r = await this.docker.pruneNetworks({ filters });
|
|
spaceReclaimed = (r as { SpaceReclaimed?: number }).SpaceReclaimed || 0;
|
|
} else if (target === 'volumes') {
|
|
const filters: Record<string, string[]> = { all: ['true'] };
|
|
if (labelFilter) filters.label = [labelFilter];
|
|
const r = await this.docker.pruneVolumes({ filters });
|
|
spaceReclaimed = r.SpaceReclaimed || 0;
|
|
}
|
|
|
|
return {
|
|
success: true,
|
|
reclaimedBytes: spaceReclaimed
|
|
};
|
|
}
|
|
|
|
public async getImages() {
|
|
const data = await this.docker.listImages({ all: false });
|
|
return this.validateApiData<any[]>(data);
|
|
}
|
|
|
|
public async getVolumes() {
|
|
const data = await this.docker.listVolumes();
|
|
const validated = this.validateApiData<any>(data);
|
|
return validated.Volumes || [];
|
|
}
|
|
|
|
public async getNetworks() {
|
|
const data = await this.docker.listNetworks();
|
|
return this.validateApiData<any[]>(data);
|
|
}
|
|
|
|
public async getClassifiedResources(knownStackNames: string[]): Promise<{
|
|
images: ClassifiedImage[];
|
|
volumes: ClassifiedVolume[];
|
|
networks: ClassifiedNetwork[];
|
|
}> {
|
|
const debug = isDebugEnabled();
|
|
const t0 = debug ? Date.now() : 0;
|
|
const knownSet = new Set(knownStackNames);
|
|
|
|
const [rawImages, rawVolumeData, rawNetworks, allContainers, projectToStack] = await Promise.all([
|
|
this.docker.listImages({ all: false }),
|
|
this.docker.listVolumes(),
|
|
this.docker.listNetworks(),
|
|
this.docker.listContainers({ all: true }),
|
|
DockerController.resolveProjectNameMap(knownStackNames),
|
|
]);
|
|
|
|
const rawVolumes: any[] = (this.validateApiData<any>(rawVolumeData)).Volumes || [];
|
|
|
|
// Build fallback lookup structures for container-to-stack resolution
|
|
const absDirToStack = DockerController.buildAbsDirMap(knownStackNames);
|
|
const resolvedBase = path.resolve(COMPOSE_DIR);
|
|
|
|
// Build imageId → stack mapping using the full fallback resolution chain
|
|
const imageToStack = new Map<string, string>();
|
|
for (const c of allContainers as any[]) {
|
|
if (!c.ImageID) continue;
|
|
const stack = DockerController.resolveContainerStack(
|
|
c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
|
);
|
|
if (stack) imageToStack.set(c.ImageID, stack);
|
|
}
|
|
|
|
const images: ClassifiedImage[] = this.validateApiData<any[]>(rawImages).map((img: any) => {
|
|
const stack = imageToStack.get(img.Id) ?? null;
|
|
const managedStatus: ClassifiedImage['managedStatus'] =
|
|
img.Containers === 0 ? 'unused' :
|
|
stack ? 'managed' : 'unmanaged';
|
|
return {
|
|
Id: img.Id,
|
|
RepoTags: img.RepoTags ?? [],
|
|
Size: img.Size ?? 0,
|
|
Containers: img.Containers ?? 0,
|
|
managedBy: stack,
|
|
managedStatus,
|
|
};
|
|
});
|
|
|
|
const volumes: ClassifiedVolume[] = rawVolumes.map((vol: any) => {
|
|
const stack = DockerController.resolveProjectLabel(vol.Labels?.['com.docker.compose.project'], knownSet, projectToStack);
|
|
const managedStatus: ClassifiedVolume['managedStatus'] = stack ? 'managed' : 'unmanaged';
|
|
return {
|
|
Name: vol.Name,
|
|
Driver: vol.Driver,
|
|
Mountpoint: vol.Mountpoint,
|
|
Size: vol.UsageData?.Size ?? 0,
|
|
CreatedAt: vol.CreatedAt ?? null,
|
|
managedBy: stack,
|
|
managedStatus,
|
|
};
|
|
});
|
|
|
|
const networks: ClassifiedNetwork[] = this.validateApiData<any[]>(rawNetworks).map((net: any) => {
|
|
if (DockerController.SYSTEM_NETWORKS.has(net.Name)) {
|
|
return { Id: net.Id, Name: net.Name, Driver: net.Driver, Scope: net.Scope, managedBy: null, managedStatus: 'system' as const };
|
|
}
|
|
const stack = DockerController.resolveProjectLabel(net.Labels?.['com.docker.compose.project'], knownSet, projectToStack);
|
|
const managedStatus: ClassifiedNetwork['managedStatus'] = stack ? 'managed' : 'unmanaged';
|
|
return {
|
|
Id: net.Id,
|
|
Name: net.Name,
|
|
Driver: net.Driver,
|
|
Scope: net.Scope,
|
|
managedBy: stack,
|
|
managedStatus,
|
|
};
|
|
});
|
|
|
|
if (debug) console.debug('[Resources:debug] Classification completed', {
|
|
ms: Date.now() - t0, images: images.length, volumes: volumes.length, networks: networks.length,
|
|
});
|
|
|
|
return { images, volumes, networks };
|
|
}
|
|
|
|
public async pruneManagedOnly(
|
|
target: 'images' | 'volumes' | 'networks',
|
|
knownStackNames: string[]
|
|
): Promise<{ success: boolean; reclaimedBytes: number }> {
|
|
const knownSet = new Set(knownStackNames);
|
|
const projectToStack = await DockerController.resolveProjectNameMap(knownStackNames);
|
|
let reclaimedBytes = 0;
|
|
|
|
if (target === 'volumes') {
|
|
const rawVolumeData = await this.docker.listVolumes();
|
|
const rawVolumes: any[] = (this.validateApiData<any>(rawVolumeData)).Volumes || [];
|
|
const prunable = rawVolumes.filter((v: any) => {
|
|
return !!DockerController.resolveProjectLabel(v.Labels?.['com.docker.compose.project'], knownSet, projectToStack)
|
|
&& (v.UsageData?.RefCount ?? 1) === 0;
|
|
});
|
|
// Removals are independent and Docker handles concurrent volume
|
|
// deletes; parallelize so wall time matches the slowest single
|
|
// remove rather than the sum of all of them.
|
|
await Promise.all(prunable.map(async (vol) => {
|
|
try {
|
|
await this.docker.getVolume(vol.Name).remove({ force: true });
|
|
reclaimedBytes += vol.UsageData?.Size ?? 0;
|
|
} catch (e) {
|
|
console.error(`[pruneManagedOnly] Failed to remove volume ${vol.Name}:`, e);
|
|
}
|
|
}));
|
|
} else if (target === 'networks') {
|
|
const rawNetworks = await this.docker.listNetworks();
|
|
const prunable = (rawNetworks as any[]).filter((n: any) => {
|
|
return !!DockerController.resolveProjectLabel(n.Labels?.['com.docker.compose.project'], knownSet, projectToStack);
|
|
});
|
|
await Promise.all(prunable.map(async (net) => {
|
|
try {
|
|
await this.docker.getNetwork(net.Id).remove({ force: true });
|
|
} catch (e) {
|
|
console.error(`[pruneManagedOnly] Failed to remove network ${net.Name}:`, e);
|
|
}
|
|
}));
|
|
} else if (target === 'images') {
|
|
const allContainers = await this.docker.listContainers({ all: true });
|
|
const resolvedBase = path.resolve(COMPOSE_DIR);
|
|
const absDirToStack = DockerController.buildAbsDirMap(knownStackNames);
|
|
const unmanagedImageIds = new Set<string>();
|
|
for (const c of allContainers as any[]) {
|
|
const stack = DockerController.resolveContainerStack(
|
|
c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
|
);
|
|
if (!stack) unmanagedImageIds.add(c.ImageID);
|
|
}
|
|
const rawImages = await this.docker.listImages({ all: false });
|
|
const prunable = (rawImages as any[]).filter((img: any) =>
|
|
img.Containers === 0 && !unmanagedImageIds.has(img.Id)
|
|
);
|
|
await Promise.all(prunable.map(async (img) => {
|
|
try {
|
|
await this.docker.getImage(img.Id).remove({ force: true });
|
|
reclaimedBytes += img.Size ?? 0;
|
|
} catch (e) {
|
|
console.error(`[pruneManagedOnly] Failed to remove image ${img.Id}:`, e);
|
|
}
|
|
}));
|
|
}
|
|
|
|
return { success: true, reclaimedBytes };
|
|
}
|
|
|
|
public async getDiskUsageClassified(knownStackNames: string[]): Promise<{
|
|
reclaimableImages: number;
|
|
reclaimableContainers: number;
|
|
reclaimableVolumes: number;
|
|
reclaimableBuildCache: number;
|
|
reclaimableImageCount: number;
|
|
reclaimableContainerCount: number;
|
|
reclaimableVolumeCount: number;
|
|
reclaimableBuildCacheCount: number;
|
|
managedImageBytes: number;
|
|
unmanagedImageBytes: number;
|
|
managedVolumeBytes: number;
|
|
unmanagedVolumeBytes: number;
|
|
}> {
|
|
const [base, classified] = await Promise.all([
|
|
this.getDiskUsage(),
|
|
this.getClassifiedResources(knownStackNames),
|
|
]);
|
|
|
|
const managedImageBytes = classified.images
|
|
.filter(i => i.managedStatus === 'managed')
|
|
.reduce((acc, i) => acc + i.Size, 0);
|
|
const unmanagedImageBytes = classified.images
|
|
.filter(i => i.managedStatus === 'unmanaged')
|
|
.reduce((acc, i) => acc + i.Size, 0);
|
|
|
|
// Volume disk usage: query raw volumes for UsageData (not available in classified resources)
|
|
const rawVolumeData = await this.docker.listVolumes();
|
|
const rawVolumes: any[] = (this.validateApiData<any>(rawVolumeData)).Volumes || [];
|
|
const projectToStack = await DockerController.resolveProjectNameMap(knownStackNames);
|
|
const knownSet = new Set(knownStackNames);
|
|
|
|
const isVolumeManaged = (v: any): boolean =>
|
|
!!DockerController.resolveProjectLabel(v.Labels?.['com.docker.compose.project'], knownSet, projectToStack);
|
|
|
|
const managedVolumeBytes = rawVolumes
|
|
.filter(isVolumeManaged)
|
|
.reduce((acc: number, v: any) => acc + (v.UsageData?.Size ?? 0), 0);
|
|
const unmanagedVolumeBytes = rawVolumes
|
|
.filter((v: any) => !isVolumeManaged(v))
|
|
.reduce((acc: number, v: any) => acc + (v.UsageData?.Size ?? 0), 0);
|
|
|
|
return { ...base, managedImageBytes, unmanagedImageBytes, managedVolumeBytes, unmanagedVolumeBytes };
|
|
}
|
|
|
|
public async removeImage(id: string) {
|
|
const image = this.docker.getImage(id);
|
|
await image.remove({ force: true });
|
|
}
|
|
|
|
public async inspectImage(id: string) {
|
|
const image = this.docker.getImage(id);
|
|
const [inspect, history] = await Promise.all([image.inspect(), image.history()]);
|
|
return { inspect, history };
|
|
}
|
|
|
|
public async removeVolume(name: string) {
|
|
const volume = this.docker.getVolume(name);
|
|
await volume.remove({ force: true });
|
|
}
|
|
|
|
public async removeNetwork(id: string) {
|
|
const network = this.docker.getNetwork(id);
|
|
await network.remove({ force: true });
|
|
}
|
|
|
|
public async inspectNetwork(id: string) {
|
|
const network = this.docker.getNetwork(id);
|
|
return await network.inspect();
|
|
}
|
|
|
|
public async createNetwork(options: CreateNetworkOptions) {
|
|
if (!options.Name || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(options.Name)) {
|
|
throw new Error('Invalid network name. Use alphanumeric characters, hyphens, underscores, and dots.');
|
|
}
|
|
return await this.docker.createNetwork(options);
|
|
}
|
|
|
|
/**
|
|
* Attach a container to a Docker network. Idempotent: if the container is
|
|
* already attached the call resolves silently. Optionally pins the
|
|
* container's IPv4 address inside the network so other services can use
|
|
* static `extra_hosts` entries against it.
|
|
*/
|
|
public async connectContainerToNetwork(
|
|
networkName: string,
|
|
containerId: string,
|
|
opts: { ipv4Address?: string } = {},
|
|
): Promise<void> {
|
|
const network = this.docker.getNetwork(networkName);
|
|
const payload: { Container: string; EndpointConfig?: { IPAMConfig?: { IPv4Address: string } } } = {
|
|
Container: containerId,
|
|
};
|
|
if (opts.ipv4Address) {
|
|
payload.EndpointConfig = { IPAMConfig: { IPv4Address: opts.ipv4Address } };
|
|
}
|
|
try {
|
|
await network.connect(payload);
|
|
} catch (err) {
|
|
if (DockerController.isAlreadyConnectedError(err)) return;
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Detach a container from a Docker network. Idempotent: if the container
|
|
* is not attached the call resolves silently.
|
|
*/
|
|
public async disconnectContainerFromNetwork(
|
|
networkName: string,
|
|
containerId: string,
|
|
): Promise<void> {
|
|
const network = this.docker.getNetwork(networkName);
|
|
try {
|
|
await network.disconnect({ Container: containerId, Force: true });
|
|
} catch (err) {
|
|
if (DockerController.isNotConnectedError(err)) return;
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
private static isAlreadyConnectedError(err: unknown): boolean {
|
|
const e = err as { statusCode?: number; message?: string };
|
|
const msg = (e?.message || '').toLowerCase();
|
|
// Docker daemon returns 403 for several distinct cases (already
|
|
// attached, host-network containers, permission denied), so match the
|
|
// message body too rather than treating any 403 as idempotent success.
|
|
if (e?.statusCode === 403 && (msg.includes('already exists') || msg.includes('already attached'))) {
|
|
return true;
|
|
}
|
|
return msg.includes('already exists') || msg.includes('already attached');
|
|
}
|
|
|
|
private static isNotConnectedError(err: unknown): boolean {
|
|
const e = err as { statusCode?: number; message?: string };
|
|
if (e?.statusCode === 404) return true;
|
|
const msg = (e?.message || '').toLowerCase();
|
|
return msg.includes('is not connected') || msg.includes('no such container');
|
|
}
|
|
|
|
public async getRunningContainers() {
|
|
const containers = await this.docker.listContainers({ all: false });
|
|
return this.validateApiData<any[]>(containers);
|
|
}
|
|
|
|
public async getAllContainers() {
|
|
const containers = await this.docker.listContainers({ all: true });
|
|
return this.validateApiData<any[]>(containers);
|
|
}
|
|
|
|
/**
|
|
* Builds topology data with 2 Docker API calls instead of N+1.
|
|
* Fetches all networks + all containers in parallel, then maps
|
|
* container-to-network relationships in memory using NetworkSettings.
|
|
*/
|
|
public async getTopologyData(
|
|
knownStackNames: string[],
|
|
includeSystem: boolean,
|
|
): Promise<TopologyNetwork[]> {
|
|
const debug = isDebugEnabled();
|
|
const t0 = debug ? Date.now() : 0;
|
|
const knownSet = new Set(knownStackNames);
|
|
|
|
const [rawNetworks, rawContainers, projectToStack] = await Promise.all([
|
|
this.docker.listNetworks(),
|
|
this.docker.listContainers({ all: true }),
|
|
DockerController.resolveProjectNameMap(knownStackNames),
|
|
]);
|
|
|
|
const absDirToStack = DockerController.buildAbsDirMap(knownStackNames);
|
|
const resolvedBase = path.resolve(COMPOSE_DIR);
|
|
|
|
const networks = this.validateApiData<any[]>(rawNetworks);
|
|
const containers = this.validateApiData<any[]>(rawContainers);
|
|
|
|
// Build network map, optionally filtering system networks
|
|
const networkMap = new Map<string, TopologyNetwork>();
|
|
for (const net of networks) {
|
|
const isSystem = DockerController.SYSTEM_NETWORKS.has(net.Name);
|
|
if (isSystem && !includeSystem) continue;
|
|
|
|
const stack = isSystem
|
|
? null
|
|
: DockerController.resolveProjectLabel(
|
|
net.Labels?.['com.docker.compose.project'],
|
|
knownSet,
|
|
projectToStack,
|
|
);
|
|
const managedStatus: TopologyNetwork['managedStatus'] = isSystem
|
|
? 'system'
|
|
: stack ? 'managed' : 'unmanaged';
|
|
|
|
networkMap.set(net.Id, {
|
|
Id: net.Id,
|
|
Name: net.Name,
|
|
Driver: net.Driver ?? 'bridge',
|
|
Scope: net.Scope ?? 'local',
|
|
managedBy: stack,
|
|
managedStatus,
|
|
containers: [],
|
|
});
|
|
}
|
|
|
|
// Map containers to their networks via NetworkSettings.
|
|
// Stack resolution is deferred until a network match is found to avoid
|
|
// wasted work for containers not attached to any tracked network.
|
|
for (const c of containers) {
|
|
const netSettings: Record<string, { NetworkID?: string; IPAddress?: string }> =
|
|
c.NetworkSettings?.Networks ?? {};
|
|
|
|
let containerStack: string | null | undefined;
|
|
let stackResolved = false;
|
|
|
|
for (const [, netInfo] of Object.entries(netSettings)) {
|
|
const netId = netInfo.NetworkID;
|
|
if (!netId) continue;
|
|
const topology = networkMap.get(netId);
|
|
if (!topology) continue;
|
|
|
|
if (!stackResolved) {
|
|
containerStack = DockerController.resolveContainerStack(
|
|
c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase,
|
|
);
|
|
stackResolved = true;
|
|
}
|
|
|
|
topology.containers.push({
|
|
id: c.Id,
|
|
name: (c.Names?.[0] ?? '').replace(/^\//, '') || (c.Id ?? '').substring(0, 12),
|
|
ip: netInfo.IPAddress ?? '',
|
|
state: c.State ?? 'unknown',
|
|
image: c.Image ?? '',
|
|
stack: containerStack ?? null,
|
|
});
|
|
}
|
|
}
|
|
|
|
const result = Array.from(networkMap.values());
|
|
|
|
if (debug) {
|
|
const totalContainers = result.reduce((sum, n) => sum + n.containers.length, 0);
|
|
console.debug('[Resources:debug] Topology built', {
|
|
ms: Date.now() - t0,
|
|
networks: result.length,
|
|
containers: totalContainers,
|
|
systemFiltered: !includeSystem,
|
|
stacksKnown: knownStackNames.length,
|
|
});
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/** Resolves a Docker Compose project label to a known Sencho stack name, or null. */
|
|
private static resolveProjectLabel(
|
|
project: string | undefined,
|
|
knownSet: Set<string>,
|
|
projectToStack: Record<string, string>,
|
|
): string | null {
|
|
if (!project) return null;
|
|
if (knownSet.has(project)) return project;
|
|
if (projectToStack[project]) return projectToStack[project];
|
|
return null;
|
|
}
|
|
|
|
/** Builds a map from absolute stack directory paths to stack names. */
|
|
private static buildAbsDirMap(stackNames: string[]): Record<string, string> {
|
|
const map: Record<string, string> = {};
|
|
for (const stackDir of stackNames) {
|
|
map[path.join(COMPOSE_DIR, stackDir)] = stackDir;
|
|
}
|
|
return map;
|
|
}
|
|
|
|
/**
|
|
* Resolves which Sencho stack a container belongs to using a multi-fallback strategy.
|
|
* Handles containers whose labels predate Sencho's reorganization of compose files into subdirectories.
|
|
*/
|
|
private static resolveContainerStack(
|
|
containerLabels: Record<string, string> | undefined,
|
|
projectToStack: Record<string, string>,
|
|
knownStackSet: Set<string>,
|
|
absDirToStack: Record<string, string>,
|
|
resolvedBase: string,
|
|
): string | null {
|
|
if (!containerLabels) return null;
|
|
|
|
// Primary: match by project name (handles name: overrides and standard directory-based names)
|
|
const project = containerLabels['com.docker.compose.project'];
|
|
if (project && projectToStack[project]) return projectToStack[project];
|
|
|
|
// Fallback 1: match by working_dir
|
|
const workingDir = containerLabels['com.docker.compose.project.working_dir'];
|
|
if (workingDir) {
|
|
const match = absDirToStack[workingDir] ?? absDirToStack[path.resolve(workingDir)];
|
|
if (match) return match;
|
|
}
|
|
|
|
// Fallback 2: match by service name
|
|
const serviceName = containerLabels['com.docker.compose.service'];
|
|
if (serviceName && knownStackSet.has(serviceName)) return serviceName;
|
|
|
|
// Fallback 3: extract stack from config_files path
|
|
const configFiles = containerLabels['com.docker.compose.project.config_files'];
|
|
if (configFiles) {
|
|
const firstFile = configFiles.split(',')[0].trim();
|
|
const resolvedFile = path.resolve(firstFile);
|
|
if (isPathWithinBase(resolvedFile, resolvedBase)) {
|
|
const relative = resolvedFile.slice(resolvedBase.length + 1);
|
|
const firstSegment = relative.split(path.sep)[0].replace(/\.(ya?ml)$/, '');
|
|
if (knownStackSet.has(firstSegment)) return firstSegment;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Builds (or returns cached) mapping from Docker project name to Sencho stack directory name.
|
|
* Compose files with a top-level `name:` field override the default project name.
|
|
*/
|
|
private static async resolveProjectNameMap(stackNames: string[]): Promise<Record<string, string>> {
|
|
return CacheService.getInstance().getOrFetch(
|
|
PROJECT_NAME_CACHE_KEY,
|
|
PROJECT_NAME_CACHE_TTL_MS,
|
|
async () => {
|
|
const map: Record<string, string> = {};
|
|
|
|
await Promise.all(stackNames.map(async (stackDir) => {
|
|
map[stackDir] = stackDir;
|
|
|
|
for (const fileName of COMPOSE_FILE_NAMES) {
|
|
const filePath = path.join(COMPOSE_DIR, stackDir, fileName);
|
|
try {
|
|
const content = await fs.readFile(filePath, 'utf-8');
|
|
const parsed = yaml.parse(content);
|
|
if (parsed?.name && typeof parsed.name === 'string') {
|
|
map[parsed.name] = stackDir;
|
|
}
|
|
break;
|
|
} catch (err: unknown) {
|
|
const code = (err as NodeJS.ErrnoException)?.code;
|
|
if (code !== 'ENOENT' && code !== 'ENOTDIR') {
|
|
console.error('[DockerController] Failed to read %s:', sanitizeForLog(filePath), sanitizeForLog((err as Error)?.message ?? String(err)));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}));
|
|
|
|
return map;
|
|
},
|
|
);
|
|
}
|
|
|
|
public async getBulkStackStatuses(stackNames: string[]): Promise<Record<string, BulkStackInfo>> {
|
|
// Run Docker API call and project name resolution in parallel
|
|
const [allContainers, projectToStack] = await Promise.all([
|
|
this.docker.listContainers({ all: true }),
|
|
DockerController.resolveProjectNameMap(stackNames),
|
|
]);
|
|
|
|
const absDirToStack = DockerController.buildAbsDirMap(stackNames);
|
|
const knownStackSet = new Set(stackNames);
|
|
const resolvedBase = path.resolve(COMPOSE_DIR);
|
|
|
|
const result: Record<string, BulkStackInfo> = {};
|
|
for (const name of stackNames) {
|
|
result[name] = { status: 'unknown' };
|
|
}
|
|
|
|
for (const container of allContainers as any[]) {
|
|
const stackDir = DockerController.resolveContainerStack(
|
|
container.Labels, projectToStack, knownStackSet, absDirToStack, resolvedBase,
|
|
);
|
|
|
|
if (!stackDir || !result[stackDir]) continue;
|
|
|
|
if (container.State === 'running') {
|
|
result[stackDir].status = 'running';
|
|
|
|
// Track the oldest running container's creation time as a proxy for
|
|
// stack uptime. Docker's listContainers payload exposes Created (unix
|
|
// seconds) but not StartedAt; for compose stacks the gap is small
|
|
// enough to treat as uptime without paying for a per-container inspect.
|
|
const created = typeof container.Created === 'number' ? container.Created : undefined;
|
|
if (created !== undefined) {
|
|
const existing = result[stackDir].runningSince;
|
|
if (existing === undefined || created < existing) {
|
|
result[stackDir].runningSince = created;
|
|
}
|
|
}
|
|
|
|
// Detect main web port (first running container with a matchable port wins)
|
|
if (result[stackDir].mainPort === undefined && Array.isArray(container.Ports) && container.Ports.length > 0) {
|
|
const ports = container.Ports as { PrivatePort?: number; PublicPort?: number }[];
|
|
let match = ports.find(p => p.PrivatePort && WEB_UI_PORTS.includes(p.PrivatePort));
|
|
if (!match) match = ports.find(p => p.PublicPort && WEB_UI_PORTS.includes(p.PublicPort));
|
|
if (!match) match = ports.find(p =>
|
|
(!p.PrivatePort || !IGNORE_PORTS.includes(p.PrivatePort)) &&
|
|
(!p.PublicPort || !IGNORE_PORTS.includes(p.PublicPort))
|
|
);
|
|
const chosen = match || ports[0];
|
|
if (chosen?.PublicPort) {
|
|
result[stackDir].mainPort = chosen.PublicPort;
|
|
}
|
|
}
|
|
} else if (result[stackDir].status !== 'running') {
|
|
result[stackDir].status = 'exited';
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Returns a map of host ports currently bound by running containers,
|
|
* with ownership info (Sencho-managed stack name or external).
|
|
*/
|
|
public async getPortsInUse(knownStackNames: string[]): Promise<Record<number, PortInUseInfo>> {
|
|
const [allContainers, projectToStack] = await Promise.all([
|
|
this.docker.listContainers({ all: false }),
|
|
DockerController.resolveProjectNameMap(knownStackNames),
|
|
]);
|
|
|
|
const absDirToStack = DockerController.buildAbsDirMap(knownStackNames);
|
|
const knownStackSet = new Set(knownStackNames);
|
|
const resolvedBase = path.resolve(COMPOSE_DIR);
|
|
|
|
const result: Record<number, PortInUseInfo> = {};
|
|
|
|
for (const container of allContainers as Array<{ Names?: string[]; Labels?: Record<string, string>; Ports?: Array<{ PublicPort?: number }> }>) {
|
|
const stackDir = DockerController.resolveContainerStack(
|
|
container.Labels, projectToStack, knownStackSet, absDirToStack, resolvedBase,
|
|
);
|
|
|
|
const containerName = (container.Names?.[0] || '').replace(/^\//, '');
|
|
|
|
if (!Array.isArray(container.Ports)) continue;
|
|
|
|
for (const port of container.Ports) {
|
|
if (!port.PublicPort || port.PublicPort <= 0) continue;
|
|
// First container to claim a port wins (avoids overwrites)
|
|
if (result[port.PublicPort]) continue;
|
|
result[port.PublicPort] = { stack: stackDir, container: containerName };
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
public async getContainersByStack(stackName: string) {
|
|
const stackDir = path.join(COMPOSE_DIR, stackName);
|
|
|
|
try {
|
|
const { stdout, stderr } = await execAsync('docker compose ps --format json -a', {
|
|
cwd: stackDir,
|
|
env: {
|
|
...process.env,
|
|
PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
|
|
}
|
|
});
|
|
|
|
// Robust JSON parsing - handle both JSON array and newline-separated JSON objects
|
|
// Docker Compose v2 may return either format depending on version
|
|
interface ComposeContainer {
|
|
ID?: string;
|
|
Name?: string;
|
|
Service?: string;
|
|
State?: string;
|
|
Status?: string;
|
|
Publishers?: { URL?: string, TargetPort?: number, PublishedPort?: number }[];
|
|
}
|
|
|
|
let containers: ComposeContainer[] = [];
|
|
|
|
// Only parse if stdout has content
|
|
if (stdout && stdout.trim() !== '') {
|
|
try {
|
|
// Try parsing as a standard JSON array
|
|
const parsed = JSON.parse(stdout);
|
|
containers = Array.isArray(parsed) ? parsed : [parsed];
|
|
} catch (parseError) {
|
|
// Fallback: parse newline-separated JSON objects, filtering out empty lines
|
|
try {
|
|
const lines = stdout.trim().split('\n').filter(line => line.trim() !== '');
|
|
containers = lines.map(line => JSON.parse(line) as ComposeContainer);
|
|
} catch (innerError) {
|
|
// Log parsing failure with stderr for debugging
|
|
console.error('Docker Compose JSON Parse Error for %s:', sanitizeForLog(stackName), sanitizeForLog(stderr || (parseError as Error).message));
|
|
// Don't return empty - trigger smart fallback below
|
|
}
|
|
}
|
|
}
|
|
|
|
// If containers found via docker compose ps, return them
|
|
if (containers.length > 0) {
|
|
// Map to frontend's expected interface
|
|
// Note: docker compose ps returns Name (singular), but frontend expects Names (array)
|
|
// Dockerode returns Names with leading slash, so we add it for compatibility
|
|
const mapped = containers.map((c) => {
|
|
let Ports: { PrivatePort: number, PublicPort: number }[] = [];
|
|
if (c.Publishers && Array.isArray(c.Publishers)) {
|
|
Ports = c.Publishers
|
|
.filter(p => typeof p.PublishedPort === 'number' && p.PublishedPort > 0)
|
|
.map(p => ({ PrivatePort: (p.TargetPort || 0) as number, PublicPort: p.PublishedPort as number }));
|
|
}
|
|
return {
|
|
Id: c.ID || '',
|
|
Names: ['/' + (c.Name || '')], // Add leading slash to match Dockerode format
|
|
Service: c.Service || '',
|
|
State: c.State || 'unknown',
|
|
Status: c.Status || '',
|
|
Ports
|
|
};
|
|
});
|
|
return await this.enrichContainers(mapped);
|
|
}
|
|
|
|
// SMART FALLBACK: Trigger when docker compose ps returns empty
|
|
// This handles legacy containers with incorrect project labels
|
|
return await this.enrichContainers(await this.smartFallback(stackName, stackDir));
|
|
|
|
} catch (error) {
|
|
// If command fails (e.g., stack not deployed, invalid YAML, missing env_file,
|
|
// or host under memory pressure causing posix_spawn to fail with ENOMEM,
|
|
// which Linux libuv can surface as ENOENT).
|
|
const execError = error as NodeJS.ErrnoException & { stderr?: string };
|
|
const mapped = describeSpawnError(execError, { command: 'docker compose ps' });
|
|
const detail = execError.stderr || mapped.message;
|
|
console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail));
|
|
|
|
// Try smart fallback even on error
|
|
return await this.enrichContainers(await this.smartFallback(stackName, stackDir));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Inspect each container to attach healthcheck status, image tag, and image digest.
|
|
* Each mapper catches its own errors, so Promise.all never rejects (allSettled adds ceremony with no behavior change).
|
|
*/
|
|
private async enrichContainers<T extends { Id?: string }>(list: T[]): Promise<Array<T & { healthStatus: 'healthy' | 'unhealthy' | 'starting' | 'none'; Image?: string; ImageID?: string }>> {
|
|
return Promise.all(list.map(async (c) => {
|
|
const base = { ...c, healthStatus: 'none' as const };
|
|
if (!c.Id) return base;
|
|
try {
|
|
const info = await this.docker.getContainer(c.Id).inspect();
|
|
const health = info.State?.Health?.Status;
|
|
const healthStatus: 'healthy' | 'unhealthy' | 'starting' | 'none' =
|
|
health === 'healthy' || health === 'unhealthy' || health === 'starting' ? health : 'none';
|
|
return { ...c, healthStatus, Image: info.Config?.Image, ImageID: info.Image };
|
|
} catch {
|
|
return base;
|
|
}
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* Smart Fallback: Find legacy containers by parsing compose YAML definitions.
|
|
* This handles containers that were deployed with incorrect project labels
|
|
* that cause `docker compose ps` to ignore them.
|
|
*/
|
|
private async smartFallback(stackName: string, stackDir: string): Promise<any[]> {
|
|
try {
|
|
// 1. Flexible Compose File Discovery
|
|
// Try multiple valid compose file names
|
|
const composeFileNames = COMPOSE_FILE_NAMES;
|
|
let yamlContent: string | null = null;
|
|
|
|
for (const fileName of composeFileNames) {
|
|
try {
|
|
yamlContent = await fs.readFile(path.join(stackDir, fileName), 'utf-8');
|
|
break; // Successfully read a file, stop trying
|
|
} catch {
|
|
// File doesn't exist, try next
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (!yamlContent) {
|
|
// No compose file found
|
|
return [];
|
|
}
|
|
|
|
const parsedYaml = yaml.parse(yamlContent);
|
|
|
|
if (!parsedYaml || !parsedYaml.services) return [];
|
|
|
|
// 2. Extract expected container names with legacy prefix support
|
|
const expectedNames: string[] = [];
|
|
for (const [serviceName, serviceConfig] of Object.entries(parsedYaml.services)) {
|
|
const config = serviceConfig as any;
|
|
if (config.container_name) {
|
|
expectedNames.push(config.container_name);
|
|
} else {
|
|
// Standard v2 naming
|
|
expectedNames.push(serviceName);
|
|
expectedNames.push(`${stackName}-${serviceName}-1`);
|
|
// Legacy project prefix catch - accounts for orphan containers
|
|
expectedNames.push(`compose-${serviceName}-1`);
|
|
expectedNames.push(`compose_${serviceName}_1`);
|
|
}
|
|
}
|
|
|
|
// 3. Query the raw Docker daemon
|
|
const allContainers = await this.docker.listContainers({ all: true });
|
|
|
|
// 4. Match containers by name
|
|
const fallbackContainers = allContainers.filter(container => {
|
|
// container.Names usually looks like ['/plex']
|
|
return container.Names.some(name => {
|
|
const strippedName = name.replace(/^\//, '');
|
|
return expectedNames.includes(strippedName);
|
|
});
|
|
});
|
|
|
|
// 5. Map to the frontend interface
|
|
return fallbackContainers.map(c => {
|
|
let Ports: { PrivatePort: number, PublicPort: number }[] = [];
|
|
if (c.Ports && Array.isArray(c.Ports)) {
|
|
Ports = c.Ports
|
|
.filter((p: any) => typeof p.PublicPort === 'number' && p.PublicPort > 0)
|
|
.map((p: any) => ({ PrivatePort: (p.PrivatePort || 0) as number, PublicPort: p.PublicPort as number }));
|
|
}
|
|
return {
|
|
Id: c.Id,
|
|
Names: c.Names,
|
|
State: c.State,
|
|
Status: c.Status,
|
|
Ports
|
|
};
|
|
});
|
|
} catch (fallbackError) {
|
|
console.error('Smart Fallback failed for %s:', sanitizeForLog(stackName), sanitizeForLog((fallbackError as Error)?.message ?? String(fallbackError)));
|
|
return [];
|
|
}
|
|
}
|
|
|
|
public async streamContainerLogs(containerId: string, req: any, res: any): Promise<void> {
|
|
const container = this.docker.getContainer(containerId);
|
|
|
|
// 1. Set SSE Headers
|
|
res.setHeader('Content-Type', 'text/event-stream');
|
|
res.setHeader('Cache-Control', 'no-cache');
|
|
res.setHeader('Connection', 'keep-alive');
|
|
res.setHeader('X-Accel-Buffering', 'no');
|
|
res.flushHeaders();
|
|
|
|
try {
|
|
const logStream = await container.logs({
|
|
follow: true,
|
|
stdout: true,
|
|
stderr: true,
|
|
tail: 100 // Send the last 100 lines immediately for context
|
|
});
|
|
|
|
// 2. Process and forward the stream
|
|
logStream.on('data', (chunk: Buffer) => {
|
|
// Docker multiplexes stdout/stderr with an 8-byte header if TTY is false.
|
|
let data = chunk;
|
|
if (chunk.length > 8 && (chunk[0] === 1 || chunk[0] === 2)) {
|
|
data = chunk.slice(8);
|
|
}
|
|
|
|
const text = data.toString('utf-8');
|
|
const lines = text.split('\n');
|
|
|
|
lines.forEach(line => {
|
|
if (line.trim()) {
|
|
res.write(`data: ${JSON.stringify(line)}\n\n`);
|
|
}
|
|
});
|
|
});
|
|
|
|
// 3. Cleanup on disconnect
|
|
req.on('close', () => {
|
|
(logStream as any).destroy();
|
|
});
|
|
|
|
} catch (error: any) {
|
|
res.write(`data: ${JSON.stringify('[Sencho] Error fetching logs: ' + error.message)}\n\n`);
|
|
res.end();
|
|
}
|
|
}
|
|
|
|
// State-safe: silently ignores 304 "already started" errors
|
|
public async startContainer(containerId: string) {
|
|
try {
|
|
const container = this.docker.getContainer(containerId);
|
|
await container.start();
|
|
} catch (error: any) {
|
|
if (error?.statusCode === 304) {
|
|
// Container already running - not an error
|
|
return;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
// State-safe: silently ignores 304 "already stopped" errors
|
|
public async stopContainer(containerId: string) {
|
|
try {
|
|
const container = this.docker.getContainer(containerId);
|
|
await container.stop();
|
|
} catch (error: any) {
|
|
if (error?.statusCode === 304) {
|
|
// Container already stopped - not an error
|
|
return;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
public async restartContainer(containerId: string) {
|
|
const container = this.docker.getContainer(containerId);
|
|
await container.restart();
|
|
}
|
|
|
|
public async getOrphanContainers(knownStackNames: string[]) {
|
|
// 1. Fetch all containers (running and stopped)
|
|
const allContainers = await this.docker.listContainers({ all: true });
|
|
|
|
// 2. Filter and categorize orphans
|
|
const orphans: Record<string, any[]> = {};
|
|
|
|
allContainers.forEach((container) => {
|
|
// Look for the docker compose project label
|
|
const projectName = container.Labels?.['com.docker.compose.project'];
|
|
|
|
// If it has a project label, but the project is NOT in our known list...
|
|
if (projectName && !knownStackNames.includes(projectName)) {
|
|
if (!orphans[projectName]) {
|
|
orphans[projectName] = [];
|
|
}
|
|
orphans[projectName].push({
|
|
Id: container.Id,
|
|
Names: container.Names,
|
|
State: container.State,
|
|
Status: container.Status,
|
|
Image: container.Image
|
|
});
|
|
}
|
|
});
|
|
|
|
return orphans;
|
|
}
|
|
|
|
public async removeContainers(containerIds: string[]) {
|
|
if (isDebugEnabled()) console.debug('[Resources:debug] removeContainers', { count: containerIds.length });
|
|
const results = [];
|
|
for (const id of containerIds) {
|
|
try {
|
|
const container = this.docker.getContainer(id);
|
|
await container.remove({ force: true });
|
|
results.push({ id, success: true });
|
|
} catch (error: any) {
|
|
console.error('Failed to remove container %s:', sanitizeForLog(id), sanitizeForLog(error.message));
|
|
results.push({ id, success: false, error: error.message });
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
public async streamStats(containerId: string, ws: WebSocket) {
|
|
const container = this.docker.getContainer(containerId);
|
|
const stats = await container.stats({ stream: true });
|
|
|
|
stats.on('data', (chunk: Buffer) => {
|
|
if (ws.readyState === WebSocket.OPEN) {
|
|
ws.send(chunk.toString());
|
|
}
|
|
});
|
|
|
|
stats.on('error', (err: Error) => {
|
|
if (ws.readyState === WebSocket.OPEN) {
|
|
ws.send(JSON.stringify({ error: err.message }));
|
|
}
|
|
});
|
|
|
|
stats.on('end', () => {
|
|
if (ws.readyState === WebSocket.OPEN) {
|
|
ws.send(JSON.stringify({ end: true }));
|
|
}
|
|
});
|
|
|
|
// Destroy the Docker stats stream when the WebSocket closes to prevent
|
|
// orphaned streams polling the daemon after client disconnect.
|
|
ws.on('close', () => {
|
|
try { (stats as any).destroy(); } catch (e) {
|
|
// Stream already ended before client disconnected
|
|
console.warn('[DockerController] Stats stream already ended on WS close:', (e as Error).message);
|
|
}
|
|
});
|
|
}
|
|
|
|
public async getContainerStatsStream(containerId: string): Promise<string> {
|
|
const container = this.docker.getContainer(containerId);
|
|
const stats = await container.stats({ stream: false });
|
|
return typeof stats === 'string' ? stats : JSON.stringify(stats);
|
|
}
|
|
|
|
/** Return the cumulative restart count for a container via inspect(). */
|
|
public async getContainerRestartCount(containerId: string): Promise<number> {
|
|
const container = this.docker.getContainer(containerId);
|
|
const info = await container.inspect();
|
|
return info.RestartCount ?? 0;
|
|
}
|
|
|
|
/**
|
|
* Exec into a container with full session isolation.
|
|
* All state (exec instance, stream) lives in this closure - no singleton traps.
|
|
* The WebSocket message handler is registered here to handle input, resize, and cleanup.
|
|
*/
|
|
public async execContainer(containerId: string, ws: WebSocket) {
|
|
try {
|
|
// Input validation
|
|
if (!containerId || typeof containerId !== 'string') {
|
|
console.warn('[Exec] Empty or invalid containerId');
|
|
if (ws.readyState === WebSocket.OPEN) {
|
|
ws.send('\r\n\x1b[31mError: No container ID provided\x1b[0m\r\n');
|
|
ws.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
const container = this.docker.getContainer(containerId);
|
|
|
|
// Verify the container is running before attempting exec
|
|
const info = await container.inspect();
|
|
if (!info.State?.Running) {
|
|
console.warn('[Exec] Container not running:', containerId);
|
|
if (ws.readyState === WebSocket.OPEN) {
|
|
ws.send('\r\n\x1b[31mError: Container is not running\x1b[0m\r\n');
|
|
ws.close();
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Try bash first, fall back to sh.
|
|
// Both exec creation AND start must be inside the try/catch because
|
|
// some runtimes reject unknown binaries at start(), not at creation.
|
|
const execOpts = { AttachStdin: true, AttachStdout: true, AttachStderr: true, Tty: true } as const;
|
|
let dockerExec: Docker.Exec;
|
|
let stream: import('stream').Duplex;
|
|
let shellType = '/bin/bash';
|
|
try {
|
|
dockerExec = await container.exec({ ...execOpts, Cmd: ['/bin/bash'] });
|
|
stream = await dockerExec.start({ hijack: true, stdin: true });
|
|
} catch {
|
|
shellType = '/bin/sh';
|
|
dockerExec = await container.exec({ ...execOpts, Cmd: ['/bin/sh'] });
|
|
stream = await dockerExec.start({ hijack: true, stdin: true });
|
|
}
|
|
|
|
if (isDebugEnabled()) console.debug('[Exec:diag] Creating exec', { containerId, shell: shellType });
|
|
console.log('[Exec] Shell session started', { containerId, shell: shellType });
|
|
|
|
// --- Downstream: container output → client ---
|
|
stream.on('data', (chunk: Buffer) => {
|
|
if (ws.readyState === WebSocket.OPEN) {
|
|
ws.send(chunk.toString());
|
|
}
|
|
});
|
|
|
|
stream.on('error', (err: Error) => {
|
|
console.error('[Exec] Stream error:', err.message, { containerId });
|
|
});
|
|
|
|
stream.on('end', () => {
|
|
console.log('[Exec] Shell session ended', { containerId, reason: 'stream-end' });
|
|
if (ws.readyState === WebSocket.OPEN) {
|
|
ws.close();
|
|
}
|
|
});
|
|
|
|
// --- Upstream: client messages → container ---
|
|
ws.on('message', (raw: WebSocket.Data) => {
|
|
try {
|
|
const msg = JSON.parse(raw.toString());
|
|
|
|
switch (msg.type) {
|
|
case 'input':
|
|
if (msg.data) {
|
|
stream.write(msg.data);
|
|
}
|
|
break;
|
|
|
|
case 'resize':
|
|
if (msg.rows && msg.cols) {
|
|
if (isDebugEnabled()) console.debug('[Exec:diag] Terminal resize', { containerId, rows: msg.rows, cols: msg.cols });
|
|
dockerExec.resize({ h: msg.rows, w: msg.cols }).catch((e: Error) => {
|
|
// Exec may have ended before resize completes
|
|
console.warn('[Exec] Resize failed (exec may have ended):', e.message);
|
|
});
|
|
}
|
|
break;
|
|
|
|
case 'ping':
|
|
// Keep-alive, no-op
|
|
break;
|
|
}
|
|
} catch (e) {
|
|
// Non-JSON or malformed WebSocket message
|
|
console.warn('[Exec] Ignoring malformed WS message:', (e as Error).message);
|
|
}
|
|
});
|
|
|
|
// --- Cleanup: prevent zombie processes ---
|
|
ws.on('close', () => {
|
|
console.log('[Exec] Shell session ended', { containerId, reason: 'ws-close' });
|
|
try {
|
|
stream.destroy();
|
|
} catch (e) {
|
|
// Stream already destroyed before WS close
|
|
console.warn('[Exec] Stream already destroyed on WS close:', (e as Error).message);
|
|
}
|
|
});
|
|
|
|
} catch (error) {
|
|
const err = error as Error;
|
|
console.error('[Exec] Failed to start shell:', err.message, { containerId });
|
|
if (ws.readyState === WebSocket.OPEN) {
|
|
ws.send(`\r\n\x1b[31mFailed to start shell: ${err.message}\x1b[0m\r\n`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export const globalDockerNetwork = { rxSec: 0, txSec: 0 };
|
|
let lastNetSum = { rx: 0, tx: 0, timestamp: Date.now() };
|
|
let isUpdatingNetwork = false;
|
|
|
|
export const updateGlobalDockerNetwork = async () => {
|
|
if (isUpdatingNetwork) return; // Prevent overlapping calls
|
|
isUpdatingNetwork = true;
|
|
try {
|
|
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
|
const dockerController = DockerController.getInstance(nodeId);
|
|
const containers = await dockerController.getRunningContainers();
|
|
|
|
const statsResults = await Promise.allSettled(
|
|
containers.map(c => dockerController.getContainerStatsStream(c.Id))
|
|
);
|
|
|
|
let currentRxSum = 0;
|
|
let currentTxSum = 0;
|
|
|
|
for (const result of statsResults) {
|
|
if (result.status === 'fulfilled') {
|
|
try {
|
|
const stats = typeof result.value === 'string' ? JSON.parse(result.value) : result.value;
|
|
if (stats.networks) {
|
|
for (const [_, net] of Object.entries(stats.networks) as any) {
|
|
currentRxSum += net.rx_bytes || 0;
|
|
currentTxSum += net.tx_bytes || 0;
|
|
}
|
|
}
|
|
} catch (e) {
|
|
// ignore parsing errors
|
|
}
|
|
}
|
|
}
|
|
|
|
const now = Date.now();
|
|
const timeDiffSeconds = (now - lastNetSum.timestamp) / 1000;
|
|
|
|
if (timeDiffSeconds > 0) {
|
|
const rxDelta = currentRxSum >= lastNetSum.rx ? currentRxSum - lastNetSum.rx : 0;
|
|
const txDelta = currentTxSum >= lastNetSum.tx ? currentTxSum - lastNetSum.tx : 0;
|
|
|
|
globalDockerNetwork.rxSec = rxDelta / timeDiffSeconds;
|
|
globalDockerNetwork.txSec = txDelta / timeDiffSeconds;
|
|
}
|
|
|
|
lastNetSum = { rx: currentRxSum, tx: currentTxSum, timestamp: now };
|
|
} catch {
|
|
// Silently skip when Docker is unreachable (e.g. no local engine).
|
|
// Network stats will remain at their last known values.
|
|
} finally {
|
|
isUpdatingNetwork = false;
|
|
}
|
|
};
|
|
|
|
// Poll network stats every 5s (reduced from 3s to lower Docker daemon pressure)
|
|
setInterval(updateGlobalDockerNetwork, 5000);
|
|
|
|
export default DockerController;
|