diff --git a/apps/dokploy/__test__/server/server-health-subnet.test.ts b/apps/dokploy/__test__/server/server-health-subnet.test.ts new file mode 100644 index 000000000..9d0f7f120 --- /dev/null +++ b/apps/dokploy/__test__/server/server-health-subnet.test.ts @@ -0,0 +1,24 @@ +import { getSubnetCapacity } from "@dokploy/server/services/server-health"; +import { describe, expect, test } from "vitest"; + +describe("getSubnetCapacity", () => { + test("returns null for missing/invalid input", () => { + expect(getSubnetCapacity(undefined)).toBeNull(); + expect(getSubnetCapacity("")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0")).toBeNull(); + expect(getSubnetCapacity("not-a-subnet")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0/33")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0/-1")).toBeNull(); + }); + + test("excludes network and broadcast addresses", () => { + expect(getSubnetCapacity("10.0.1.0/24")).toBe(254); + expect(getSubnetCapacity("10.0.0.0/16")).toBe(65534); + expect(getSubnetCapacity("10.99.99.0/30")).toBe(2); + }); + + test("returns 0 for subnets too small to hold a host", () => { + expect(getSubnetCapacity("10.0.0.0/31")).toBe(0); + expect(getSubnetCapacity("10.0.0.0/32")).toBe(0); + }); +}); diff --git a/apps/dokploy/components/dashboard/docker/health/show-health.tsx b/apps/dokploy/components/dashboard/docker/health/show-health.tsx new file mode 100644 index 000000000..003b21c7f --- /dev/null +++ b/apps/dokploy/components/dashboard/docker/health/show-health.tsx @@ -0,0 +1,454 @@ +import { + AlertTriangle, + Cpu, + Download, + HardDrive, + Loader2, + Network, + RefreshCw, + Server as ServerIcon, +} from "lucide-react"; +import { useState } from "react"; +import { CodeEditor } from "@/components/shared/code-editor"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; +import { Progress } from "@/components/ui/progress"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { api } from "@/utils/api"; + +interface Props { + serverId?: string; +} + +const bytesToGb = (bytes: number) => (bytes / 1024 ** 3).toFixed(1); +const nanoCpusToCores = (nanoCpus: number) => (nanoCpus / 1e9).toFixed(2); +const pct = (used: number, total: number) => + Math.min(100, Math.round((used / (total || 1)) * 100)); + +const SINCE_HOURS_OPTIONS = [ + { label: "Last hour", value: 1 }, + { label: "Last 6 hours", value: 6 }, + { label: "Last 24 hours", value: 24 }, + { label: "Last 7 days", value: 168 }, +]; + +export const ShowHealth = ({ serverId }: Props) => { + const [sinceHours, setSinceHours] = useState(24); + const { + data: health, + isFetching, + refetch, + isFetched, + } = api.docker.getServerHealth.useQuery( + { serverId, sinceHours }, + { refetchOnMount: false, refetchOnWindowFocus: false }, + ); + + const memUsedPct = health + ? pct(health.resources.memUsedBytes, health.resources.memTotalBytes) + : 0; + const diskUsedPct = health + ? pct(health.disk.usedBytes, health.disk.totalBytes) + : 0; + const inotifyUsedPct = health + ? pct(health.inotify.currentInstances, health.inotify.maxInstances) + : 0; + + const daemonLogsWindowText = health?.daemonLogsWindow + ? `Logs from ${new Date(health.daemonLogsWindow.fromEpoch * 1000).toLocaleString()} to ${new Date(health.daemonLogsWindow.toEpoch * 1000).toLocaleString()}` + : null; + const daemonErrorsText = [ + daemonLogsWindowText, + health && health.daemonErrors.length > 0 + ? health.daemonErrors.join("\n") + : "(no daemon errors matched in this window)", + ] + .filter(Boolean) + .join("\n\n"); + + const buildHealthReport = () => { + if (!health) return ""; + const lines: string[] = []; + lines.push("# Dokploy server health report"); + lines.push(`Generated: ${new Date(health.checkedAt).toLocaleString()}`); + lines.push( + `Window: ${SINCE_HOURS_OPTIONS.find((o) => o.value === sinceHours)?.label ?? `${sinceHours}h`}`, + ); + lines.push(""); + + lines.push("## Containers & services"); + lines.push(`Containers: ${health.containers.containerCount}`); + lines.push(`Swarm services: ${health.containers.serviceCount}`); + lines.push(""); + + lines.push("## Host resources"); + lines.push( + `Memory: ${bytesToGb(health.resources.memUsedBytes)} / ${bytesToGb(health.resources.memTotalBytes)} GB`, + ); + lines.push(`CPU cores: ${health.resources.cpuCount}`); + lines.push(""); + + lines.push("## Disk (/)"); + lines.push( + `${bytesToGb(health.disk.usedBytes)} / ${bytesToGb(health.disk.totalBytes)} GB`, + ); + lines.push(""); + + lines.push("## Inotify"); + lines.push( + `max_user_instances: ${health.inotify.currentInstances} / ${health.inotify.maxInstances}`, + ); + lines.push(`max_user_watches: ${health.inotify.maxWatches}`); + lines.push(`max_queued_events: ${health.inotify.maxQueuedEvents}`); + lines.push( + `Persisted in sysctl: ${health.inotify.persisted ? "yes" : "no"}`, + ); + lines.push(""); + + lines.push("## Docker networks"); + lines.push(`Total networks: ${health.dockerNetworks.count}`); + if (health.dockerNetworks.addressPools) { + lines.push( + `default-address-pools: ${JSON.stringify(health.dockerNetworks.addressPools)}`, + ); + } + if (health.dockerNetworks.usageError) { + lines.push( + `Could not read network IP usage: ${health.dockerNetworks.usageError}`, + ); + } else if (health.dockerNetworks.usage.length > 0) { + lines.push(""); + lines.push( + "| Network | Driver | Subnet | IPs in use | Capacity | Usage |", + ); + lines.push("|---|---|---|---|---|---|"); + for (const n of health.dockerNetworks.usage) { + lines.push( + `| ${n.name} | ${n.driver} | ${n.subnet ?? "auto"} | ${n.containersInUse} | ${n.capacity ?? "—"} | ${n.percentUsed ?? "—"}% |`, + ); + } + } + lines.push(""); + + if (health.reservation) { + lines.push("## Memory/CPU reservation"); + lines.push( + `Reserved memory: ${bytesToGb(health.reservation.memoryReservedBytes)} GB`, + ); + lines.push( + `Reserved CPU: ${nanoCpusToCores(health.reservation.cpuReservedNanoCpus)} cores`, + ); + lines.push(`Across ${health.reservation.appCount} Application services`); + if (health.reservation.unsupportedComposeCount > 0) { + lines.push( + `${health.reservation.unsupportedComposeCount} Compose service(s) not included (not tracked per-service)`, + ); + } + lines.push(""); + } + + lines.push("## Docker daemon errors (raw)"); + lines.push("```"); + lines.push(daemonErrorsText); + lines.push("```"); + + return lines.join("\n"); + }; + + const handleDownload = () => { + const report = buildHealthReport(); + if (!report) return; + const blob = new Blob([report], { type: "text/markdown" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `dokploy-health-report-${new Date().toISOString().replace(/[:.]/g, "-")}.md`; + a.click(); + URL.revokeObjectURL(url); + }; + + return ( +
+ +
+
+
+

Server diagnostics

+

+ Runs a read-only check over SSH (inotify limits, disk, Docker + network pool, daemon errors). Runs automatically when you open + this tab — click Re-check to refresh. +

+
+
+ + + {health && !health.error && ( + + )} +
+
+ + {isFetching && !health && ( +
+ + Checking server health… +
+ )} + + {health?.error && ( +
+ Couldn't read server health: {health.error} +
+ )} + + {health && !health.error && ( +
+ +
+ + Containers & services +
+
+ {health.containers.containerCount} +
+
+ containers · {health.containers.serviceCount} swarm services +
+ + + +
+ + Host resources +
+
+ Memory: {bytesToGb(health.resources.memUsedBytes)} /{" "} + {bytesToGb(health.resources.memTotalBytes)} GB +
+ +
+ {health.resources.cpuCount} CPU cores +
+ + + +
+ + Disk (/) +
+
+ {bytesToGb(health.disk.usedBytes)} /{" "} + {bytesToGb(health.disk.totalBytes)} GB +
+ + + + +
+ + + Inotify + + + {health.inotify.persisted ? "persisted" : "runtime only"} + +
+
+ max_user_instances: {health.inotify.currentInstances} /{" "} + {health.inotify.maxInstances.toLocaleString()} ( + {inotifyUsedPct}%) +
+ +
+ max_user_watches: {health.inotify.maxWatches.toLocaleString()}{" "} + · max_queued_events:{" "} + {health.inotify.maxQueuedEvents.toLocaleString()} +
+ + + +
+ + Docker networks +
+
+ {health.dockerNetworks.count} +
+
+ The default Docker address pool is exhausted around ~30 + networks unless default-address-pools is + configured in /etc/docker/daemon.json. + {health.dockerNetworks.addressPools + ? " Custom pool detected on this server." + : ""} +
+ + + {health.reservation && ( + +
+ + Memory/CPU reservation +
+
+ {bytesToGb(health.reservation.memoryReservedBytes)} GB + reserved ·{" "} + {nanoCpusToCores(health.reservation.cpuReservedNanoCpus)}{" "} + CPUs reserved +
+
+ Across {health.reservation.appCount} Application services. + {health.reservation.unsupportedComposeCount > 0 + ? ` ${health.reservation.unsupportedComposeCount} Compose service(s) on this server aren't included — reservations aren't tracked per-service for Compose.` + : ""} +
+ + )} +
+ )} + + {health && + !health.error && + (health.dockerNetworks.usage.length > 0 || + health.dockerNetworks.usageError) && ( +
+

+ + Network IP usage +

+

+ IPs assigned to containers vs. the subnet's usable capacity, + per network — including reserved networks like{" "} + dokploy-network. A network stuck near 100% blocks + new containers from starting even when the host has plenty of + other resources free. +

+ {health.dockerNetworks.usageError ? ( +
+ Couldn't read network IP usage:{" "} + {health.dockerNetworks.usageError} +
+ ) : ( +
+ + + + Network + Driver + Subnet + IPs in use + Usage + + + + {health.dockerNetworks.usage.map((n) => ( + + + {n.name} + + + {n.driver} + + + {n.subnet ?? "Auto"} + + + {n.capacity !== null + ? `${n.containersInUse} / ${n.capacity}` + : n.containersInUse} + + + {n.percentUsed !== null ? ( + = 90 + ? "red" + : n.percentUsed >= 70 + ? "yellow" + : "green" + } + > + {n.percentUsed}% + + ) : ( + + — + + )} + + + ))} + +
+
+ )} +
+ )} + + {health && !health.error && ( +
+

+ Docker daemon errors ( + {SINCE_HOURS_OPTIONS.find( + (o) => o.value === sinceHours, + )?.label.toLowerCase()} + ) +

+ +
+ )} +
+ +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx b/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx new file mode 100644 index 000000000..50daf8a4e --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx @@ -0,0 +1,28 @@ +import { Stethoscope } from "lucide-react"; +import { useState } from "react"; +import { ShowHealth } from "@/components/dashboard/docker/health/show-health"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogTrigger } from "@/components/ui/dialog"; + +interface Props { + serverId: string; +} + +export const ShowHealthModal = ({ serverId }: Props) => { + const [isOpen, setIsOpen] = useState(false); + + return ( + + + + + +
+ +
+ + + ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx index 1fade3f5a..93f4d977e 100644 --- a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx @@ -35,6 +35,7 @@ import { TerminalModal } from "../web-server/terminal-modal"; import { ShowServerActions } from "./actions/show-server-actions"; import { HandleServers } from "./handle-servers"; import { SetupServer } from "./setup-server"; +import { ShowHealthModal } from "./show-health-modal"; import { ShowMonitoringModal } from "./show-monitoring-modal"; import { WelcomeSubscription } from "./welcome-stripe/welcome-subscription"; @@ -324,6 +325,24 @@ export const ShowServers = () => { )} + {permissions?.docker.read && + permissions?.server.read && + server.sshKeyId && + !isBuildServer && ( + + +
+ +
+ + +

Health

+ + + )} +
{permissions?.server.delete && ( diff --git a/apps/dokploy/pages/dashboard/docker.tsx b/apps/dokploy/pages/dashboard/docker.tsx index c73627c77..27a54373a 100644 --- a/apps/dokploy/pages/dashboard/docker.tsx +++ b/apps/dokploy/pages/dashboard/docker.tsx @@ -6,6 +6,7 @@ import type { ReactElement } from "react"; import superjson from "superjson"; import { ShowDiskUsage } from "@/components/dashboard/docker/disk-usage/show-disk-usage"; import { ShowDockerEvents } from "@/components/dashboard/docker/events/show-docker-events"; +import { ShowHealth } from "@/components/dashboard/docker/health/show-health"; import { ShowImages } from "@/components/dashboard/docker/images/show-images"; import { ShowContainers } from "@/components/dashboard/docker/show/show-containers"; import { ShowVolumes } from "@/components/dashboard/docker/volumes/show-volumes"; @@ -18,15 +19,21 @@ import { ServerFilter } from "@/components/shared/server-filter"; import { Card } from "@/components/ui/card"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { appRouter } from "@/server/api/root"; +import { api } from "@/utils/api"; const DEFAULT_TAB = "containers"; const DEFAULT_SWARM_TAB = "overview"; const Dashboard = () => { const router = useRouter(); + const { data: permissions } = api.user.getPermissions.useQuery(); + // Host-level diagnostics, so this needs server.read on top of docker.read. + const canSeeHealth = !!permissions?.docker.read && !!permissions?.server.read; - const activeTab = + const queryTab = typeof router.query.tab === "string" ? router.query.tab : DEFAULT_TAB; + const activeTab = + queryTab === "health" && !canSeeHealth ? DEFAULT_TAB : queryTab; const activeSwarmTab = typeof router.query.subtab === "string" @@ -70,6 +77,7 @@ const Dashboard = () => { Networks Events Disk Usage + {canSeeHealth && Health} @@ -111,6 +119,11 @@ const Dashboard = () => { + {canSeeHealth && ( + + + + )} )} diff --git a/apps/dokploy/server/api/routers/docker.ts b/apps/dokploy/server/api/routers/docker.ts index e025b3434..010b7723f 100644 --- a/apps/dokploy/server/api/routers/docker.ts +++ b/apps/dokploy/server/api/routers/docker.ts @@ -11,6 +11,7 @@ import { getContainersByAppLabel, getContainersByAppNameMatch, getDockerEvents, + getServerHealth as getServerHealthData, getServiceContainersByAppName, getStackContainersByAppName, listContainerFiles, @@ -18,11 +19,12 @@ import { uploadFileToContainer, writeContainerFile, } from "@dokploy/server"; +import { checkPermission } from "@dokploy/server/services/permission"; import { TRPCError } from "@trpc/server"; import { z } from "zod"; import { audit } from "@/server/api/utils/audit"; import { uploadFileToContainerSchema } from "@/utils/schema"; -import { createTRPCRouter, withPermission } from "../trpc"; +import { createTRPCRouter, protectedProcedure, withPermission } from "../trpc"; export const containerIdRegex = /^[a-zA-Z0-9.\-_]+$/; @@ -52,6 +54,29 @@ export const dockerRouter = createTRPCRouter({ return await getContainers(input.serverId); }), + // Host-level diagnostics, so this requires docker.read AND server.read. + getServerHealth: protectedProcedure + .input( + z.object({ + serverId: z.string().optional(), + sinceHours: z.number().int().min(1).max(168).optional(), + }), + ) + .query(async ({ input, ctx }) => { + await checkPermission(ctx, { docker: ["read"], server: ["read"] }); + if (input.serverId) { + const server = await findServerById(input.serverId); + if (server.organizationId !== ctx.session?.activeOrganizationId) { + throw new TRPCError({ code: "UNAUTHORIZED" }); + } + } + return await getServerHealthData( + ctx.session.activeOrganizationId, + input.serverId, + input.sinceHours, + ); + }), + restartContainer: withPermission("docker", "read") .input( z.object({ diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index b2a0c7f12..d0f843204 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -50,6 +50,7 @@ export * from "./services/rollbacks"; export * from "./services/schedule"; export * from "./services/security"; export * from "./services/server"; +export * from "./services/server-health"; export * from "./services/settings"; export * from "./services/ssh-key"; export * from "./services/user"; diff --git a/packages/server/src/services/server-health.ts b/packages/server/src/services/server-health.ts new file mode 100644 index 000000000..a7d5a21a2 --- /dev/null +++ b/packages/server/src/services/server-health.ts @@ -0,0 +1,385 @@ +import { db } from "@dokploy/server/db"; +import { + applications, + compose, + environments, + projects, +} from "@dokploy/server/db/schema"; +import { + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import { TRPCError } from "@trpc/server"; +import { and, eq, isNull } from "drizzle-orm"; +import { IS_CLOUD } from "../constants"; +import { getRemoteDocker } from "../utils/servers/remote-docker"; + +export interface NetworkIpUsage { + name: string; + driver: string; + subnet: string | null; + containersInUse: number; + capacity: number | null; + percentUsed: number | null; +} + +export interface ServerHealthResult { + checkedAt: string; + containers: { + containerCount: number; + serviceCount: number; + }; + resources: { + memTotalBytes: number; + memUsedBytes: number; + cpuCount: number; + }; + inotify: { + maxWatches: number; + maxInstances: number; + maxQueuedEvents: number; + currentInstances: number; + persisted: boolean; + }; + disk: { + totalBytes: number; + usedBytes: number; + }; + dockerNetworks: { + count: number; + addressPools: unknown | null; + /** Per-network IP utilization, including reserved networks like dokploy-network */ + usage: NetworkIpUsage[]; + /** Set when `usage` couldn't be read, so the UI can tell that apart from "no networks" */ + usageError?: string; + }; + daemonErrors: string[]; + /** The window actually passed to `journalctl --since`, computed on the target machine's clock */ + daemonLogsWindow: { fromEpoch: number; toEpoch: number } | null; + reservation: { + memoryReservedBytes: number; + cpuReservedNanoCpus: number; + appCount: number; + unsupportedComposeCount: number; + } | null; + error?: string; +} + +interface RawHealthOutput { + containerCount?: number | string; + serviceCount?: number | string; + memTotalBytes?: number | string; + memUsedBytes?: number | string; + cpuCount?: number | string; + inotifyMaxWatches?: number | string; + inotifyMaxInstances?: number | string; + inotifyMaxQueuedEvents?: number | string; + inotifyCurrentInstances?: number | string; + inotifyPersistedCount?: number | string; + diskTotalBytes?: number | string; + diskUsedBytes?: number | string; + networkCount?: number | string; + daemonConfigBase64?: string; + daemonErrorsBase64?: string; + daemonLogsFromEpoch?: number | string; + daemonLogsToEpoch?: number | string; +} + +const toInt = (value: unknown): number => { + const n = Number.parseInt(String(value ?? "0"), 10); + return Number.isFinite(n) ? n : 0; +}; + +const b64Decode = (value?: string): string => { + if (!value) return ""; + try { + return Buffer.from(value, "base64").toString("utf-8"); + } catch { + return ""; + } +}; + +// Single read-only script, one SSH/exec round-trip; each check falls back to 0/empty on its own. +const buildHealthScript = (sinceHours: number) => ` +containerCount=$(docker ps -a -q 2>/dev/null | wc -l | tr -d ' ') +serviceCount=$(docker service ls --format '{{.Name}}' 2>/dev/null | wc -l | tr -d ' ') + +memTotal=$(free -b 2>/dev/null | awk '/^Mem:/{print $2}'); [ -z "$memTotal" ] && memTotal=0 +memUsed=$(free -b 2>/dev/null | awk '/^Mem:/{print $3}'); [ -z "$memUsed" ] && memUsed=0 +cpuCount=$(nproc 2>/dev/null); [ -z "$cpuCount" ] && cpuCount=0 + +inotifyMaxWatches=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null); [ -z "$inotifyMaxWatches" ] && inotifyMaxWatches=0 +inotifyMaxInstances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null); [ -z "$inotifyMaxInstances" ] && inotifyMaxInstances=0 +inotifyMaxQueued=$(cat /proc/sys/fs/inotify/max_queued_events 2>/dev/null); [ -z "$inotifyMaxQueued" ] && inotifyMaxQueued=0 +inotifyCurrentInstances=$(find /proc/*/fd -lname 'anon_inode:inotify*' 2>/dev/null | wc -l | tr -d ' ') +inotifyPersistedCount=$(grep -rl inotify /etc/sysctl.conf /etc/sysctl.d/ 2>/dev/null | wc -l | tr -d ' ') + +diskTotal=$(df -B1 / 2>/dev/null | awk 'NR==2{print $2}'); [ -z "$diskTotal" ] && diskTotal=0 +diskUsed=$(df -B1 / 2>/dev/null | awk 'NR==2{print $3}'); [ -z "$diskUsed" ] && diskUsed=0 + +networkCount=$(docker network ls -q 2>/dev/null | wc -l | tr -d ' ') +daemonConfigB64=$(cat /etc/docker/daemon.json 2>/dev/null | base64 2>/dev/null | tr -d '\\n') + +daemonLogsToEpoch=$(date +%s 2>/dev/null); [ -z "$daemonLogsToEpoch" ] && daemonLogsToEpoch=0 +daemonLogsFromEpoch=$((daemonLogsToEpoch - ${sinceHours} * 3600)) + +daemonErrorsB64=$(journalctl -u docker --no-pager --since "${sinceHours} hours ago" 2>/dev/null | grep -iE "inotify|too many open|cannot allocate|oom|conntrack|no space|pids.max|fork:|resource temporarily unavailable|could not find an available ip|no available ip|task allocation failure|address already in use" | tail -n 50 | base64 2>/dev/null | tr -d '\\n') + +printf '{"containerCount":%s,"serviceCount":%s,"memTotalBytes":%s,"memUsedBytes":%s,"cpuCount":%s,"inotifyMaxWatches":%s,"inotifyMaxInstances":%s,"inotifyMaxQueuedEvents":%s,"inotifyCurrentInstances":%s,"inotifyPersistedCount":%s,"diskTotalBytes":%s,"diskUsedBytes":%s,"networkCount":%s,"daemonConfigBase64":"%s","daemonErrorsBase64":"%s","daemonLogsFromEpoch":%s,"daemonLogsToEpoch":%s}' "$containerCount" "$serviceCount" "$memTotal" "$memUsed" "$cpuCount" "$inotifyMaxWatches" "$inotifyMaxInstances" "$inotifyMaxQueued" "$inotifyCurrentInstances" "$inotifyPersistedCount" "$diskTotal" "$diskUsed" "$networkCount" "$daemonConfigB64" "$daemonErrorsB64" "$daemonLogsFromEpoch" "$daemonLogsToEpoch" +`; + +const emptyResult = (error: unknown): ServerHealthResult => ({ + checkedAt: new Date().toISOString(), + containers: { containerCount: 0, serviceCount: 0 }, + resources: { memTotalBytes: 0, memUsedBytes: 0, cpuCount: 0 }, + inotify: { + maxWatches: 0, + maxInstances: 0, + maxQueuedEvents: 0, + currentInstances: 0, + persisted: false, + }, + disk: { totalBytes: 0, usedBytes: 0 }, + dockerNetworks: { count: 0, addressPools: null, usage: [] }, + daemonErrors: [], + daemonLogsWindow: null, + reservation: null, + error: + error instanceof Error ? error.message : "Could not read server health", +}); + +/** Usable host addresses in a subnet, excluding network + broadcast. Null if the subnet is missing/invalid. */ +export const getSubnetCapacity = ( + subnet: string | undefined, +): number | null => { + if (!subnet) return null; + const prefix = Number.parseInt(subnet.split("/")[1] ?? "", 10); + if (Number.isNaN(prefix) || prefix < 0 || prefix > 32) return null; + const hostBits = 32 - prefix; + if (hostBits <= 1) return 0; + return 2 ** hostBits - 2; +}; + +type DockerNetworkSummary = { + Name: string; + Driver: string; + IPAM?: { Config?: Array<{ Subnet?: string }> | null }; +}; + +type InspectableNetwork = { + inspect: (opts?: { verbose?: boolean }) => Promise<{ + Containers?: Record; + }>; +}; + +// Includes reserved networks like dokploy-network, which are excluded from the `network` table/UI. +const getNetworksIpUsage = async ( + serverId: string | undefined, +): Promise => { + const docker = await getRemoteDocker(serverId); + const dockerNetworks = + (await docker.listNetworks()) as DockerNetworkSummary[]; + const relevant = dockerNetworks.filter( + (n) => n.Driver === "bridge" || n.Driver === "overlay", + ); + + const rows = await Promise.all( + relevant.map(async (n): Promise => { + // IPv4 entry specifically: dual-stack networks list an IPv6 config too, and it may come first. + const subnet = n.IPAM?.Config?.find( + (c) => c.Subnet && !c.Subnet.includes(":"), + )?.Subnet; + const capacity = getSubnetCapacity(subnet); + let containersInUse = 0; + try { + // Cast the object (not the detached method) so `net.inspect(...)` stays a bound method call. + const net = docker.getNetwork(n.Name) as unknown as InspectableNetwork; + // verbose: true for full swarm-wide container list on overlay networks + const info = await net.inspect( + n.Driver === "overlay" ? { verbose: true } : undefined, + ); + containersInUse = Object.keys(info.Containers ?? {}).length; + } catch { + // Network may have disappeared between list and inspect; report 0 + } + return { + name: n.Name, + driver: n.Driver, + subnet: subnet ?? null, + containersInUse, + capacity, + percentUsed: + capacity && capacity > 0 + ? Math.round((containersInUse / capacity) * 100) + : null, + }; + }), + ); + + return rows.sort((a, b) => (b.percentUsed ?? -1) - (a.percentUsed ?? -1)); +}; + +// Sums Application memoryReservation/cpuReservation; Compose has no such columns. +export const getReservationSummary = async ( + orgId: string, + serverId?: string, +) => { + const appRows = await db + .select({ + memoryReservation: applications.memoryReservation, + cpuReservation: applications.cpuReservation, + }) + .from(applications) + .innerJoin( + environments, + eq(applications.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where( + and( + eq(projects.organizationId, orgId), + serverId + ? eq(applications.serverId, serverId) + : isNull(applications.serverId), + ), + ); + + let memoryReservedBytes = 0; + let cpuReservedNanoCpus = 0; + for (const row of appRows) { + if (row.memoryReservation) { + memoryReservedBytes += toInt(row.memoryReservation); + } + if (row.cpuReservation) { + cpuReservedNanoCpus += toInt(row.cpuReservation); + } + } + + const composeRows = await db + .select({ composeId: compose.composeId }) + .from(compose) + .innerJoin( + environments, + eq(compose.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where( + and( + eq(projects.organizationId, orgId), + serverId ? eq(compose.serverId, serverId) : isNull(compose.serverId), + ), + ); + + return { + memoryReservedBytes, + cpuReservedNanoCpus, + appCount: appRows.length, + unsupportedComposeCount: composeRows.length, + }; +}; + +export const getServerHealth = async ( + orgId: string, + serverId?: string, + sinceHours = 24, +): Promise => { + if (IS_CLOUD && !serverId) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Server is required", + }); + } + + const hours = Math.max( + 1, + Math.min(168, Math.floor(Number(sinceHours) || 24)), + ); + const script = buildHealthScript(hours); + + let stdout = ""; + try { + const result = serverId + ? await execAsyncRemote(serverId, script) + : await execAsync(script); + stdout = result.stdout; + } catch (error) { + return emptyResult(error); + } + + let parsed: RawHealthOutput; + try { + parsed = JSON.parse(stdout.trim()); + } catch { + return emptyResult(new Error("Could not parse server health output")); + } + + const daemonErrors = b64Decode(parsed.daemonErrorsBase64) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + + let addressPools: unknown = null; + const daemonConfigText = b64Decode(parsed.daemonConfigBase64); + if (daemonConfigText) { + try { + addressPools = + JSON.parse(daemonConfigText)?.["default-address-pools"] ?? null; + } catch { + addressPools = null; + } + } + + // Not part of the shell script above: needs structured per-network data from the Docker API. + const [reservation, networkUsageResult] = await Promise.all([ + getReservationSummary(orgId, serverId).catch(() => null), + getNetworksIpUsage(serverId).then( + (usage) => ({ usage, error: undefined }), + (error: unknown) => ({ + usage: [] as NetworkIpUsage[], + error: + error instanceof Error + ? error.message + : "Failed to read network usage", + }), + ), + ]); + + return { + checkedAt: new Date().toISOString(), + containers: { + containerCount: toInt(parsed.containerCount), + serviceCount: toInt(parsed.serviceCount), + }, + resources: { + memTotalBytes: toInt(parsed.memTotalBytes), + memUsedBytes: toInt(parsed.memUsedBytes), + cpuCount: toInt(parsed.cpuCount), + }, + inotify: { + maxWatches: toInt(parsed.inotifyMaxWatches), + maxInstances: toInt(parsed.inotifyMaxInstances), + maxQueuedEvents: toInt(parsed.inotifyMaxQueuedEvents), + currentInstances: toInt(parsed.inotifyCurrentInstances), + persisted: toInt(parsed.inotifyPersistedCount) > 0, + }, + disk: { + totalBytes: toInt(parsed.diskTotalBytes), + usedBytes: toInt(parsed.diskUsedBytes), + }, + dockerNetworks: { + count: toInt(parsed.networkCount), + addressPools, + usage: networkUsageResult.usage, + usageError: networkUsageResult.error, + }, + daemonErrors, + daemonLogsWindow: + toInt(parsed.daemonLogsToEpoch) > 0 + ? { + fromEpoch: toInt(parsed.daemonLogsFromEpoch), + toEpoch: toInt(parsed.daemonLogsToEpoch), + } + : null, + reservation, + }; +};