diff --git a/apps/dokploy/__test__/server/server-health-subnet.test.ts b/apps/dokploy/__test__/server/server-health-subnet.test.ts new file mode 100644 index 000000000..9d0f7f120 --- /dev/null +++ b/apps/dokploy/__test__/server/server-health-subnet.test.ts @@ -0,0 +1,24 @@ +import { getSubnetCapacity } from "@dokploy/server/services/server-health"; +import { describe, expect, test } from "vitest"; + +describe("getSubnetCapacity", () => { + test("returns null for missing/invalid input", () => { + expect(getSubnetCapacity(undefined)).toBeNull(); + expect(getSubnetCapacity("")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0")).toBeNull(); + expect(getSubnetCapacity("not-a-subnet")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0/33")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0/-1")).toBeNull(); + }); + + test("excludes network and broadcast addresses", () => { + expect(getSubnetCapacity("10.0.1.0/24")).toBe(254); + expect(getSubnetCapacity("10.0.0.0/16")).toBe(65534); + expect(getSubnetCapacity("10.99.99.0/30")).toBe(2); + }); + + test("returns 0 for subnets too small to hold a host", () => { + expect(getSubnetCapacity("10.0.0.0/31")).toBe(0); + expect(getSubnetCapacity("10.0.0.0/32")).toBe(0); + }); +}); diff --git a/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts b/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts new file mode 100644 index 000000000..e1272d9b1 --- /dev/null +++ b/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts @@ -0,0 +1,55 @@ +import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; +import { describe, expect, test } from "vitest"; + +describe("classifyFailure", () => { + test("returns null for empty/null/undefined input", () => { + expect(classifyFailure(null)).toBeNull(); + expect(classifyFailure(undefined)).toBeNull(); + expect(classifyFailure("")).toBeNull(); + }); + + test("returns null when nothing matches", () => { + expect(classifyFailure("Build succeeded, deploying now")).toBeNull(); + }); + + test.each([ + [ + "inotify-limit", + "Error: ENOSPC: System limit for number of file watchers reached, inotify watch limit reached", + ], + [ + "oom", + "Container killed process, out of memory: Kill process 1234 (node) score", + ], + [ + "address-pool-exhausted", + "Error response from daemon: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network", + ], + [ + "network-ip-exhausted", + 'time="2026-08-12T19:10:14Z" level=error msg="task allocation failure" error="failed to allocate network IP for task kow0rt7j4nk166qill9vgb6gw network cuz8knvfyobvqsp10yvh39iu7: could not find an available IP"', + ], + [ + "network-attach-timeout", + "Error response from daemon: failed to set up container networking: attaching to network failed, make sure your network options are correct and check manager logs: context deadline exceeded", + ], + ["disk-full", "write /var/lib/docker/tmp/x: no space left on device"], + ["conntrack-full", "nf_conntrack: table full, dropping packet"], + ["fd-limit", "accept4: too many open files"], + ["pid-limit", "fork: retry: Resource temporarily unavailable"], + ])("classifies %s pattern", (expectedId, logLine) => { + expect(classifyFailure(logLine)?.id).toBe(expectedId); + }); + + test("network-ip-exhausted and address-pool-exhausted stay distinct", () => { + const poolExhausted = + "could not find an available, non-overlapping IPv4 address pool"; + const ipExhausted = "could not find an available IP"; + expect(classifyFailure(poolExhausted)?.id).toBe("address-pool-exhausted"); + expect(classifyFailure(ipExhausted)?.id).toBe("network-ip-exhausted"); + }); + + test("matches case-insensitively", () => { + expect(classifyFailure("NO SPACE LEFT ON DEVICE")?.id).toBe("disk-full"); + }); +}); diff --git a/apps/dokploy/components/dashboard/docker/health/show-health.tsx b/apps/dokploy/components/dashboard/docker/health/show-health.tsx new file mode 100644 index 000000000..6d2a5f3c3 --- /dev/null +++ b/apps/dokploy/components/dashboard/docker/health/show-health.tsx @@ -0,0 +1,719 @@ +import { format } from "date-fns"; +import { + AlertTriangle, + Calendar as CalendarIcon, + Cpu, + Download, + HardDrive, + Info, + Loader2, + Network, + RefreshCw, + Server as ServerIcon, +} from "lucide-react"; +import Link from "next/link"; +import { useEffect, useState } from "react"; +import { CodeEditor } from "@/components/shared/code-editor"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Calendar } from "@/components/ui/calendar"; +import { Card } from "@/components/ui/card"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { Progress } from "@/components/ui/progress"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { api } from "@/utils/api"; + +interface Props { + serverId?: string; +} + +const bytesToGb = (bytes: number) => (bytes / 1024 ** 3).toFixed(1); +const nanoCpusToCores = (nanoCpus: number) => (nanoCpus / 1e9).toFixed(2); +const pct = (used: number, total: number) => + Math.min(100, Math.round((used / (total || 1)) * 100)); + +const SINCE_HOURS_OPTIONS = [ + { label: "Last hour", value: 1 }, + { label: "Last 6 hours", value: 6 }, + { label: "Last 24 hours", value: 24 }, + { label: "Last 7 days", value: 168 }, +]; + +const getDefaultDateRange = () => { + const to = new Date(); + const from = new Date(); + from.setDate(from.getDate() - 7); + return { from, to }; +}; + +const getServiceInfo = (deployment: { + application?: { + applicationId: string; + name: string; + environment?: { + environmentId: string; + project?: { projectId: string; name: string }; + }; + } | null; + compose?: { + composeId: string; + name: string; + environment?: { + environmentId: string; + project?: { projectId: string; name: string }; + }; + } | null; +}) => { + const app = deployment.application; + const comp = deployment.compose; + if (app?.environment?.project) { + return { + name: app.name, + projectId: app.environment.project.projectId, + projectName: app.environment.project.name, + href: `/dashboard/project/${app.environment.project.projectId}/environment/${app.environment.environmentId}/services/application/${app.applicationId}`, + }; + } + if (comp?.environment?.project) { + return { + name: comp.name, + projectId: comp.environment.project.projectId, + projectName: comp.environment.project.name, + href: `/dashboard/project/${comp.environment.project.projectId}/environment/${comp.environment.environmentId}/services/compose/${comp.composeId}`, + }; + } + return null; +}; + +export const ShowHealth = ({ serverId }: Props) => { + const [sinceHours, setSinceHours] = useState(24); + const { + data: health, + isFetching, + refetch, + isFetched, + } = api.docker.getServerHealth.useQuery( + { serverId, sinceHours }, + { enabled: false }, + ); + + const [dateRange, setDateRange] = useState<{ + from: Date | undefined; + to: Date | undefined; + }>(() => getDefaultDateRange()); + const [hasCustomRange, setHasCustomRange] = useState(false); + const [projectId, setProjectId] = useState(undefined); + const { data: projects } = api.project.all.useQuery(); + + const { + data: failedDeployments, + isFetching: isFetchingFailed, + refetch: refetchFailed, + isFetched: isFetchedFailed, + } = api.deployment.allFailedCentralized.useQuery( + { + dateFrom: dateRange.from?.toISOString(), + dateTo: dateRange.to?.toISOString(), + projectId, + }, + { enabled: false }, + ); + + // Mount-only: filter changes require an explicit Check click, not an auto-refetch. + useEffect(() => { + refetch(); + refetchFailed(); + }, []); + + // Refreshes the default range's "to" bound to now right before checking. + const [checkFailedTick, setCheckFailedTick] = useState(0); + useEffect(() => { + if (checkFailedTick > 0) refetchFailed(); + }, [checkFailedTick]); + const handleCheckFailed = () => { + if (!hasCustomRange) setDateRange(getDefaultDateRange()); + setCheckFailedTick((t) => t + 1); + }; + + const memUsedPct = health + ? pct(health.resources.memUsedBytes, health.resources.memTotalBytes) + : 0; + const diskUsedPct = health + ? pct(health.disk.usedBytes, health.disk.totalBytes) + : 0; + const inotifyUsedPct = health + ? pct(health.inotify.currentInstances, health.inotify.maxInstances) + : 0; + + const daemonLogsWindowText = health?.daemonLogsWindow + ? `Logs from ${new Date(health.daemonLogsWindow.fromEpoch * 1000).toLocaleString()} to ${new Date(health.daemonLogsWindow.toEpoch * 1000).toLocaleString()}` + : null; + const daemonErrorsText = [ + daemonLogsWindowText, + health && health.daemonErrors.length > 0 + ? health.daemonErrors.join("\n") + : "(no daemon errors matched in this window)", + ] + .filter(Boolean) + .join("\n\n"); + + const buildHealthReport = () => { + if (!health) return ""; + const lines: string[] = []; + lines.push("# Dokploy server health report"); + lines.push(`Generated: ${new Date(health.checkedAt).toLocaleString()}`); + lines.push( + `Window: ${SINCE_HOURS_OPTIONS.find((o) => o.value === sinceHours)?.label ?? `${sinceHours}h`}`, + ); + lines.push(""); + + lines.push("## Containers & services"); + lines.push(`Containers: ${health.containers.containerCount}`); + lines.push(`Swarm services: ${health.containers.serviceCount}`); + lines.push(""); + + lines.push("## Host resources"); + lines.push( + `Memory: ${bytesToGb(health.resources.memUsedBytes)} / ${bytesToGb(health.resources.memTotalBytes)} GB`, + ); + lines.push(`CPU cores: ${health.resources.cpuCount}`); + lines.push(""); + + lines.push("## Disk (/)"); + lines.push( + `${bytesToGb(health.disk.usedBytes)} / ${bytesToGb(health.disk.totalBytes)} GB`, + ); + lines.push(""); + + lines.push("## Inotify"); + lines.push( + `max_user_instances: ${health.inotify.currentInstances} / ${health.inotify.maxInstances}`, + ); + lines.push(`max_user_watches: ${health.inotify.maxWatches}`); + lines.push(`max_queued_events: ${health.inotify.maxQueuedEvents}`); + lines.push( + `Persisted in sysctl: ${health.inotify.persisted ? "yes" : "no"}`, + ); + lines.push(""); + + lines.push("## Docker networks"); + lines.push(`Total networks: ${health.dockerNetworks.count}`); + if (health.dockerNetworks.addressPools) { + lines.push( + `default-address-pools: ${JSON.stringify(health.dockerNetworks.addressPools)}`, + ); + } + if (health.dockerNetworks.usageError) { + lines.push( + `Could not read network IP usage: ${health.dockerNetworks.usageError}`, + ); + } else if (health.dockerNetworks.usage.length > 0) { + lines.push(""); + lines.push( + "| Network | Driver | Subnet | IPs in use | Capacity | Usage |", + ); + lines.push("|---|---|---|---|---|---|"); + for (const n of health.dockerNetworks.usage) { + lines.push( + `| ${n.name} | ${n.driver} | ${n.subnet ?? "auto"} | ${n.containersInUse} | ${n.capacity ?? "—"} | ${n.percentUsed ?? "—"}% |`, + ); + } + } + lines.push(""); + + if (health.reservation) { + lines.push("## Memory/CPU reservation"); + lines.push( + `Reserved memory: ${bytesToGb(health.reservation.memoryReservedBytes)} GB`, + ); + lines.push( + `Reserved CPU: ${nanoCpusToCores(health.reservation.cpuReservedNanoCpus)} cores`, + ); + lines.push(`Across ${health.reservation.appCount} Application services`); + if (health.reservation.unsupportedComposeCount > 0) { + lines.push( + `${health.reservation.unsupportedComposeCount} Compose service(s) not included (not tracked per-service)`, + ); + } + lines.push(""); + } + + lines.push("## Failed deployments"); + if (failedDeployments && failedDeployments.length > 0) { + for (const d of failedDeployments) { + const info = getServiceInfo(d); + const cause = d.classification?.label ?? "Unclassified"; + lines.push( + `- ${format(new Date(d.createdAt), "PPp")} | ${info?.name ?? "—"} | ${info?.projectName ?? "—"} | ${cause}`, + ); + } + } else { + lines.push("(none checked, or none in range)"); + } + lines.push(""); + + lines.push("## Docker daemon errors (raw)"); + lines.push("```"); + lines.push(daemonErrorsText); + lines.push("```"); + + return lines.join("\n"); + }; + + const handleDownload = () => { + const report = buildHealthReport(); + if (!report) return; + const blob = new Blob([report], { type: "text/markdown" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `dokploy-health-report-${new Date().toISOString().replace(/[:.]/g, "-")}.md`; + a.click(); + URL.revokeObjectURL(url); + }; + + return ( +
+ +
+
+
+

Server diagnostics

+

+ Runs a read-only check over SSH (inotify limits, disk, Docker + network pool, daemon errors). Nothing runs automatically — click + to check. +

+
+
+ + + {health && !health.error && ( + + )} +
+
+ + {!isFetched && !isFetching && ( +
+ + + Nothing checked yet — click "Check server health" above. + +
+ )} + + {health?.error && ( +
+ Couldn't read server health: {health.error} +
+ )} + + {health && !health.error && ( +
+ +
+ + Containers & services +
+
+ {health.containers.containerCount} +
+
+ containers · {health.containers.serviceCount} swarm services +
+ + + +
+ + Host resources +
+
+ Memory: {bytesToGb(health.resources.memUsedBytes)} /{" "} + {bytesToGb(health.resources.memTotalBytes)} GB +
+ +
+ {health.resources.cpuCount} CPU cores +
+ + + +
+ + Disk (/) +
+
+ {bytesToGb(health.disk.usedBytes)} /{" "} + {bytesToGb(health.disk.totalBytes)} GB +
+ + + + +
+ + + Inotify + + + {health.inotify.persisted ? "persisted" : "runtime only"} + +
+
+ max_user_instances: {health.inotify.currentInstances} /{" "} + {health.inotify.maxInstances.toLocaleString()} ( + {inotifyUsedPct}%) +
+ +
+ max_user_watches: {health.inotify.maxWatches.toLocaleString()}{" "} + · max_queued_events:{" "} + {health.inotify.maxQueuedEvents.toLocaleString()} +
+ + + +
+ + Docker networks +
+
+ {health.dockerNetworks.count} +
+
+ The default Docker address pool is exhausted around ~30 + networks unless default-address-pools is + configured in /etc/docker/daemon.json. + {health.dockerNetworks.addressPools + ? " Custom pool detected on this server." + : ""} +
+ + + {health.reservation && ( + +
+ + Memory/CPU reservation +
+
+ {bytesToGb(health.reservation.memoryReservedBytes)} GB + reserved ·{" "} + {nanoCpusToCores(health.reservation.cpuReservedNanoCpus)}{" "} + CPUs reserved +
+
+ Across {health.reservation.appCount} Application services. + {health.reservation.unsupportedComposeCount > 0 + ? ` ${health.reservation.unsupportedComposeCount} Compose service(s) on this server aren't included — reservations aren't tracked per-service for Compose.` + : ""} +
+ + )} +
+ )} + + {health && + !health.error && + (health.dockerNetworks.usage.length > 0 || + health.dockerNetworks.usageError) && ( +
+

+ + Network IP usage +

+

+ IPs assigned to containers vs. the subnet's usable capacity, + per network — including reserved networks like{" "} + dokploy-network. A network stuck near 100% blocks + new containers from starting even when the host has plenty of + other resources free. +

+ {health.dockerNetworks.usageError ? ( +
+ Couldn't read network IP usage:{" "} + {health.dockerNetworks.usageError} +
+ ) : ( +
+ + + + Network + Driver + Subnet + IPs in use + Usage + + + + {health.dockerNetworks.usage.map((n) => ( + + + {n.name} + + + {n.driver} + + + {n.subnet ?? "Auto"} + + + {n.capacity !== null + ? `${n.containersInUse} / ${n.capacity}` + : n.containersInUse} + + + {n.percentUsed !== null ? ( + = 90 + ? "red" + : n.percentUsed >= 70 + ? "yellow" + : "green" + } + > + {n.percentUsed}% + + ) : ( + + — + + )} + + + ))} + +
+
+ )} +
+ )} + + {health && !health.error && ( +
+

+ Docker daemon errors ( + {SINCE_HOURS_OPTIONS.find( + (o) => o.value === sinceHours, + )?.label.toLowerCase()} + ) +

+ +
+ )} +
+ + + +
+
+
+

Failed deployments

+

+ Deployments that ended in error, with an automatic cause guess + based on the same patterns used to triage failures manually. +

+
+
+ + + + + + + { + setHasCustomRange(true); + setDateRange({ from: range?.from, to: range?.to }); + }} + numberOfMonths={2} + /> + + + +
+
+ + {!isFetchedFailed && !isFetchingFailed && ( +
+ Nothing checked yet. +
+ )} + + {isFetchedFailed && (failedDeployments?.length ?? 0) === 0 && ( +
+ No failed deployments in this range. +
+ )} + + {(failedDeployments?.length ?? 0) > 0 && ( + + + + Date + Service + Project + Likely cause + + + + {failedDeployments?.map((deployment) => { + const info = getServiceInfo(deployment); + return ( + + + {format(new Date(deployment.createdAt), "PPp")} + + + {info ? ( + + {info.name} + + ) : ( + "—" + )} + + + {info ? ( + + {info.projectName} + + ) : ( + "—" + )} + + + + {deployment.classification ? ( + + {deployment.classification.label} + + ) : ( + + Unclassified + + )} + {deployment.rawError && ( + + + + + + {deployment.rawError} + + + )} + + + + ); + })} + +
+ )} +
+ +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx b/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx new file mode 100644 index 000000000..50daf8a4e --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx @@ -0,0 +1,28 @@ +import { Stethoscope } from "lucide-react"; +import { useState } from "react"; +import { ShowHealth } from "@/components/dashboard/docker/health/show-health"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogTrigger } from "@/components/ui/dialog"; + +interface Props { + serverId: string; +} + +export const ShowHealthModal = ({ serverId }: Props) => { + const [isOpen, setIsOpen] = useState(false); + + return ( + + + + + +
+ +
+ + + ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx index 1fade3f5a..93f4d977e 100644 --- a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx @@ -35,6 +35,7 @@ import { TerminalModal } from "../web-server/terminal-modal"; import { ShowServerActions } from "./actions/show-server-actions"; import { HandleServers } from "./handle-servers"; import { SetupServer } from "./setup-server"; +import { ShowHealthModal } from "./show-health-modal"; import { ShowMonitoringModal } from "./show-monitoring-modal"; import { WelcomeSubscription } from "./welcome-stripe/welcome-subscription"; @@ -324,6 +325,24 @@ export const ShowServers = () => { )} + {permissions?.docker.read && + permissions?.server.read && + server.sshKeyId && + !isBuildServer && ( + + +
+ +
+ + +

Health

+ + + )} +
{permissions?.server.delete && ( diff --git a/apps/dokploy/pages/dashboard/docker.tsx b/apps/dokploy/pages/dashboard/docker.tsx index c73627c77..d2dbba70d 100644 --- a/apps/dokploy/pages/dashboard/docker.tsx +++ b/apps/dokploy/pages/dashboard/docker.tsx @@ -6,6 +6,7 @@ import type { ReactElement } from "react"; import superjson from "superjson"; import { ShowDiskUsage } from "@/components/dashboard/docker/disk-usage/show-disk-usage"; import { ShowDockerEvents } from "@/components/dashboard/docker/events/show-docker-events"; +import { ShowHealth } from "@/components/dashboard/docker/health/show-health"; import { ShowImages } from "@/components/dashboard/docker/images/show-images"; import { ShowContainers } from "@/components/dashboard/docker/show/show-containers"; import { ShowVolumes } from "@/components/dashboard/docker/volumes/show-volumes"; @@ -18,15 +19,25 @@ import { ServerFilter } from "@/components/shared/server-filter"; import { Card } from "@/components/ui/card"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { appRouter } from "@/server/api/root"; +import { api } from "@/utils/api"; const DEFAULT_TAB = "containers"; const DEFAULT_SWARM_TAB = "overview"; const Dashboard = () => { const router = useRouter(); + const { data: isCloud } = api.settings.isCloud.useQuery(); + const { data: permissions } = api.user.getPermissions.useQuery(); + // Host-level diagnostics, so this needs server.read on top of docker.read. + const canSeeHealth = !!permissions?.docker.read && !!permissions?.server.read; - const activeTab = + const queryTab = typeof router.query.tab === "string" ? router.query.tab : DEFAULT_TAB; + const activeTab = + (isCloud && queryTab === "networks") || + (queryTab === "health" && !canSeeHealth) + ? DEFAULT_TAB + : queryTab; const activeSwarmTab = typeof router.query.subtab === "string" @@ -67,9 +78,10 @@ const Dashboard = () => { Swarm Images Volumes - Networks + {!isCloud && Networks} Events Disk Usage + {canSeeHealth && Health} @@ -102,15 +114,22 @@ const Dashboard = () => { - - - + {!isCloud && ( + + + + )} + {canSeeHealth && ( + + + + )} )} diff --git a/apps/dokploy/server/api/routers/deployment.ts b/apps/dokploy/server/api/routers/deployment.ts index dcc74226a..8f19427cf 100644 --- a/apps/dokploy/server/api/routers/deployment.ts +++ b/apps/dokploy/server/api/routers/deployment.ts @@ -6,6 +6,7 @@ import { findAllDeploymentsByServerId, findAllDeploymentsCentralized, findDeploymentById, + findFailedDeploymentsCentralized, findScheduleById, IS_CLOUD, removeDeployment, @@ -14,6 +15,7 @@ import { } from "@dokploy/server"; import { db } from "@dokploy/server/db"; import { + checkPermission, checkServicePermissionAndAccess, findMemberByUserId, } from "@dokploy/server/services/permission"; @@ -27,6 +29,7 @@ import { apiFindAllByCompose, apiFindAllByServer, apiFindAllByType, + apiFindFailedDeployments, deployments, server, } from "@/server/db/schema"; @@ -78,6 +81,23 @@ export const deploymentRouter = createTRPCRouter({ }, ), + // Reads log tails over SSH, so same host-level gate as docker.getServerHealth. + allFailedCentralized: protectedProcedure + .input(apiFindFailedDeployments) + .query(async ({ ctx, input }) => { + await checkPermission(ctx, { + deployment: ["read"], + docker: ["read"], + server: ["read"], + }); + const orgId = ctx.session.activeOrganizationId; + const accessedServices = + ctx.user.role !== "owner" && ctx.user.role !== "admin" + ? (await findMemberByUserId(ctx.user.id, orgId)).accessedServices + : null; + return findFailedDeploymentsCentralized(orgId, accessedServices, input); + }), + queueList: withPermission("deployment", "read").query(async ({ ctx }) => { const orgId = ctx.session.activeOrganizationId; let rows: QueueJobRow[]; diff --git a/apps/dokploy/server/api/routers/docker.ts b/apps/dokploy/server/api/routers/docker.ts index e025b3434..010b7723f 100644 --- a/apps/dokploy/server/api/routers/docker.ts +++ b/apps/dokploy/server/api/routers/docker.ts @@ -11,6 +11,7 @@ import { getContainersByAppLabel, getContainersByAppNameMatch, getDockerEvents, + getServerHealth as getServerHealthData, getServiceContainersByAppName, getStackContainersByAppName, listContainerFiles, @@ -18,11 +19,12 @@ import { uploadFileToContainer, writeContainerFile, } from "@dokploy/server"; +import { checkPermission } from "@dokploy/server/services/permission"; import { TRPCError } from "@trpc/server"; import { z } from "zod"; import { audit } from "@/server/api/utils/audit"; import { uploadFileToContainerSchema } from "@/utils/schema"; -import { createTRPCRouter, withPermission } from "../trpc"; +import { createTRPCRouter, protectedProcedure, withPermission } from "../trpc"; export const containerIdRegex = /^[a-zA-Z0-9.\-_]+$/; @@ -52,6 +54,29 @@ export const dockerRouter = createTRPCRouter({ return await getContainers(input.serverId); }), + // Host-level diagnostics, so this requires docker.read AND server.read. + getServerHealth: protectedProcedure + .input( + z.object({ + serverId: z.string().optional(), + sinceHours: z.number().int().min(1).max(168).optional(), + }), + ) + .query(async ({ input, ctx }) => { + await checkPermission(ctx, { docker: ["read"], server: ["read"] }); + if (input.serverId) { + const server = await findServerById(input.serverId); + if (server.organizationId !== ctx.session?.activeOrganizationId) { + throw new TRPCError({ code: "UNAUTHORIZED" }); + } + } + return await getServerHealthData( + ctx.session.activeOrganizationId, + input.serverId, + input.sinceHours, + ); + }), + restartContainer: withPermission("docker", "read") .input( z.object({ diff --git a/packages/server/src/db/schema/deployment.ts b/packages/server/src/db/schema/deployment.ts index 0c1b547e0..c3aea8345 100644 --- a/packages/server/src/db/schema/deployment.ts +++ b/packages/server/src/db/schema/deployment.ts @@ -233,3 +233,10 @@ export const apiFindAllByType = z.object({ "volumeBackup", ]), }); + +export const apiFindFailedDeployments = z.object({ + dateFrom: z.string().datetime().optional(), + dateTo: z.string().datetime().optional(), + projectId: z.string().optional(), + environmentId: z.string().optional(), +}); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index b2a0c7f12..3c91bca59 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -50,6 +50,7 @@ export * from "./services/rollbacks"; export * from "./services/schedule"; export * from "./services/security"; export * from "./services/server"; +export * from "./services/server-health"; export * from "./services/settings"; export * from "./services/ssh-key"; export * from "./services/user"; @@ -95,6 +96,7 @@ export * from "./utils/builders/utils"; export * from "./utils/cluster/upload"; export * from "./utils/crons/enterprise"; export * from "./utils/databases/rebuild"; +export * from "./utils/deployment-failure-classifier"; export * from "./utils/docker/collision"; export * from "./utils/docker/compose"; export * from "./utils/docker/compose/configs"; diff --git a/packages/server/src/services/deployment.ts b/packages/server/src/services/deployment.ts index fd61fc106..3d5a5915d 100644 --- a/packages/server/src/services/deployment.ts +++ b/packages/server/src/services/deployment.ts @@ -10,12 +10,14 @@ import { type apiCreateDeploymentSchedule, type apiCreateDeploymentServer, type apiCreateDeploymentVolumeBackup, + type apiFindFailedDeployments, applications, compose, deployments, environments, projects, } from "@dokploy/server/db/schema"; +import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; import { removeDirectoryIfExistsContent } from "@dokploy/server/utils/filesystem/directory"; import { execAsync, @@ -23,7 +25,7 @@ import { } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { format } from "date-fns"; -import { and, desc, eq, inArray, or, sql } from "drizzle-orm"; +import { and, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm"; import type { z } from "zod"; import { type Application, @@ -942,6 +944,185 @@ export const findAllDeploymentsCentralized = async ( }); }; +// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg above to avoid +// touching that existing code path; adds projectId/environmentId filtering. +type FailedDeploymentIdFilters = { projectId?: string; environmentId?: string }; + +async function getApplicationIdsForFailedDeployments( + orgId: string, + accessedServices: string[] | null, + filters: FailedDeploymentIdFilters, +): Promise { + const conditions = [ + eq(projects.organizationId, orgId), + ...(accessedServices !== null + ? [inArray(applications.applicationId, accessedServices)] + : []), + ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), + ...(filters.environmentId + ? [eq(environments.environmentId, filters.environmentId)] + : []), + ]; + const rows = await db + .select({ applicationId: applications.applicationId }) + .from(applications) + .innerJoin( + environments, + eq(applications.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where(and(...conditions)); + return rows.map((r) => r.applicationId); +} + +async function getComposeIdsForFailedDeployments( + orgId: string, + accessedServices: string[] | null, + filters: FailedDeploymentIdFilters, +): Promise { + const conditions = [ + eq(projects.organizationId, orgId), + ...(accessedServices !== null + ? [inArray(compose.composeId, accessedServices)] + : []), + ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), + ...(filters.environmentId + ? [eq(environments.environmentId, filters.environmentId)] + : []), + ]; + const rows = await db + .select({ composeId: compose.composeId }) + .from(compose) + .innerJoin( + environments, + eq(compose.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where(and(...conditions)); + return rows.map((r) => r.composeId); +} + +const readLogTail = async ( + logPath: string | null, + serverId: string | null | undefined, +) => { + if (!logPath) return ""; + const command = `tail -c 4096 "${logPath}" 2>/dev/null || echo ""`; + const { stdout } = serverId + ? await execAsyncRemote(serverId, command) + : await execAsync(command); + return stdout; +}; + +// Caps simultaneous SSH connections when classifying a batch of failures. +const LOG_TAIL_CONCURRENCY = 8; + +async function mapWithConcurrency( + items: T[], + limit: number, + fn: (item: T) => Promise, +): Promise { + const results = new Array(items.length); + let nextIndex = 0; + const worker = async () => { + while (nextIndex < items.length) { + const current = nextIndex++; + results[current] = await fn(items[current] as T); + } + }; + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, worker), + ); + return results; +} + +export type FailedDeploymentFilters = z.infer; + +// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed +// deployments, each tagged with a classifyFailure() guess from its log tail. +export const findFailedDeploymentsCentralized = async ( + orgId: string, + accessedServices: string[] | null, + filters: FailedDeploymentFilters = {}, +) => { + if (accessedServices !== null && accessedServices.length === 0) { + return []; + } + + const idFilters: FailedDeploymentIdFilters = { + projectId: filters.projectId, + environmentId: filters.environmentId, + }; + + const [appIds, compIds] = await Promise.all([ + getApplicationIdsForFailedDeployments(orgId, accessedServices, idFilters), + getComposeIdsForFailedDeployments(orgId, accessedServices, idFilters), + ]); + + if (appIds.length === 0 && compIds.length === 0) { + return []; + } + + // createdAt is stored as text via toISOString(); normalize the filters the + // same way so the string range comparison below stays valid. + const dateFrom = filters.dateFrom + ? new Date(filters.dateFrom).toISOString() + : undefined; + const dateTo = filters.dateTo + ? new Date(filters.dateTo).toISOString() + : undefined; + + const serviceConditions = [ + ...(appIds.length > 0 ? [inArray(deployments.applicationId, appIds)] : []), + ...(compIds.length > 0 ? [inArray(deployments.composeId, compIds)] : []), + ]; + + const conditions = [ + eq(deployments.status, "error"), + serviceConditions.length === 1 + ? serviceConditions[0] + : or(...serviceConditions), + ...(dateFrom ? [gte(deployments.createdAt, dateFrom)] : []), + ...(dateTo ? [lte(deployments.createdAt, dateTo)] : []), + ]; + + const failed = await db.query.deployments.findMany({ + where: and(...conditions), + orderBy: desc(deployments.createdAt), + with: centralizedDeploymentsWith, + limit: 200, + }); + + // Skip further attempts to a server once it's timed out once, so a batch + // of failures on one unreachable host doesn't pay the SSH timeout N times. + const deadServers = new Set(); + return mapWithConcurrency( + failed, + LOG_TAIL_CONCURRENCY, + async (deployment) => { + const serverId = + deployment.buildServer?.serverId || + deployment.server?.serverId || + deployment.application?.server?.serverId || + deployment.compose?.server?.serverId || + null; + let logTail = ""; + if (!serverId || !deadServers.has(serverId)) { + try { + logTail = await readLogTail(deployment.logPath, serverId); + } catch { + if (serverId) deadServers.add(serverId); + } + } + const combined = `${deployment.errorMessage ?? ""}\n${logTail}`; + const classification = classifyFailure(combined); + // Tail end only: the actual failure line is almost always near the end of the log. + const rawError = combined.trim().slice(-1000) || null; + return { ...deployment, classification, rawError }; + }, + ); +}; + export const updateDeployment = async ( deploymentId: string, deploymentData: Partial, diff --git a/packages/server/src/services/server-health.ts b/packages/server/src/services/server-health.ts new file mode 100644 index 000000000..5da66a511 --- /dev/null +++ b/packages/server/src/services/server-health.ts @@ -0,0 +1,386 @@ +import { db } from "@dokploy/server/db"; +import { + applications, + compose, + environments, + projects, +} from "@dokploy/server/db/schema"; +import { + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import { TRPCError } from "@trpc/server"; +import { and, eq, isNull } from "drizzle-orm"; +import { IS_CLOUD } from "../constants"; +import { getRemoteDocker } from "../utils/servers/remote-docker"; + +export interface NetworkIpUsage { + name: string; + driver: string; + subnet: string | null; + containersInUse: number; + capacity: number | null; + percentUsed: number | null; +} + +export interface ServerHealthResult { + checkedAt: string; + containers: { + containerCount: number; + serviceCount: number; + }; + resources: { + memTotalBytes: number; + memUsedBytes: number; + cpuCount: number; + }; + inotify: { + maxWatches: number; + maxInstances: number; + maxQueuedEvents: number; + currentInstances: number; + persisted: boolean; + }; + disk: { + totalBytes: number; + usedBytes: number; + }; + dockerNetworks: { + count: number; + addressPools: unknown | null; + /** Per-network IP utilization, including reserved networks like dokploy-network */ + usage: NetworkIpUsage[]; + /** Set when `usage` couldn't be read, so the UI can tell that apart from "no networks" */ + usageError?: string; + }; + daemonErrors: string[]; + /** The window actually passed to `journalctl --since`, computed on the target machine's clock */ + daemonLogsWindow: { fromEpoch: number; toEpoch: number } | null; + reservation: { + memoryReservedBytes: number; + cpuReservedNanoCpus: number; + appCount: number; + unsupportedComposeCount: number; + } | null; + error?: string; +} + +interface RawHealthOutput { + containerCount?: number | string; + serviceCount?: number | string; + memTotalBytes?: number | string; + memUsedBytes?: number | string; + cpuCount?: number | string; + inotifyMaxWatches?: number | string; + inotifyMaxInstances?: number | string; + inotifyMaxQueuedEvents?: number | string; + inotifyCurrentInstances?: number | string; + inotifyPersistedCount?: number | string; + diskTotalBytes?: number | string; + diskUsedBytes?: number | string; + networkCount?: number | string; + daemonConfigBase64?: string; + daemonErrorsBase64?: string; + daemonLogsFromEpoch?: number | string; + daemonLogsToEpoch?: number | string; +} + +const toInt = (value: unknown): number => { + const n = Number.parseInt(String(value ?? "0"), 10); + return Number.isFinite(n) ? n : 0; +}; + +const b64Decode = (value?: string): string => { + if (!value) return ""; + try { + return Buffer.from(value, "base64").toString("utf-8"); + } catch { + return ""; + } +}; + +// Single read-only script, one SSH/exec round-trip; each check falls back to 0/empty on its own. +const buildHealthScript = (sinceHours: number) => ` +containerCount=$(docker ps -a -q 2>/dev/null | wc -l | tr -d ' ') +serviceCount=$(docker service ls --format '{{.Name}}' 2>/dev/null | wc -l | tr -d ' ') + +memTotal=$(free -b 2>/dev/null | awk '/^Mem:/{print $2}'); [ -z "$memTotal" ] && memTotal=0 +memUsed=$(free -b 2>/dev/null | awk '/^Mem:/{print $3}'); [ -z "$memUsed" ] && memUsed=0 +cpuCount=$(nproc 2>/dev/null); [ -z "$cpuCount" ] && cpuCount=0 + +inotifyMaxWatches=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null); [ -z "$inotifyMaxWatches" ] && inotifyMaxWatches=0 +inotifyMaxInstances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null); [ -z "$inotifyMaxInstances" ] && inotifyMaxInstances=0 +inotifyMaxQueued=$(cat /proc/sys/fs/inotify/max_queued_events 2>/dev/null); [ -z "$inotifyMaxQueued" ] && inotifyMaxQueued=0 +inotifyCurrentInstances=$(find /proc/*/fd -lname 'anon_inode:inotify*' 2>/dev/null | wc -l | tr -d ' ') +inotifyPersistedCount=$(grep -rl inotify /etc/sysctl.conf /etc/sysctl.d/ 2>/dev/null | wc -l | tr -d ' ') + +diskTotal=$(df -B1 / 2>/dev/null | awk 'NR==2{print $2}'); [ -z "$diskTotal" ] && diskTotal=0 +diskUsed=$(df -B1 / 2>/dev/null | awk 'NR==2{print $3}'); [ -z "$diskUsed" ] && diskUsed=0 + +networkCount=$(docker network ls -q 2>/dev/null | wc -l | tr -d ' ') +daemonConfigB64=$(cat /etc/docker/daemon.json 2>/dev/null | base64 2>/dev/null | tr -d '\\n') + +daemonLogsToEpoch=$(date +%s 2>/dev/null); [ -z "$daemonLogsToEpoch" ] && daemonLogsToEpoch=0 +daemonLogsFromEpoch=$((daemonLogsToEpoch - ${sinceHours} * 3600)) + +daemonErrorsB64=$(journalctl -u docker --no-pager --since "${sinceHours} hours ago" 2>/dev/null | grep -iE "inotify|too many open|cannot allocate|oom|conntrack|no space|pids.max|fork:|resource temporarily unavailable|could not find an available ip|no available ip|task allocation failure|address already in use" | tail -n 50 | base64 2>/dev/null | tr -d '\\n') + +printf '{"containerCount":%s,"serviceCount":%s,"memTotalBytes":%s,"memUsedBytes":%s,"cpuCount":%s,"inotifyMaxWatches":%s,"inotifyMaxInstances":%s,"inotifyMaxQueuedEvents":%s,"inotifyCurrentInstances":%s,"inotifyPersistedCount":%s,"diskTotalBytes":%s,"diskUsedBytes":%s,"networkCount":%s,"daemonConfigBase64":"%s","daemonErrorsBase64":"%s","daemonLogsFromEpoch":%s,"daemonLogsToEpoch":%s}' "$containerCount" "$serviceCount" "$memTotal" "$memUsed" "$cpuCount" "$inotifyMaxWatches" "$inotifyMaxInstances" "$inotifyMaxQueued" "$inotifyCurrentInstances" "$inotifyPersistedCount" "$diskTotal" "$diskUsed" "$networkCount" "$daemonConfigB64" "$daemonErrorsB64" "$daemonLogsFromEpoch" "$daemonLogsToEpoch" +`; + +const emptyResult = (error: unknown): ServerHealthResult => ({ + checkedAt: new Date().toISOString(), + containers: { containerCount: 0, serviceCount: 0 }, + resources: { memTotalBytes: 0, memUsedBytes: 0, cpuCount: 0 }, + inotify: { + maxWatches: 0, + maxInstances: 0, + maxQueuedEvents: 0, + currentInstances: 0, + persisted: false, + }, + disk: { totalBytes: 0, usedBytes: 0 }, + dockerNetworks: { count: 0, addressPools: null, usage: [] }, + daemonErrors: [], + daemonLogsWindow: null, + reservation: null, + error: + error instanceof Error ? error.message : "Could not read server health", +}); + +/** Usable host addresses in a subnet, excluding network + broadcast. Null if the subnet is missing/invalid. */ +export const getSubnetCapacity = ( + subnet: string | undefined, +): number | null => { + if (!subnet) return null; + const prefix = Number.parseInt(subnet.split("/")[1] ?? "", 10); + if (Number.isNaN(prefix) || prefix < 0 || prefix > 32) return null; + const hostBits = 32 - prefix; + if (hostBits <= 1) return 0; + return 2 ** hostBits - 2; +}; + +type DockerNetworkSummary = { + Name: string; + Driver: string; + IPAM?: { Config?: Array<{ Subnet?: string }> | null }; +}; + +type InspectableNetwork = { + inspect: (opts?: { verbose?: boolean }) => Promise<{ + Containers?: Record; + }>; +}; + +// Includes reserved networks like dokploy-network, which are excluded from the `network` table/UI. +const getNetworksIpUsage = async ( + serverId: string | undefined, +): Promise => { + const docker = await getRemoteDocker(serverId); + const dockerNetworks = + (await docker.listNetworks()) as DockerNetworkSummary[]; + const relevant = dockerNetworks.filter( + (n) => n.Driver === "bridge" || n.Driver === "overlay", + ); + + const rows = await Promise.all( + relevant.map(async (n): Promise => { + // IPv4 entry specifically: dual-stack networks list an IPv6 config too, and it may come first. + const subnet = n.IPAM?.Config?.find( + (c) => c.Subnet && !c.Subnet.includes(":"), + )?.Subnet; + const capacity = getSubnetCapacity(subnet); + let containersInUse = 0; + try { + // Cast the object (not the detached method) so `net.inspect(...)` stays a bound method call. + const net = docker.getNetwork(n.Name) as unknown as InspectableNetwork; + // verbose: true for full swarm-wide container list on overlay networks + const info = await net.inspect( + n.Driver === "overlay" ? { verbose: true } : undefined, + ); + containersInUse = Object.keys(info.Containers ?? {}).length; + } catch { + // Network may have disappeared between list and inspect; report 0 + } + return { + name: n.Name, + driver: n.Driver, + subnet: subnet ?? null, + containersInUse, + capacity, + percentUsed: + capacity && capacity > 0 + ? Math.round((containersInUse / capacity) * 100) + : null, + }; + }), + ); + + return rows.sort((a, b) => (b.percentUsed ?? -1) - (a.percentUsed ?? -1)); +}; + +// Sums Application memoryReservation/cpuReservation (bytes/NanoCPUs, see +// calculateResources in utils/docker/utils.ts). Compose has no such columns. +export const getReservationSummary = async ( + orgId: string, + serverId?: string, +) => { + const appRows = await db + .select({ + memoryReservation: applications.memoryReservation, + cpuReservation: applications.cpuReservation, + }) + .from(applications) + .innerJoin( + environments, + eq(applications.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where( + and( + eq(projects.organizationId, orgId), + serverId + ? eq(applications.serverId, serverId) + : isNull(applications.serverId), + ), + ); + + let memoryReservedBytes = 0; + let cpuReservedNanoCpus = 0; + for (const row of appRows) { + if (row.memoryReservation) { + memoryReservedBytes += toInt(row.memoryReservation); + } + if (row.cpuReservation) { + cpuReservedNanoCpus += toInt(row.cpuReservation); + } + } + + const composeRows = await db + .select({ composeId: compose.composeId }) + .from(compose) + .innerJoin( + environments, + eq(compose.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where( + and( + eq(projects.organizationId, orgId), + serverId ? eq(compose.serverId, serverId) : isNull(compose.serverId), + ), + ); + + return { + memoryReservedBytes, + cpuReservedNanoCpus, + appCount: appRows.length, + unsupportedComposeCount: composeRows.length, + }; +}; + +export const getServerHealth = async ( + orgId: string, + serverId?: string, + sinceHours = 24, +): Promise => { + if (IS_CLOUD && !serverId) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Server is required", + }); + } + + const hours = Math.max( + 1, + Math.min(168, Math.floor(Number(sinceHours) || 24)), + ); + const script = buildHealthScript(hours); + + let stdout = ""; + try { + const result = serverId + ? await execAsyncRemote(serverId, script) + : await execAsync(script); + stdout = result.stdout; + } catch (error) { + return emptyResult(error); + } + + let parsed: RawHealthOutput; + try { + parsed = JSON.parse(stdout.trim()); + } catch { + return emptyResult(new Error("Could not parse server health output")); + } + + const daemonErrors = b64Decode(parsed.daemonErrorsBase64) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + + let addressPools: unknown = null; + const daemonConfigText = b64Decode(parsed.daemonConfigBase64); + if (daemonConfigText) { + try { + addressPools = + JSON.parse(daemonConfigText)?.["default-address-pools"] ?? null; + } catch { + addressPools = null; + } + } + + // Not part of the shell script above: needs structured per-network data from the Docker API. + const [reservation, networkUsageResult] = await Promise.all([ + getReservationSummary(orgId, serverId).catch(() => null), + getNetworksIpUsage(serverId).then( + (usage) => ({ usage, error: undefined }), + (error: unknown) => ({ + usage: [] as NetworkIpUsage[], + error: + error instanceof Error + ? error.message + : "Failed to read network usage", + }), + ), + ]); + + return { + checkedAt: new Date().toISOString(), + containers: { + containerCount: toInt(parsed.containerCount), + serviceCount: toInt(parsed.serviceCount), + }, + resources: { + memTotalBytes: toInt(parsed.memTotalBytes), + memUsedBytes: toInt(parsed.memUsedBytes), + cpuCount: toInt(parsed.cpuCount), + }, + inotify: { + maxWatches: toInt(parsed.inotifyMaxWatches), + maxInstances: toInt(parsed.inotifyMaxInstances), + maxQueuedEvents: toInt(parsed.inotifyMaxQueuedEvents), + currentInstances: toInt(parsed.inotifyCurrentInstances), + persisted: toInt(parsed.inotifyPersistedCount) > 0, + }, + disk: { + totalBytes: toInt(parsed.diskTotalBytes), + usedBytes: toInt(parsed.diskUsedBytes), + }, + dockerNetworks: { + count: toInt(parsed.networkCount), + addressPools, + usage: networkUsageResult.usage, + usageError: networkUsageResult.error, + }, + daemonErrors, + daemonLogsWindow: + toInt(parsed.daemonLogsToEpoch) > 0 + ? { + fromEpoch: toInt(parsed.daemonLogsFromEpoch), + toEpoch: toInt(parsed.daemonLogsToEpoch), + } + : null, + reservation, + }; +}; diff --git a/packages/server/src/utils/deployment-failure-classifier.ts b/packages/server/src/utils/deployment-failure-classifier.ts new file mode 100644 index 000000000..c2d8af883 --- /dev/null +++ b/packages/server/src/utils/deployment-failure-classifier.ts @@ -0,0 +1,69 @@ +export interface FailurePattern { + id: string; + label: string; + pattern: RegExp; +} + +export const FAILURE_PATTERNS: FailurePattern[] = [ + { + id: "inotify-limit", + label: "Inotify limit reached", + pattern: /max_user_instances|max_user_watches|inotify watch limit reached/i, + }, + { + id: "oom", + label: "Out of memory (OOM)", + pattern: /\boom[-_ ]?kill(er|ed)?\b|killed process|out of memory/i, + }, + { + id: "address-pool-exhausted", + label: "Docker network address pool exhausted", + pattern: + /could not find an available[,]? non-overlapping|available.*address pool/i, + }, + { + id: "network-ip-exhausted", + label: "Network out of IP addresses", + pattern: + /could not find an available ip|no available ip addresses|task allocation failure/i, + }, + { + id: "network-attach-timeout", + label: "Network attach timed out", + pattern: + /attaching to network failed|failed to set up container networking/i, + }, + { + id: "disk-full", + label: "Disk space exhausted", + pattern: /no space left on device/i, + }, + { + id: "conntrack-full", + label: "Connection tracking table full", + pattern: /nf_conntrack:?\s*table full/i, + }, + { + id: "fd-limit", + label: "File descriptor limit reached", + pattern: /too many open files/i, + }, + { + id: "pid-limit", + label: "Process/PID limit reached", + pattern: + /pids\.max|cannot allocate memory|fork:.*resource temporarily unavailable/i, + }, +]; + +export const classifyFailure = ( + text: string | null | undefined, +): { id: string; label: string } | null => { + if (!text) return null; + for (const entry of FAILURE_PATTERNS) { + if (entry.pattern.test(text)) { + return { id: entry.id, label: entry.label }; + } + } + return null; +};