From 4bf6dfff13154318dfa77a09fc8c74ccdf17d80b Mon Sep 17 00:00:00 2001 From: Narciso Date: Tue, 11 Aug 2026 15:10:13 -0400 Subject: [PATCH] feat: add Docker Health diagnostics tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server diagnostics tab (Docker dashboard) with: - Per-network IP usage: subnet capacity vs. containers in use, including reserved networks like dokploy-network which are excluded from the managed Networks tab. Surfaces read errors instead of silently showing zeroes when the Docker API call fails. - Broadened daemon-error detection and failure classification for Swarm network failures: IP pool exhaustion on an existing network ("could not find an available IP", "task allocation failure") and network attach timeouts ("attaching to network failed", "context deadline exceeded") — both previously fell through as Unclassified. Verified end-to-end against a real systemd-managed dockerd. - Failed-deployment rows now carry the raw log tail used for classification, shown on hover and linked to their service/project, so a guess (or lack of one) can be verified against the actual error and followed straight to the app. - Daemon-error panel is a read-only code editor, always visible (not hidden when empty), prefixed with the actual "Logs from X to Y" window passed to journalctl so the --since window can be verified. - "Download report" exports everything on screen (metrics + raw logs) as Markdown, so a customer can send one file instead of a screen-share. - Health check now IS_CLOUD-gated the same way network sync already is: a cloud org can no longer omit serverId and have the check silently run against the shared platform host instead of their own server. Includes an inotify diagnostics card (current vs. max_user_instances/ watches/queued_events) and a failed-deployments log with automatic cause classification based on daemon/log error patterns. --- .../server/server-health-subnet.test.ts | 24 + .../deployment-failure-classifier.test.ts | 55 ++ .../dashboard/docker/health/show-health.tsx | 719 ++++++++++++++++++ .../settings/servers/show-health-modal.tsx | 28 + .../settings/servers/show-servers.tsx | 19 + apps/dokploy/pages/dashboard/docker.tsx | 29 +- apps/dokploy/server/api/routers/deployment.ts | 20 + apps/dokploy/server/api/routers/docker.ts | 27 +- packages/server/src/db/schema/deployment.ts | 7 + packages/server/src/index.ts | 2 + packages/server/src/services/deployment.ts | 183 ++++- packages/server/src/services/server-health.ts | 386 ++++++++++ .../utils/deployment-failure-classifier.ts | 69 ++ 13 files changed, 1561 insertions(+), 7 deletions(-) create mode 100644 apps/dokploy/__test__/server/server-health-subnet.test.ts create mode 100644 apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts create mode 100644 apps/dokploy/components/dashboard/docker/health/show-health.tsx create mode 100644 apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx create mode 100644 packages/server/src/services/server-health.ts create mode 100644 packages/server/src/utils/deployment-failure-classifier.ts diff --git a/apps/dokploy/__test__/server/server-health-subnet.test.ts b/apps/dokploy/__test__/server/server-health-subnet.test.ts new file mode 100644 index 000000000..9d0f7f120 --- /dev/null +++ b/apps/dokploy/__test__/server/server-health-subnet.test.ts @@ -0,0 +1,24 @@ +import { getSubnetCapacity } from "@dokploy/server/services/server-health"; +import { describe, expect, test } from "vitest"; + +describe("getSubnetCapacity", () => { + test("returns null for missing/invalid input", () => { + expect(getSubnetCapacity(undefined)).toBeNull(); + expect(getSubnetCapacity("")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0")).toBeNull(); + expect(getSubnetCapacity("not-a-subnet")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0/33")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0/-1")).toBeNull(); + }); + + test("excludes network and broadcast addresses", () => { + expect(getSubnetCapacity("10.0.1.0/24")).toBe(254); + expect(getSubnetCapacity("10.0.0.0/16")).toBe(65534); + expect(getSubnetCapacity("10.99.99.0/30")).toBe(2); + }); + + test("returns 0 for subnets too small to hold a host", () => { + expect(getSubnetCapacity("10.0.0.0/31")).toBe(0); + expect(getSubnetCapacity("10.0.0.0/32")).toBe(0); + }); +}); diff --git a/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts b/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts new file mode 100644 index 000000000..e1272d9b1 --- /dev/null +++ b/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts @@ -0,0 +1,55 @@ +import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; +import { describe, expect, test } from "vitest"; + +describe("classifyFailure", () => { + test("returns null for empty/null/undefined input", () => { + expect(classifyFailure(null)).toBeNull(); + expect(classifyFailure(undefined)).toBeNull(); + expect(classifyFailure("")).toBeNull(); + }); + + test("returns null when nothing matches", () => { + expect(classifyFailure("Build succeeded, deploying now")).toBeNull(); + }); + + test.each([ + [ + "inotify-limit", + "Error: ENOSPC: System limit for number of file watchers reached, inotify watch limit reached", + ], + [ + "oom", + "Container killed process, out of memory: Kill process 1234 (node) score", + ], + [ + "address-pool-exhausted", + "Error response from daemon: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network", + ], + [ + "network-ip-exhausted", + 'time="2026-08-12T19:10:14Z" level=error msg="task allocation failure" error="failed to allocate network IP for task kow0rt7j4nk166qill9vgb6gw network cuz8knvfyobvqsp10yvh39iu7: could not find an available IP"', + ], + [ + "network-attach-timeout", + "Error response from daemon: failed to set up container networking: attaching to network failed, make sure your network options are correct and check manager logs: context deadline exceeded", + ], + ["disk-full", "write /var/lib/docker/tmp/x: no space left on device"], + ["conntrack-full", "nf_conntrack: table full, dropping packet"], + ["fd-limit", "accept4: too many open files"], + ["pid-limit", "fork: retry: Resource temporarily unavailable"], + ])("classifies %s pattern", (expectedId, logLine) => { + expect(classifyFailure(logLine)?.id).toBe(expectedId); + }); + + test("network-ip-exhausted and address-pool-exhausted stay distinct", () => { + const poolExhausted = + "could not find an available, non-overlapping IPv4 address pool"; + const ipExhausted = "could not find an available IP"; + expect(classifyFailure(poolExhausted)?.id).toBe("address-pool-exhausted"); + expect(classifyFailure(ipExhausted)?.id).toBe("network-ip-exhausted"); + }); + + test("matches case-insensitively", () => { + expect(classifyFailure("NO SPACE LEFT ON DEVICE")?.id).toBe("disk-full"); + }); +}); diff --git a/apps/dokploy/components/dashboard/docker/health/show-health.tsx b/apps/dokploy/components/dashboard/docker/health/show-health.tsx new file mode 100644 index 000000000..6d2a5f3c3 --- /dev/null +++ b/apps/dokploy/components/dashboard/docker/health/show-health.tsx @@ -0,0 +1,719 @@ +import { format } from "date-fns"; +import { + AlertTriangle, + Calendar as CalendarIcon, + Cpu, + Download, + HardDrive, + Info, + Loader2, + Network, + RefreshCw, + Server as ServerIcon, +} from "lucide-react"; +import Link from "next/link"; +import { useEffect, useState } from "react"; +import { CodeEditor } from "@/components/shared/code-editor"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Calendar } from "@/components/ui/calendar"; +import { Card } from "@/components/ui/card"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { Progress } from "@/components/ui/progress"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { api } from "@/utils/api"; + +interface Props { + serverId?: string; +} + +const bytesToGb = (bytes: number) => (bytes / 1024 ** 3).toFixed(1); +const nanoCpusToCores = (nanoCpus: number) => (nanoCpus / 1e9).toFixed(2); +const pct = (used: number, total: number) => + Math.min(100, Math.round((used / (total || 1)) * 100)); + +const SINCE_HOURS_OPTIONS = [ + { label: "Last hour", value: 1 }, + { label: "Last 6 hours", value: 6 }, + { label: "Last 24 hours", value: 24 }, + { label: "Last 7 days", value: 168 }, +]; + +const getDefaultDateRange = () => { + const to = new Date(); + const from = new Date(); + from.setDate(from.getDate() - 7); + return { from, to }; +}; + +const getServiceInfo = (deployment: { + application?: { + applicationId: string; + name: string; + environment?: { + environmentId: string; + project?: { projectId: string; name: string }; + }; + } | null; + compose?: { + composeId: string; + name: string; + environment?: { + environmentId: string; + project?: { projectId: string; name: string }; + }; + } | null; +}) => { + const app = deployment.application; + const comp = deployment.compose; + if (app?.environment?.project) { + return { + name: app.name, + projectId: app.environment.project.projectId, + projectName: app.environment.project.name, + href: `/dashboard/project/${app.environment.project.projectId}/environment/${app.environment.environmentId}/services/application/${app.applicationId}`, + }; + } + if (comp?.environment?.project) { + return { + name: comp.name, + projectId: comp.environment.project.projectId, + projectName: comp.environment.project.name, + href: `/dashboard/project/${comp.environment.project.projectId}/environment/${comp.environment.environmentId}/services/compose/${comp.composeId}`, + }; + } + return null; +}; + +export const ShowHealth = ({ serverId }: Props) => { + const [sinceHours, setSinceHours] = useState(24); + const { + data: health, + isFetching, + refetch, + isFetched, + } = api.docker.getServerHealth.useQuery( + { serverId, sinceHours }, + { enabled: false }, + ); + + const [dateRange, setDateRange] = useState<{ + from: Date | undefined; + to: Date | undefined; + }>(() => getDefaultDateRange()); + const [hasCustomRange, setHasCustomRange] = useState(false); + const [projectId, setProjectId] = useState(undefined); + const { data: projects } = api.project.all.useQuery(); + + const { + data: failedDeployments, + isFetching: isFetchingFailed, + refetch: refetchFailed, + isFetched: isFetchedFailed, + } = api.deployment.allFailedCentralized.useQuery( + { + dateFrom: dateRange.from?.toISOString(), + dateTo: dateRange.to?.toISOString(), + projectId, + }, + { enabled: false }, + ); + + // Mount-only: filter changes require an explicit Check click, not an auto-refetch. + useEffect(() => { + refetch(); + refetchFailed(); + }, []); + + // Refreshes the default range's "to" bound to now right before checking. + const [checkFailedTick, setCheckFailedTick] = useState(0); + useEffect(() => { + if (checkFailedTick > 0) refetchFailed(); + }, [checkFailedTick]); + const handleCheckFailed = () => { + if (!hasCustomRange) setDateRange(getDefaultDateRange()); + setCheckFailedTick((t) => t + 1); + }; + + const memUsedPct = health + ? pct(health.resources.memUsedBytes, health.resources.memTotalBytes) + : 0; + const diskUsedPct = health + ? pct(health.disk.usedBytes, health.disk.totalBytes) + : 0; + const inotifyUsedPct = health + ? pct(health.inotify.currentInstances, health.inotify.maxInstances) + : 0; + + const daemonLogsWindowText = health?.daemonLogsWindow + ? `Logs from ${new Date(health.daemonLogsWindow.fromEpoch * 1000).toLocaleString()} to ${new Date(health.daemonLogsWindow.toEpoch * 1000).toLocaleString()}` + : null; + const daemonErrorsText = [ + daemonLogsWindowText, + health && health.daemonErrors.length > 0 + ? health.daemonErrors.join("\n") + : "(no daemon errors matched in this window)", + ] + .filter(Boolean) + .join("\n\n"); + + const buildHealthReport = () => { + if (!health) return ""; + const lines: string[] = []; + lines.push("# Dokploy server health report"); + lines.push(`Generated: ${new Date(health.checkedAt).toLocaleString()}`); + lines.push( + `Window: ${SINCE_HOURS_OPTIONS.find((o) => o.value === sinceHours)?.label ?? `${sinceHours}h`}`, + ); + lines.push(""); + + lines.push("## Containers & services"); + lines.push(`Containers: ${health.containers.containerCount}`); + lines.push(`Swarm services: ${health.containers.serviceCount}`); + lines.push(""); + + lines.push("## Host resources"); + lines.push( + `Memory: ${bytesToGb(health.resources.memUsedBytes)} / ${bytesToGb(health.resources.memTotalBytes)} GB`, + ); + lines.push(`CPU cores: ${health.resources.cpuCount}`); + lines.push(""); + + lines.push("## Disk (/)"); + lines.push( + `${bytesToGb(health.disk.usedBytes)} / ${bytesToGb(health.disk.totalBytes)} GB`, + ); + lines.push(""); + + lines.push("## Inotify"); + lines.push( + `max_user_instances: ${health.inotify.currentInstances} / ${health.inotify.maxInstances}`, + ); + lines.push(`max_user_watches: ${health.inotify.maxWatches}`); + lines.push(`max_queued_events: ${health.inotify.maxQueuedEvents}`); + lines.push( + `Persisted in sysctl: ${health.inotify.persisted ? "yes" : "no"}`, + ); + lines.push(""); + + lines.push("## Docker networks"); + lines.push(`Total networks: ${health.dockerNetworks.count}`); + if (health.dockerNetworks.addressPools) { + lines.push( + `default-address-pools: ${JSON.stringify(health.dockerNetworks.addressPools)}`, + ); + } + if (health.dockerNetworks.usageError) { + lines.push( + `Could not read network IP usage: ${health.dockerNetworks.usageError}`, + ); + } else if (health.dockerNetworks.usage.length > 0) { + lines.push(""); + lines.push( + "| Network | Driver | Subnet | IPs in use | Capacity | Usage |", + ); + lines.push("|---|---|---|---|---|---|"); + for (const n of health.dockerNetworks.usage) { + lines.push( + `| ${n.name} | ${n.driver} | ${n.subnet ?? "auto"} | ${n.containersInUse} | ${n.capacity ?? "—"} | ${n.percentUsed ?? "—"}% |`, + ); + } + } + lines.push(""); + + if (health.reservation) { + lines.push("## Memory/CPU reservation"); + lines.push( + `Reserved memory: ${bytesToGb(health.reservation.memoryReservedBytes)} GB`, + ); + lines.push( + `Reserved CPU: ${nanoCpusToCores(health.reservation.cpuReservedNanoCpus)} cores`, + ); + lines.push(`Across ${health.reservation.appCount} Application services`); + if (health.reservation.unsupportedComposeCount > 0) { + lines.push( + `${health.reservation.unsupportedComposeCount} Compose service(s) not included (not tracked per-service)`, + ); + } + lines.push(""); + } + + lines.push("## Failed deployments"); + if (failedDeployments && failedDeployments.length > 0) { + for (const d of failedDeployments) { + const info = getServiceInfo(d); + const cause = d.classification?.label ?? "Unclassified"; + lines.push( + `- ${format(new Date(d.createdAt), "PPp")} | ${info?.name ?? "—"} | ${info?.projectName ?? "—"} | ${cause}`, + ); + } + } else { + lines.push("(none checked, or none in range)"); + } + lines.push(""); + + lines.push("## Docker daemon errors (raw)"); + lines.push("```"); + lines.push(daemonErrorsText); + lines.push("```"); + + return lines.join("\n"); + }; + + const handleDownload = () => { + const report = buildHealthReport(); + if (!report) return; + const blob = new Blob([report], { type: "text/markdown" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `dokploy-health-report-${new Date().toISOString().replace(/[:.]/g, "-")}.md`; + a.click(); + URL.revokeObjectURL(url); + }; + + return ( +
+ +
+
+
+

Server diagnostics

+

+ Runs a read-only check over SSH (inotify limits, disk, Docker + network pool, daemon errors). Nothing runs automatically — click + to check. +

+
+
+ + + {health && !health.error && ( + + )} +
+
+ + {!isFetched && !isFetching && ( +
+ + + Nothing checked yet — click "Check server health" above. + +
+ )} + + {health?.error && ( +
+ Couldn't read server health: {health.error} +
+ )} + + {health && !health.error && ( +
+ +
+ + Containers & services +
+
+ {health.containers.containerCount} +
+
+ containers · {health.containers.serviceCount} swarm services +
+ + + +
+ + Host resources +
+
+ Memory: {bytesToGb(health.resources.memUsedBytes)} /{" "} + {bytesToGb(health.resources.memTotalBytes)} GB +
+ +
+ {health.resources.cpuCount} CPU cores +
+ + + +
+ + Disk (/) +
+
+ {bytesToGb(health.disk.usedBytes)} /{" "} + {bytesToGb(health.disk.totalBytes)} GB +
+ + + + +
+ + + Inotify + + + {health.inotify.persisted ? "persisted" : "runtime only"} + +
+
+ max_user_instances: {health.inotify.currentInstances} /{" "} + {health.inotify.maxInstances.toLocaleString()} ( + {inotifyUsedPct}%) +
+ +
+ max_user_watches: {health.inotify.maxWatches.toLocaleString()}{" "} + · max_queued_events:{" "} + {health.inotify.maxQueuedEvents.toLocaleString()} +
+ + + +
+ + Docker networks +
+
+ {health.dockerNetworks.count} +
+
+ The default Docker address pool is exhausted around ~30 + networks unless default-address-pools is + configured in /etc/docker/daemon.json. + {health.dockerNetworks.addressPools + ? " Custom pool detected on this server." + : ""} +
+ + + {health.reservation && ( + +
+ + Memory/CPU reservation +
+
+ {bytesToGb(health.reservation.memoryReservedBytes)} GB + reserved ·{" "} + {nanoCpusToCores(health.reservation.cpuReservedNanoCpus)}{" "} + CPUs reserved +
+
+ Across {health.reservation.appCount} Application services. + {health.reservation.unsupportedComposeCount > 0 + ? ` ${health.reservation.unsupportedComposeCount} Compose service(s) on this server aren't included — reservations aren't tracked per-service for Compose.` + : ""} +
+ + )} +
+ )} + + {health && + !health.error && + (health.dockerNetworks.usage.length > 0 || + health.dockerNetworks.usageError) && ( +
+

+ + Network IP usage +

+

+ IPs assigned to containers vs. the subnet's usable capacity, + per network — including reserved networks like{" "} + dokploy-network. A network stuck near 100% blocks + new containers from starting even when the host has plenty of + other resources free. +

+ {health.dockerNetworks.usageError ? ( +
+ Couldn't read network IP usage:{" "} + {health.dockerNetworks.usageError} +
+ ) : ( +
+ + + + Network + Driver + Subnet + IPs in use + Usage + + + + {health.dockerNetworks.usage.map((n) => ( + + + {n.name} + + + {n.driver} + + + {n.subnet ?? "Auto"} + + + {n.capacity !== null + ? `${n.containersInUse} / ${n.capacity}` + : n.containersInUse} + + + {n.percentUsed !== null ? ( + = 90 + ? "red" + : n.percentUsed >= 70 + ? "yellow" + : "green" + } + > + {n.percentUsed}% + + ) : ( + + — + + )} + + + ))} + +
+
+ )} +
+ )} + + {health && !health.error && ( +
+

+ Docker daemon errors ( + {SINCE_HOURS_OPTIONS.find( + (o) => o.value === sinceHours, + )?.label.toLowerCase()} + ) +

+ +
+ )} +
+ + + +
+
+
+

Failed deployments

+

+ Deployments that ended in error, with an automatic cause guess + based on the same patterns used to triage failures manually. +

+
+
+ + + + + + + { + setHasCustomRange(true); + setDateRange({ from: range?.from, to: range?.to }); + }} + numberOfMonths={2} + /> + + + +
+
+ + {!isFetchedFailed && !isFetchingFailed && ( +
+ Nothing checked yet. +
+ )} + + {isFetchedFailed && (failedDeployments?.length ?? 0) === 0 && ( +
+ No failed deployments in this range. +
+ )} + + {(failedDeployments?.length ?? 0) > 0 && ( + + + + Date + Service + Project + Likely cause + + + + {failedDeployments?.map((deployment) => { + const info = getServiceInfo(deployment); + return ( + + + {format(new Date(deployment.createdAt), "PPp")} + + + {info ? ( + + {info.name} + + ) : ( + "—" + )} + + + {info ? ( + + {info.projectName} + + ) : ( + "—" + )} + + + + {deployment.classification ? ( + + {deployment.classification.label} + + ) : ( + + Unclassified + + )} + {deployment.rawError && ( + + + + + + {deployment.rawError} + + + )} + + + + ); + })} + +
+ )} +
+ +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx b/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx new file mode 100644 index 000000000..50daf8a4e --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx @@ -0,0 +1,28 @@ +import { Stethoscope } from "lucide-react"; +import { useState } from "react"; +import { ShowHealth } from "@/components/dashboard/docker/health/show-health"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogTrigger } from "@/components/ui/dialog"; + +interface Props { + serverId: string; +} + +export const ShowHealthModal = ({ serverId }: Props) => { + const [isOpen, setIsOpen] = useState(false); + + return ( + + + + + +
+ +
+ + + ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx index 1fade3f5a..93f4d977e 100644 --- a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx @@ -35,6 +35,7 @@ import { TerminalModal } from "../web-server/terminal-modal"; import { ShowServerActions } from "./actions/show-server-actions"; import { HandleServers } from "./handle-servers"; import { SetupServer } from "./setup-server"; +import { ShowHealthModal } from "./show-health-modal"; import { ShowMonitoringModal } from "./show-monitoring-modal"; import { WelcomeSubscription } from "./welcome-stripe/welcome-subscription"; @@ -324,6 +325,24 @@ export const ShowServers = () => { )} + {permissions?.docker.read && + permissions?.server.read && + server.sshKeyId && + !isBuildServer && ( + + +
+ +
+ + +

Health

+ + + )} +
{permissions?.server.delete && ( diff --git a/apps/dokploy/pages/dashboard/docker.tsx b/apps/dokploy/pages/dashboard/docker.tsx index c73627c77..d2dbba70d 100644 --- a/apps/dokploy/pages/dashboard/docker.tsx +++ b/apps/dokploy/pages/dashboard/docker.tsx @@ -6,6 +6,7 @@ import type { ReactElement } from "react"; import superjson from "superjson"; import { ShowDiskUsage } from "@/components/dashboard/docker/disk-usage/show-disk-usage"; import { ShowDockerEvents } from "@/components/dashboard/docker/events/show-docker-events"; +import { ShowHealth } from "@/components/dashboard/docker/health/show-health"; import { ShowImages } from "@/components/dashboard/docker/images/show-images"; import { ShowContainers } from "@/components/dashboard/docker/show/show-containers"; import { ShowVolumes } from "@/components/dashboard/docker/volumes/show-volumes"; @@ -18,15 +19,25 @@ import { ServerFilter } from "@/components/shared/server-filter"; import { Card } from "@/components/ui/card"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { appRouter } from "@/server/api/root"; +import { api } from "@/utils/api"; const DEFAULT_TAB = "containers"; const DEFAULT_SWARM_TAB = "overview"; const Dashboard = () => { const router = useRouter(); + const { data: isCloud } = api.settings.isCloud.useQuery(); + const { data: permissions } = api.user.getPermissions.useQuery(); + // Host-level diagnostics, so this needs server.read on top of docker.read. + const canSeeHealth = !!permissions?.docker.read && !!permissions?.server.read; - const activeTab = + const queryTab = typeof router.query.tab === "string" ? router.query.tab : DEFAULT_TAB; + const activeTab = + (isCloud && queryTab === "networks") || + (queryTab === "health" && !canSeeHealth) + ? DEFAULT_TAB + : queryTab; const activeSwarmTab = typeof router.query.subtab === "string" @@ -67,9 +78,10 @@ const Dashboard = () => { Swarm Images Volumes - Networks + {!isCloud && Networks} Events Disk Usage + {canSeeHealth && Health} @@ -102,15 +114,22 @@ const Dashboard = () => { - - - + {!isCloud && ( + + + + )} + {canSeeHealth && ( + + + + )} )} diff --git a/apps/dokploy/server/api/routers/deployment.ts b/apps/dokploy/server/api/routers/deployment.ts index dcc74226a..8f19427cf 100644 --- a/apps/dokploy/server/api/routers/deployment.ts +++ b/apps/dokploy/server/api/routers/deployment.ts @@ -6,6 +6,7 @@ import { findAllDeploymentsByServerId, findAllDeploymentsCentralized, findDeploymentById, + findFailedDeploymentsCentralized, findScheduleById, IS_CLOUD, removeDeployment, @@ -14,6 +15,7 @@ import { } from "@dokploy/server"; import { db } from "@dokploy/server/db"; import { + checkPermission, checkServicePermissionAndAccess, findMemberByUserId, } from "@dokploy/server/services/permission"; @@ -27,6 +29,7 @@ import { apiFindAllByCompose, apiFindAllByServer, apiFindAllByType, + apiFindFailedDeployments, deployments, server, } from "@/server/db/schema"; @@ -78,6 +81,23 @@ export const deploymentRouter = createTRPCRouter({ }, ), + // Reads log tails over SSH, so same host-level gate as docker.getServerHealth. + allFailedCentralized: protectedProcedure + .input(apiFindFailedDeployments) + .query(async ({ ctx, input }) => { + await checkPermission(ctx, { + deployment: ["read"], + docker: ["read"], + server: ["read"], + }); + const orgId = ctx.session.activeOrganizationId; + const accessedServices = + ctx.user.role !== "owner" && ctx.user.role !== "admin" + ? (await findMemberByUserId(ctx.user.id, orgId)).accessedServices + : null; + return findFailedDeploymentsCentralized(orgId, accessedServices, input); + }), + queueList: withPermission("deployment", "read").query(async ({ ctx }) => { const orgId = ctx.session.activeOrganizationId; let rows: QueueJobRow[]; diff --git a/apps/dokploy/server/api/routers/docker.ts b/apps/dokploy/server/api/routers/docker.ts index e025b3434..010b7723f 100644 --- a/apps/dokploy/server/api/routers/docker.ts +++ b/apps/dokploy/server/api/routers/docker.ts @@ -11,6 +11,7 @@ import { getContainersByAppLabel, getContainersByAppNameMatch, getDockerEvents, + getServerHealth as getServerHealthData, getServiceContainersByAppName, getStackContainersByAppName, listContainerFiles, @@ -18,11 +19,12 @@ import { uploadFileToContainer, writeContainerFile, } from "@dokploy/server"; +import { checkPermission } from "@dokploy/server/services/permission"; import { TRPCError } from "@trpc/server"; import { z } from "zod"; import { audit } from "@/server/api/utils/audit"; import { uploadFileToContainerSchema } from "@/utils/schema"; -import { createTRPCRouter, withPermission } from "../trpc"; +import { createTRPCRouter, protectedProcedure, withPermission } from "../trpc"; export const containerIdRegex = /^[a-zA-Z0-9.\-_]+$/; @@ -52,6 +54,29 @@ export const dockerRouter = createTRPCRouter({ return await getContainers(input.serverId); }), + // Host-level diagnostics, so this requires docker.read AND server.read. + getServerHealth: protectedProcedure + .input( + z.object({ + serverId: z.string().optional(), + sinceHours: z.number().int().min(1).max(168).optional(), + }), + ) + .query(async ({ input, ctx }) => { + await checkPermission(ctx, { docker: ["read"], server: ["read"] }); + if (input.serverId) { + const server = await findServerById(input.serverId); + if (server.organizationId !== ctx.session?.activeOrganizationId) { + throw new TRPCError({ code: "UNAUTHORIZED" }); + } + } + return await getServerHealthData( + ctx.session.activeOrganizationId, + input.serverId, + input.sinceHours, + ); + }), + restartContainer: withPermission("docker", "read") .input( z.object({ diff --git a/packages/server/src/db/schema/deployment.ts b/packages/server/src/db/schema/deployment.ts index 0c1b547e0..c3aea8345 100644 --- a/packages/server/src/db/schema/deployment.ts +++ b/packages/server/src/db/schema/deployment.ts @@ -233,3 +233,10 @@ export const apiFindAllByType = z.object({ "volumeBackup", ]), }); + +export const apiFindFailedDeployments = z.object({ + dateFrom: z.string().datetime().optional(), + dateTo: z.string().datetime().optional(), + projectId: z.string().optional(), + environmentId: z.string().optional(), +}); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index b2a0c7f12..3c91bca59 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -50,6 +50,7 @@ export * from "./services/rollbacks"; export * from "./services/schedule"; export * from "./services/security"; export * from "./services/server"; +export * from "./services/server-health"; export * from "./services/settings"; export * from "./services/ssh-key"; export * from "./services/user"; @@ -95,6 +96,7 @@ export * from "./utils/builders/utils"; export * from "./utils/cluster/upload"; export * from "./utils/crons/enterprise"; export * from "./utils/databases/rebuild"; +export * from "./utils/deployment-failure-classifier"; export * from "./utils/docker/collision"; export * from "./utils/docker/compose"; export * from "./utils/docker/compose/configs"; diff --git a/packages/server/src/services/deployment.ts b/packages/server/src/services/deployment.ts index fd61fc106..3d5a5915d 100644 --- a/packages/server/src/services/deployment.ts +++ b/packages/server/src/services/deployment.ts @@ -10,12 +10,14 @@ import { type apiCreateDeploymentSchedule, type apiCreateDeploymentServer, type apiCreateDeploymentVolumeBackup, + type apiFindFailedDeployments, applications, compose, deployments, environments, projects, } from "@dokploy/server/db/schema"; +import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; import { removeDirectoryIfExistsContent } from "@dokploy/server/utils/filesystem/directory"; import { execAsync, @@ -23,7 +25,7 @@ import { } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { format } from "date-fns"; -import { and, desc, eq, inArray, or, sql } from "drizzle-orm"; +import { and, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm"; import type { z } from "zod"; import { type Application, @@ -942,6 +944,185 @@ export const findAllDeploymentsCentralized = async ( }); }; +// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg above to avoid +// touching that existing code path; adds projectId/environmentId filtering. +type FailedDeploymentIdFilters = { projectId?: string; environmentId?: string }; + +async function getApplicationIdsForFailedDeployments( + orgId: string, + accessedServices: string[] | null, + filters: FailedDeploymentIdFilters, +): Promise { + const conditions = [ + eq(projects.organizationId, orgId), + ...(accessedServices !== null + ? [inArray(applications.applicationId, accessedServices)] + : []), + ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), + ...(filters.environmentId + ? [eq(environments.environmentId, filters.environmentId)] + : []), + ]; + const rows = await db + .select({ applicationId: applications.applicationId }) + .from(applications) + .innerJoin( + environments, + eq(applications.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where(and(...conditions)); + return rows.map((r) => r.applicationId); +} + +async function getComposeIdsForFailedDeployments( + orgId: string, + accessedServices: string[] | null, + filters: FailedDeploymentIdFilters, +): Promise { + const conditions = [ + eq(projects.organizationId, orgId), + ...(accessedServices !== null + ? [inArray(compose.composeId, accessedServices)] + : []), + ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), + ...(filters.environmentId + ? [eq(environments.environmentId, filters.environmentId)] + : []), + ]; + const rows = await db + .select({ composeId: compose.composeId }) + .from(compose) + .innerJoin( + environments, + eq(compose.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where(and(...conditions)); + return rows.map((r) => r.composeId); +} + +const readLogTail = async ( + logPath: string | null, + serverId: string | null | undefined, +) => { + if (!logPath) return ""; + const command = `tail -c 4096 "${logPath}" 2>/dev/null || echo ""`; + const { stdout } = serverId + ? await execAsyncRemote(serverId, command) + : await execAsync(command); + return stdout; +}; + +// Caps simultaneous SSH connections when classifying a batch of failures. +const LOG_TAIL_CONCURRENCY = 8; + +async function mapWithConcurrency( + items: T[], + limit: number, + fn: (item: T) => Promise, +): Promise { + const results = new Array(items.length); + let nextIndex = 0; + const worker = async () => { + while (nextIndex < items.length) { + const current = nextIndex++; + results[current] = await fn(items[current] as T); + } + }; + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, worker), + ); + return results; +} + +export type FailedDeploymentFilters = z.infer; + +// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed +// deployments, each tagged with a classifyFailure() guess from its log tail. +export const findFailedDeploymentsCentralized = async ( + orgId: string, + accessedServices: string[] | null, + filters: FailedDeploymentFilters = {}, +) => { + if (accessedServices !== null && accessedServices.length === 0) { + return []; + } + + const idFilters: FailedDeploymentIdFilters = { + projectId: filters.projectId, + environmentId: filters.environmentId, + }; + + const [appIds, compIds] = await Promise.all([ + getApplicationIdsForFailedDeployments(orgId, accessedServices, idFilters), + getComposeIdsForFailedDeployments(orgId, accessedServices, idFilters), + ]); + + if (appIds.length === 0 && compIds.length === 0) { + return []; + } + + // createdAt is stored as text via toISOString(); normalize the filters the + // same way so the string range comparison below stays valid. + const dateFrom = filters.dateFrom + ? new Date(filters.dateFrom).toISOString() + : undefined; + const dateTo = filters.dateTo + ? new Date(filters.dateTo).toISOString() + : undefined; + + const serviceConditions = [ + ...(appIds.length > 0 ? [inArray(deployments.applicationId, appIds)] : []), + ...(compIds.length > 0 ? [inArray(deployments.composeId, compIds)] : []), + ]; + + const conditions = [ + eq(deployments.status, "error"), + serviceConditions.length === 1 + ? serviceConditions[0] + : or(...serviceConditions), + ...(dateFrom ? [gte(deployments.createdAt, dateFrom)] : []), + ...(dateTo ? [lte(deployments.createdAt, dateTo)] : []), + ]; + + const failed = await db.query.deployments.findMany({ + where: and(...conditions), + orderBy: desc(deployments.createdAt), + with: centralizedDeploymentsWith, + limit: 200, + }); + + // Skip further attempts to a server once it's timed out once, so a batch + // of failures on one unreachable host doesn't pay the SSH timeout N times. + const deadServers = new Set(); + return mapWithConcurrency( + failed, + LOG_TAIL_CONCURRENCY, + async (deployment) => { + const serverId = + deployment.buildServer?.serverId || + deployment.server?.serverId || + deployment.application?.server?.serverId || + deployment.compose?.server?.serverId || + null; + let logTail = ""; + if (!serverId || !deadServers.has(serverId)) { + try { + logTail = await readLogTail(deployment.logPath, serverId); + } catch { + if (serverId) deadServers.add(serverId); + } + } + const combined = `${deployment.errorMessage ?? ""}\n${logTail}`; + const classification = classifyFailure(combined); + // Tail end only: the actual failure line is almost always near the end of the log. + const rawError = combined.trim().slice(-1000) || null; + return { ...deployment, classification, rawError }; + }, + ); +}; + export const updateDeployment = async ( deploymentId: string, deploymentData: Partial, diff --git a/packages/server/src/services/server-health.ts b/packages/server/src/services/server-health.ts new file mode 100644 index 000000000..5da66a511 --- /dev/null +++ b/packages/server/src/services/server-health.ts @@ -0,0 +1,386 @@ +import { db } from "@dokploy/server/db"; +import { + applications, + compose, + environments, + projects, +} from "@dokploy/server/db/schema"; +import { + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import { TRPCError } from "@trpc/server"; +import { and, eq, isNull } from "drizzle-orm"; +import { IS_CLOUD } from "../constants"; +import { getRemoteDocker } from "../utils/servers/remote-docker"; + +export interface NetworkIpUsage { + name: string; + driver: string; + subnet: string | null; + containersInUse: number; + capacity: number | null; + percentUsed: number | null; +} + +export interface ServerHealthResult { + checkedAt: string; + containers: { + containerCount: number; + serviceCount: number; + }; + resources: { + memTotalBytes: number; + memUsedBytes: number; + cpuCount: number; + }; + inotify: { + maxWatches: number; + maxInstances: number; + maxQueuedEvents: number; + currentInstances: number; + persisted: boolean; + }; + disk: { + totalBytes: number; + usedBytes: number; + }; + dockerNetworks: { + count: number; + addressPools: unknown | null; + /** Per-network IP utilization, including reserved networks like dokploy-network */ + usage: NetworkIpUsage[]; + /** Set when `usage` couldn't be read, so the UI can tell that apart from "no networks" */ + usageError?: string; + }; + daemonErrors: string[]; + /** The window actually passed to `journalctl --since`, computed on the target machine's clock */ + daemonLogsWindow: { fromEpoch: number; toEpoch: number } | null; + reservation: { + memoryReservedBytes: number; + cpuReservedNanoCpus: number; + appCount: number; + unsupportedComposeCount: number; + } | null; + error?: string; +} + +interface RawHealthOutput { + containerCount?: number | string; + serviceCount?: number | string; + memTotalBytes?: number | string; + memUsedBytes?: number | string; + cpuCount?: number | string; + inotifyMaxWatches?: number | string; + inotifyMaxInstances?: number | string; + inotifyMaxQueuedEvents?: number | string; + inotifyCurrentInstances?: number | string; + inotifyPersistedCount?: number | string; + diskTotalBytes?: number | string; + diskUsedBytes?: number | string; + networkCount?: number | string; + daemonConfigBase64?: string; + daemonErrorsBase64?: string; + daemonLogsFromEpoch?: number | string; + daemonLogsToEpoch?: number | string; +} + +const toInt = (value: unknown): number => { + const n = Number.parseInt(String(value ?? "0"), 10); + return Number.isFinite(n) ? n : 0; +}; + +const b64Decode = (value?: string): string => { + if (!value) return ""; + try { + return Buffer.from(value, "base64").toString("utf-8"); + } catch { + return ""; + } +}; + +// Single read-only script, one SSH/exec round-trip; each check falls back to 0/empty on its own. +const buildHealthScript = (sinceHours: number) => ` +containerCount=$(docker ps -a -q 2>/dev/null | wc -l | tr -d ' ') +serviceCount=$(docker service ls --format '{{.Name}}' 2>/dev/null | wc -l | tr -d ' ') + +memTotal=$(free -b 2>/dev/null | awk '/^Mem:/{print $2}'); [ -z "$memTotal" ] && memTotal=0 +memUsed=$(free -b 2>/dev/null | awk '/^Mem:/{print $3}'); [ -z "$memUsed" ] && memUsed=0 +cpuCount=$(nproc 2>/dev/null); [ -z "$cpuCount" ] && cpuCount=0 + +inotifyMaxWatches=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null); [ -z "$inotifyMaxWatches" ] && inotifyMaxWatches=0 +inotifyMaxInstances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null); [ -z "$inotifyMaxInstances" ] && inotifyMaxInstances=0 +inotifyMaxQueued=$(cat /proc/sys/fs/inotify/max_queued_events 2>/dev/null); [ -z "$inotifyMaxQueued" ] && inotifyMaxQueued=0 +inotifyCurrentInstances=$(find /proc/*/fd -lname 'anon_inode:inotify*' 2>/dev/null | wc -l | tr -d ' ') +inotifyPersistedCount=$(grep -rl inotify /etc/sysctl.conf /etc/sysctl.d/ 2>/dev/null | wc -l | tr -d ' ') + +diskTotal=$(df -B1 / 2>/dev/null | awk 'NR==2{print $2}'); [ -z "$diskTotal" ] && diskTotal=0 +diskUsed=$(df -B1 / 2>/dev/null | awk 'NR==2{print $3}'); [ -z "$diskUsed" ] && diskUsed=0 + +networkCount=$(docker network ls -q 2>/dev/null | wc -l | tr -d ' ') +daemonConfigB64=$(cat /etc/docker/daemon.json 2>/dev/null | base64 2>/dev/null | tr -d '\\n') + +daemonLogsToEpoch=$(date +%s 2>/dev/null); [ -z "$daemonLogsToEpoch" ] && daemonLogsToEpoch=0 +daemonLogsFromEpoch=$((daemonLogsToEpoch - ${sinceHours} * 3600)) + +daemonErrorsB64=$(journalctl -u docker --no-pager --since "${sinceHours} hours ago" 2>/dev/null | grep -iE "inotify|too many open|cannot allocate|oom|conntrack|no space|pids.max|fork:|resource temporarily unavailable|could not find an available ip|no available ip|task allocation failure|address already in use" | tail -n 50 | base64 2>/dev/null | tr -d '\\n') + +printf '{"containerCount":%s,"serviceCount":%s,"memTotalBytes":%s,"memUsedBytes":%s,"cpuCount":%s,"inotifyMaxWatches":%s,"inotifyMaxInstances":%s,"inotifyMaxQueuedEvents":%s,"inotifyCurrentInstances":%s,"inotifyPersistedCount":%s,"diskTotalBytes":%s,"diskUsedBytes":%s,"networkCount":%s,"daemonConfigBase64":"%s","daemonErrorsBase64":"%s","daemonLogsFromEpoch":%s,"daemonLogsToEpoch":%s}' "$containerCount" "$serviceCount" "$memTotal" "$memUsed" "$cpuCount" "$inotifyMaxWatches" "$inotifyMaxInstances" "$inotifyMaxQueued" "$inotifyCurrentInstances" "$inotifyPersistedCount" "$diskTotal" "$diskUsed" "$networkCount" "$daemonConfigB64" "$daemonErrorsB64" "$daemonLogsFromEpoch" "$daemonLogsToEpoch" +`; + +const emptyResult = (error: unknown): ServerHealthResult => ({ + checkedAt: new Date().toISOString(), + containers: { containerCount: 0, serviceCount: 0 }, + resources: { memTotalBytes: 0, memUsedBytes: 0, cpuCount: 0 }, + inotify: { + maxWatches: 0, + maxInstances: 0, + maxQueuedEvents: 0, + currentInstances: 0, + persisted: false, + }, + disk: { totalBytes: 0, usedBytes: 0 }, + dockerNetworks: { count: 0, addressPools: null, usage: [] }, + daemonErrors: [], + daemonLogsWindow: null, + reservation: null, + error: + error instanceof Error ? error.message : "Could not read server health", +}); + +/** Usable host addresses in a subnet, excluding network + broadcast. Null if the subnet is missing/invalid. */ +export const getSubnetCapacity = ( + subnet: string | undefined, +): number | null => { + if (!subnet) return null; + const prefix = Number.parseInt(subnet.split("/")[1] ?? "", 10); + if (Number.isNaN(prefix) || prefix < 0 || prefix > 32) return null; + const hostBits = 32 - prefix; + if (hostBits <= 1) return 0; + return 2 ** hostBits - 2; +}; + +type DockerNetworkSummary = { + Name: string; + Driver: string; + IPAM?: { Config?: Array<{ Subnet?: string }> | null }; +}; + +type InspectableNetwork = { + inspect: (opts?: { verbose?: boolean }) => Promise<{ + Containers?: Record; + }>; +}; + +// Includes reserved networks like dokploy-network, which are excluded from the `network` table/UI. +const getNetworksIpUsage = async ( + serverId: string | undefined, +): Promise => { + const docker = await getRemoteDocker(serverId); + const dockerNetworks = + (await docker.listNetworks()) as DockerNetworkSummary[]; + const relevant = dockerNetworks.filter( + (n) => n.Driver === "bridge" || n.Driver === "overlay", + ); + + const rows = await Promise.all( + relevant.map(async (n): Promise => { + // IPv4 entry specifically: dual-stack networks list an IPv6 config too, and it may come first. + const subnet = n.IPAM?.Config?.find( + (c) => c.Subnet && !c.Subnet.includes(":"), + )?.Subnet; + const capacity = getSubnetCapacity(subnet); + let containersInUse = 0; + try { + // Cast the object (not the detached method) so `net.inspect(...)` stays a bound method call. + const net = docker.getNetwork(n.Name) as unknown as InspectableNetwork; + // verbose: true for full swarm-wide container list on overlay networks + const info = await net.inspect( + n.Driver === "overlay" ? { verbose: true } : undefined, + ); + containersInUse = Object.keys(info.Containers ?? {}).length; + } catch { + // Network may have disappeared between list and inspect; report 0 + } + return { + name: n.Name, + driver: n.Driver, + subnet: subnet ?? null, + containersInUse, + capacity, + percentUsed: + capacity && capacity > 0 + ? Math.round((containersInUse / capacity) * 100) + : null, + }; + }), + ); + + return rows.sort((a, b) => (b.percentUsed ?? -1) - (a.percentUsed ?? -1)); +}; + +// Sums Application memoryReservation/cpuReservation (bytes/NanoCPUs, see +// calculateResources in utils/docker/utils.ts). Compose has no such columns. +export const getReservationSummary = async ( + orgId: string, + serverId?: string, +) => { + const appRows = await db + .select({ + memoryReservation: applications.memoryReservation, + cpuReservation: applications.cpuReservation, + }) + .from(applications) + .innerJoin( + environments, + eq(applications.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where( + and( + eq(projects.organizationId, orgId), + serverId + ? eq(applications.serverId, serverId) + : isNull(applications.serverId), + ), + ); + + let memoryReservedBytes = 0; + let cpuReservedNanoCpus = 0; + for (const row of appRows) { + if (row.memoryReservation) { + memoryReservedBytes += toInt(row.memoryReservation); + } + if (row.cpuReservation) { + cpuReservedNanoCpus += toInt(row.cpuReservation); + } + } + + const composeRows = await db + .select({ composeId: compose.composeId }) + .from(compose) + .innerJoin( + environments, + eq(compose.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where( + and( + eq(projects.organizationId, orgId), + serverId ? eq(compose.serverId, serverId) : isNull(compose.serverId), + ), + ); + + return { + memoryReservedBytes, + cpuReservedNanoCpus, + appCount: appRows.length, + unsupportedComposeCount: composeRows.length, + }; +}; + +export const getServerHealth = async ( + orgId: string, + serverId?: string, + sinceHours = 24, +): Promise => { + if (IS_CLOUD && !serverId) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Server is required", + }); + } + + const hours = Math.max( + 1, + Math.min(168, Math.floor(Number(sinceHours) || 24)), + ); + const script = buildHealthScript(hours); + + let stdout = ""; + try { + const result = serverId + ? await execAsyncRemote(serverId, script) + : await execAsync(script); + stdout = result.stdout; + } catch (error) { + return emptyResult(error); + } + + let parsed: RawHealthOutput; + try { + parsed = JSON.parse(stdout.trim()); + } catch { + return emptyResult(new Error("Could not parse server health output")); + } + + const daemonErrors = b64Decode(parsed.daemonErrorsBase64) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + + let addressPools: unknown = null; + const daemonConfigText = b64Decode(parsed.daemonConfigBase64); + if (daemonConfigText) { + try { + addressPools = + JSON.parse(daemonConfigText)?.["default-address-pools"] ?? null; + } catch { + addressPools = null; + } + } + + // Not part of the shell script above: needs structured per-network data from the Docker API. + const [reservation, networkUsageResult] = await Promise.all([ + getReservationSummary(orgId, serverId).catch(() => null), + getNetworksIpUsage(serverId).then( + (usage) => ({ usage, error: undefined }), + (error: unknown) => ({ + usage: [] as NetworkIpUsage[], + error: + error instanceof Error + ? error.message + : "Failed to read network usage", + }), + ), + ]); + + return { + checkedAt: new Date().toISOString(), + containers: { + containerCount: toInt(parsed.containerCount), + serviceCount: toInt(parsed.serviceCount), + }, + resources: { + memTotalBytes: toInt(parsed.memTotalBytes), + memUsedBytes: toInt(parsed.memUsedBytes), + cpuCount: toInt(parsed.cpuCount), + }, + inotify: { + maxWatches: toInt(parsed.inotifyMaxWatches), + maxInstances: toInt(parsed.inotifyMaxInstances), + maxQueuedEvents: toInt(parsed.inotifyMaxQueuedEvents), + currentInstances: toInt(parsed.inotifyCurrentInstances), + persisted: toInt(parsed.inotifyPersistedCount) > 0, + }, + disk: { + totalBytes: toInt(parsed.diskTotalBytes), + usedBytes: toInt(parsed.diskUsedBytes), + }, + dockerNetworks: { + count: toInt(parsed.networkCount), + addressPools, + usage: networkUsageResult.usage, + usageError: networkUsageResult.error, + }, + daemonErrors, + daemonLogsWindow: + toInt(parsed.daemonLogsToEpoch) > 0 + ? { + fromEpoch: toInt(parsed.daemonLogsFromEpoch), + toEpoch: toInt(parsed.daemonLogsToEpoch), + } + : null, + reservation, + }; +}; diff --git a/packages/server/src/utils/deployment-failure-classifier.ts b/packages/server/src/utils/deployment-failure-classifier.ts new file mode 100644 index 000000000..c2d8af883 --- /dev/null +++ b/packages/server/src/utils/deployment-failure-classifier.ts @@ -0,0 +1,69 @@ +export interface FailurePattern { + id: string; + label: string; + pattern: RegExp; +} + +export const FAILURE_PATTERNS: FailurePattern[] = [ + { + id: "inotify-limit", + label: "Inotify limit reached", + pattern: /max_user_instances|max_user_watches|inotify watch limit reached/i, + }, + { + id: "oom", + label: "Out of memory (OOM)", + pattern: /\boom[-_ ]?kill(er|ed)?\b|killed process|out of memory/i, + }, + { + id: "address-pool-exhausted", + label: "Docker network address pool exhausted", + pattern: + /could not find an available[,]? non-overlapping|available.*address pool/i, + }, + { + id: "network-ip-exhausted", + label: "Network out of IP addresses", + pattern: + /could not find an available ip|no available ip addresses|task allocation failure/i, + }, + { + id: "network-attach-timeout", + label: "Network attach timed out", + pattern: + /attaching to network failed|failed to set up container networking/i, + }, + { + id: "disk-full", + label: "Disk space exhausted", + pattern: /no space left on device/i, + }, + { + id: "conntrack-full", + label: "Connection tracking table full", + pattern: /nf_conntrack:?\s*table full/i, + }, + { + id: "fd-limit", + label: "File descriptor limit reached", + pattern: /too many open files/i, + }, + { + id: "pid-limit", + label: "Process/PID limit reached", + pattern: + /pids\.max|cannot allocate memory|fork:.*resource temporarily unavailable/i, + }, +]; + +export const classifyFailure = ( + text: string | null | undefined, +): { id: string; label: string } | null => { + if (!text) return null; + for (const entry of FAILURE_PATTERNS) { + if (entry.pattern.test(text)) { + return { id: entry.id, label: entry.label }; + } + } + return null; +};