From 4bf6dfff13154318dfa77a09fc8c74ccdf17d80b Mon Sep 17 00:00:00 2001 From: Narciso Date: Tue, 11 Aug 2026 15:10:13 -0400 Subject: [PATCH 1/5] feat: add Docker Health diagnostics tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server diagnostics tab (Docker dashboard) with: - Per-network IP usage: subnet capacity vs. containers in use, including reserved networks like dokploy-network which are excluded from the managed Networks tab. Surfaces read errors instead of silently showing zeroes when the Docker API call fails. - Broadened daemon-error detection and failure classification for Swarm network failures: IP pool exhaustion on an existing network ("could not find an available IP", "task allocation failure") and network attach timeouts ("attaching to network failed", "context deadline exceeded") — both previously fell through as Unclassified. Verified end-to-end against a real systemd-managed dockerd. - Failed-deployment rows now carry the raw log tail used for classification, shown on hover and linked to their service/project, so a guess (or lack of one) can be verified against the actual error and followed straight to the app. - Daemon-error panel is a read-only code editor, always visible (not hidden when empty), prefixed with the actual "Logs from X to Y" window passed to journalctl so the --since window can be verified. - "Download report" exports everything on screen (metrics + raw logs) as Markdown, so a customer can send one file instead of a screen-share. - Health check now IS_CLOUD-gated the same way network sync already is: a cloud org can no longer omit serverId and have the check silently run against the shared platform host instead of their own server. Includes an inotify diagnostics card (current vs. max_user_instances/ watches/queued_events) and a failed-deployments log with automatic cause classification based on daemon/log error patterns. --- .../server/server-health-subnet.test.ts | 24 + .../deployment-failure-classifier.test.ts | 55 ++ .../dashboard/docker/health/show-health.tsx | 719 ++++++++++++++++++ .../settings/servers/show-health-modal.tsx | 28 + .../settings/servers/show-servers.tsx | 19 + apps/dokploy/pages/dashboard/docker.tsx | 29 +- apps/dokploy/server/api/routers/deployment.ts | 20 + apps/dokploy/server/api/routers/docker.ts | 27 +- packages/server/src/db/schema/deployment.ts | 7 + packages/server/src/index.ts | 2 + packages/server/src/services/deployment.ts | 183 ++++- packages/server/src/services/server-health.ts | 386 ++++++++++ .../utils/deployment-failure-classifier.ts | 69 ++ 13 files changed, 1561 insertions(+), 7 deletions(-) create mode 100644 apps/dokploy/__test__/server/server-health-subnet.test.ts create mode 100644 apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts create mode 100644 apps/dokploy/components/dashboard/docker/health/show-health.tsx create mode 100644 apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx create mode 100644 packages/server/src/services/server-health.ts create mode 100644 packages/server/src/utils/deployment-failure-classifier.ts diff --git a/apps/dokploy/__test__/server/server-health-subnet.test.ts b/apps/dokploy/__test__/server/server-health-subnet.test.ts new file mode 100644 index 000000000..9d0f7f120 --- /dev/null +++ b/apps/dokploy/__test__/server/server-health-subnet.test.ts @@ -0,0 +1,24 @@ +import { getSubnetCapacity } from "@dokploy/server/services/server-health"; +import { describe, expect, test } from "vitest"; + +describe("getSubnetCapacity", () => { + test("returns null for missing/invalid input", () => { + expect(getSubnetCapacity(undefined)).toBeNull(); + expect(getSubnetCapacity("")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0")).toBeNull(); + expect(getSubnetCapacity("not-a-subnet")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0/33")).toBeNull(); + expect(getSubnetCapacity("10.0.0.0/-1")).toBeNull(); + }); + + test("excludes network and broadcast addresses", () => { + expect(getSubnetCapacity("10.0.1.0/24")).toBe(254); + expect(getSubnetCapacity("10.0.0.0/16")).toBe(65534); + expect(getSubnetCapacity("10.99.99.0/30")).toBe(2); + }); + + test("returns 0 for subnets too small to hold a host", () => { + expect(getSubnetCapacity("10.0.0.0/31")).toBe(0); + expect(getSubnetCapacity("10.0.0.0/32")).toBe(0); + }); +}); diff --git a/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts b/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts new file mode 100644 index 000000000..e1272d9b1 --- /dev/null +++ b/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts @@ -0,0 +1,55 @@ +import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; +import { describe, expect, test } from "vitest"; + +describe("classifyFailure", () => { + test("returns null for empty/null/undefined input", () => { + expect(classifyFailure(null)).toBeNull(); + expect(classifyFailure(undefined)).toBeNull(); + expect(classifyFailure("")).toBeNull(); + }); + + test("returns null when nothing matches", () => { + expect(classifyFailure("Build succeeded, deploying now")).toBeNull(); + }); + + test.each([ + [ + "inotify-limit", + "Error: ENOSPC: System limit for number of file watchers reached, inotify watch limit reached", + ], + [ + "oom", + "Container killed process, out of memory: Kill process 1234 (node) score", + ], + [ + "address-pool-exhausted", + "Error response from daemon: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network", + ], + [ + "network-ip-exhausted", + 'time="2026-08-12T19:10:14Z" level=error msg="task allocation failure" error="failed to allocate network IP for task kow0rt7j4nk166qill9vgb6gw network cuz8knvfyobvqsp10yvh39iu7: could not find an available IP"', + ], + [ + "network-attach-timeout", + "Error response from daemon: failed to set up container networking: attaching to network failed, make sure your network options are correct and check manager logs: context deadline exceeded", + ], + ["disk-full", "write /var/lib/docker/tmp/x: no space left on device"], + ["conntrack-full", "nf_conntrack: table full, dropping packet"], + ["fd-limit", "accept4: too many open files"], + ["pid-limit", "fork: retry: Resource temporarily unavailable"], + ])("classifies %s pattern", (expectedId, logLine) => { + expect(classifyFailure(logLine)?.id).toBe(expectedId); + }); + + test("network-ip-exhausted and address-pool-exhausted stay distinct", () => { + const poolExhausted = + "could not find an available, non-overlapping IPv4 address pool"; + const ipExhausted = "could not find an available IP"; + expect(classifyFailure(poolExhausted)?.id).toBe("address-pool-exhausted"); + expect(classifyFailure(ipExhausted)?.id).toBe("network-ip-exhausted"); + }); + + test("matches case-insensitively", () => { + expect(classifyFailure("NO SPACE LEFT ON DEVICE")?.id).toBe("disk-full"); + }); +}); diff --git a/apps/dokploy/components/dashboard/docker/health/show-health.tsx b/apps/dokploy/components/dashboard/docker/health/show-health.tsx new file mode 100644 index 000000000..6d2a5f3c3 --- /dev/null +++ b/apps/dokploy/components/dashboard/docker/health/show-health.tsx @@ -0,0 +1,719 @@ +import { format } from "date-fns"; +import { + AlertTriangle, + Calendar as CalendarIcon, + Cpu, + Download, + HardDrive, + Info, + Loader2, + Network, + RefreshCw, + Server as ServerIcon, +} from "lucide-react"; +import Link from "next/link"; +import { useEffect, useState } from "react"; +import { CodeEditor } from "@/components/shared/code-editor"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Calendar } from "@/components/ui/calendar"; +import { Card } from "@/components/ui/card"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { Progress } from "@/components/ui/progress"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { api } from "@/utils/api"; + +interface Props { + serverId?: string; +} + +const bytesToGb = (bytes: number) => (bytes / 1024 ** 3).toFixed(1); +const nanoCpusToCores = (nanoCpus: number) => (nanoCpus / 1e9).toFixed(2); +const pct = (used: number, total: number) => + Math.min(100, Math.round((used / (total || 1)) * 100)); + +const SINCE_HOURS_OPTIONS = [ + { label: "Last hour", value: 1 }, + { label: "Last 6 hours", value: 6 }, + { label: "Last 24 hours", value: 24 }, + { label: "Last 7 days", value: 168 }, +]; + +const getDefaultDateRange = () => { + const to = new Date(); + const from = new Date(); + from.setDate(from.getDate() - 7); + return { from, to }; +}; + +const getServiceInfo = (deployment: { + application?: { + applicationId: string; + name: string; + environment?: { + environmentId: string; + project?: { projectId: string; name: string }; + }; + } | null; + compose?: { + composeId: string; + name: string; + environment?: { + environmentId: string; + project?: { projectId: string; name: string }; + }; + } | null; +}) => { + const app = deployment.application; + const comp = deployment.compose; + if (app?.environment?.project) { + return { + name: app.name, + projectId: app.environment.project.projectId, + projectName: app.environment.project.name, + href: `/dashboard/project/${app.environment.project.projectId}/environment/${app.environment.environmentId}/services/application/${app.applicationId}`, + }; + } + if (comp?.environment?.project) { + return { + name: comp.name, + projectId: comp.environment.project.projectId, + projectName: comp.environment.project.name, + href: `/dashboard/project/${comp.environment.project.projectId}/environment/${comp.environment.environmentId}/services/compose/${comp.composeId}`, + }; + } + return null; +}; + +export const ShowHealth = ({ serverId }: Props) => { + const [sinceHours, setSinceHours] = useState(24); + const { + data: health, + isFetching, + refetch, + isFetched, + } = api.docker.getServerHealth.useQuery( + { serverId, sinceHours }, + { enabled: false }, + ); + + const [dateRange, setDateRange] = useState<{ + from: Date | undefined; + to: Date | undefined; + }>(() => getDefaultDateRange()); + const [hasCustomRange, setHasCustomRange] = useState(false); + const [projectId, setProjectId] = useState(undefined); + const { data: projects } = api.project.all.useQuery(); + + const { + data: failedDeployments, + isFetching: isFetchingFailed, + refetch: refetchFailed, + isFetched: isFetchedFailed, + } = api.deployment.allFailedCentralized.useQuery( + { + dateFrom: dateRange.from?.toISOString(), + dateTo: dateRange.to?.toISOString(), + projectId, + }, + { enabled: false }, + ); + + // Mount-only: filter changes require an explicit Check click, not an auto-refetch. + useEffect(() => { + refetch(); + refetchFailed(); + }, []); + + // Refreshes the default range's "to" bound to now right before checking. + const [checkFailedTick, setCheckFailedTick] = useState(0); + useEffect(() => { + if (checkFailedTick > 0) refetchFailed(); + }, [checkFailedTick]); + const handleCheckFailed = () => { + if (!hasCustomRange) setDateRange(getDefaultDateRange()); + setCheckFailedTick((t) => t + 1); + }; + + const memUsedPct = health + ? pct(health.resources.memUsedBytes, health.resources.memTotalBytes) + : 0; + const diskUsedPct = health + ? pct(health.disk.usedBytes, health.disk.totalBytes) + : 0; + const inotifyUsedPct = health + ? pct(health.inotify.currentInstances, health.inotify.maxInstances) + : 0; + + const daemonLogsWindowText = health?.daemonLogsWindow + ? `Logs from ${new Date(health.daemonLogsWindow.fromEpoch * 1000).toLocaleString()} to ${new Date(health.daemonLogsWindow.toEpoch * 1000).toLocaleString()}` + : null; + const daemonErrorsText = [ + daemonLogsWindowText, + health && health.daemonErrors.length > 0 + ? health.daemonErrors.join("\n") + : "(no daemon errors matched in this window)", + ] + .filter(Boolean) + .join("\n\n"); + + const buildHealthReport = () => { + if (!health) return ""; + const lines: string[] = []; + lines.push("# Dokploy server health report"); + lines.push(`Generated: ${new Date(health.checkedAt).toLocaleString()}`); + lines.push( + `Window: ${SINCE_HOURS_OPTIONS.find((o) => o.value === sinceHours)?.label ?? `${sinceHours}h`}`, + ); + lines.push(""); + + lines.push("## Containers & services"); + lines.push(`Containers: ${health.containers.containerCount}`); + lines.push(`Swarm services: ${health.containers.serviceCount}`); + lines.push(""); + + lines.push("## Host resources"); + lines.push( + `Memory: ${bytesToGb(health.resources.memUsedBytes)} / ${bytesToGb(health.resources.memTotalBytes)} GB`, + ); + lines.push(`CPU cores: ${health.resources.cpuCount}`); + lines.push(""); + + lines.push("## Disk (/)"); + lines.push( + `${bytesToGb(health.disk.usedBytes)} / ${bytesToGb(health.disk.totalBytes)} GB`, + ); + lines.push(""); + + lines.push("## Inotify"); + lines.push( + `max_user_instances: ${health.inotify.currentInstances} / ${health.inotify.maxInstances}`, + ); + lines.push(`max_user_watches: ${health.inotify.maxWatches}`); + lines.push(`max_queued_events: ${health.inotify.maxQueuedEvents}`); + lines.push( + `Persisted in sysctl: ${health.inotify.persisted ? "yes" : "no"}`, + ); + lines.push(""); + + lines.push("## Docker networks"); + lines.push(`Total networks: ${health.dockerNetworks.count}`); + if (health.dockerNetworks.addressPools) { + lines.push( + `default-address-pools: ${JSON.stringify(health.dockerNetworks.addressPools)}`, + ); + } + if (health.dockerNetworks.usageError) { + lines.push( + `Could not read network IP usage: ${health.dockerNetworks.usageError}`, + ); + } else if (health.dockerNetworks.usage.length > 0) { + lines.push(""); + lines.push( + "| Network | Driver | Subnet | IPs in use | Capacity | Usage |", + ); + lines.push("|---|---|---|---|---|---|"); + for (const n of health.dockerNetworks.usage) { + lines.push( + `| ${n.name} | ${n.driver} | ${n.subnet ?? "auto"} | ${n.containersInUse} | ${n.capacity ?? "—"} | ${n.percentUsed ?? "—"}% |`, + ); + } + } + lines.push(""); + + if (health.reservation) { + lines.push("## Memory/CPU reservation"); + lines.push( + `Reserved memory: ${bytesToGb(health.reservation.memoryReservedBytes)} GB`, + ); + lines.push( + `Reserved CPU: ${nanoCpusToCores(health.reservation.cpuReservedNanoCpus)} cores`, + ); + lines.push(`Across ${health.reservation.appCount} Application services`); + if (health.reservation.unsupportedComposeCount > 0) { + lines.push( + `${health.reservation.unsupportedComposeCount} Compose service(s) not included (not tracked per-service)`, + ); + } + lines.push(""); + } + + lines.push("## Failed deployments"); + if (failedDeployments && failedDeployments.length > 0) { + for (const d of failedDeployments) { + const info = getServiceInfo(d); + const cause = d.classification?.label ?? "Unclassified"; + lines.push( + `- ${format(new Date(d.createdAt), "PPp")} | ${info?.name ?? "—"} | ${info?.projectName ?? "—"} | ${cause}`, + ); + } + } else { + lines.push("(none checked, or none in range)"); + } + lines.push(""); + + lines.push("## Docker daemon errors (raw)"); + lines.push("```"); + lines.push(daemonErrorsText); + lines.push("```"); + + return lines.join("\n"); + }; + + const handleDownload = () => { + const report = buildHealthReport(); + if (!report) return; + const blob = new Blob([report], { type: "text/markdown" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `dokploy-health-report-${new Date().toISOString().replace(/[:.]/g, "-")}.md`; + a.click(); + URL.revokeObjectURL(url); + }; + + return ( +
+ +
+
+
+

Server diagnostics

+

+ Runs a read-only check over SSH (inotify limits, disk, Docker + network pool, daemon errors). Nothing runs automatically — click + to check. +

+
+
+ + + {health && !health.error && ( + + )} +
+
+ + {!isFetched && !isFetching && ( +
+ + + Nothing checked yet — click "Check server health" above. + +
+ )} + + {health?.error && ( +
+ Couldn't read server health: {health.error} +
+ )} + + {health && !health.error && ( +
+ +
+ + Containers & services +
+
+ {health.containers.containerCount} +
+
+ containers · {health.containers.serviceCount} swarm services +
+
+ + +
+ + Host resources +
+
+ Memory: {bytesToGb(health.resources.memUsedBytes)} /{" "} + {bytesToGb(health.resources.memTotalBytes)} GB +
+ +
+ {health.resources.cpuCount} CPU cores +
+
+ + +
+ + Disk (/) +
+
+ {bytesToGb(health.disk.usedBytes)} /{" "} + {bytesToGb(health.disk.totalBytes)} GB +
+ +
+ + +
+ + + Inotify + + + {health.inotify.persisted ? "persisted" : "runtime only"} + +
+
+ max_user_instances: {health.inotify.currentInstances} /{" "} + {health.inotify.maxInstances.toLocaleString()} ( + {inotifyUsedPct}%) +
+ +
+ max_user_watches: {health.inotify.maxWatches.toLocaleString()}{" "} + · max_queued_events:{" "} + {health.inotify.maxQueuedEvents.toLocaleString()} +
+
+ + +
+ + Docker networks +
+
+ {health.dockerNetworks.count} +
+
+ The default Docker address pool is exhausted around ~30 + networks unless default-address-pools is + configured in /etc/docker/daemon.json. + {health.dockerNetworks.addressPools + ? " Custom pool detected on this server." + : ""} +
+
+ + {health.reservation && ( + +
+ + Memory/CPU reservation +
+
+ {bytesToGb(health.reservation.memoryReservedBytes)} GB + reserved ·{" "} + {nanoCpusToCores(health.reservation.cpuReservedNanoCpus)}{" "} + CPUs reserved +
+
+ Across {health.reservation.appCount} Application services. + {health.reservation.unsupportedComposeCount > 0 + ? ` ${health.reservation.unsupportedComposeCount} Compose service(s) on this server aren't included — reservations aren't tracked per-service for Compose.` + : ""} +
+
+ )} +
+ )} + + {health && + !health.error && + (health.dockerNetworks.usage.length > 0 || + health.dockerNetworks.usageError) && ( +
+

+ + Network IP usage +

+

+ IPs assigned to containers vs. the subnet's usable capacity, + per network — including reserved networks like{" "} + dokploy-network. A network stuck near 100% blocks + new containers from starting even when the host has plenty of + other resources free. +

+ {health.dockerNetworks.usageError ? ( +
+ Couldn't read network IP usage:{" "} + {health.dockerNetworks.usageError} +
+ ) : ( +
+ + + + Network + Driver + Subnet + IPs in use + Usage + + + + {health.dockerNetworks.usage.map((n) => ( + + + {n.name} + + + {n.driver} + + + {n.subnet ?? "Auto"} + + + {n.capacity !== null + ? `${n.containersInUse} / ${n.capacity}` + : n.containersInUse} + + + {n.percentUsed !== null ? ( + = 90 + ? "red" + : n.percentUsed >= 70 + ? "yellow" + : "green" + } + > + {n.percentUsed}% + + ) : ( + + — + + )} + + + ))} + +
+
+ )} +
+ )} + + {health && !health.error && ( +
+

+ Docker daemon errors ( + {SINCE_HOURS_OPTIONS.find( + (o) => o.value === sinceHours, + )?.label.toLowerCase()} + ) +

+ +
+ )} +
+
+ + +
+
+
+

Failed deployments

+

+ Deployments that ended in error, with an automatic cause guess + based on the same patterns used to triage failures manually. +

+
+
+ + + + + + + { + setHasCustomRange(true); + setDateRange({ from: range?.from, to: range?.to }); + }} + numberOfMonths={2} + /> + + + +
+
+ + {!isFetchedFailed && !isFetchingFailed && ( +
+ Nothing checked yet. +
+ )} + + {isFetchedFailed && (failedDeployments?.length ?? 0) === 0 && ( +
+ No failed deployments in this range. +
+ )} + + {(failedDeployments?.length ?? 0) > 0 && ( + + + + Date + Service + Project + Likely cause + + + + {failedDeployments?.map((deployment) => { + const info = getServiceInfo(deployment); + return ( + + + {format(new Date(deployment.createdAt), "PPp")} + + + {info ? ( + + {info.name} + + ) : ( + "—" + )} + + + {info ? ( + + {info.projectName} + + ) : ( + "—" + )} + + + + {deployment.classification ? ( + + {deployment.classification.label} + + ) : ( + + Unclassified + + )} + {deployment.rawError && ( + + + + + + {deployment.rawError} + + + )} + + + + ); + })} + +
+ )} +
+
+
+ ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx b/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx new file mode 100644 index 000000000..50daf8a4e --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx @@ -0,0 +1,28 @@ +import { Stethoscope } from "lucide-react"; +import { useState } from "react"; +import { ShowHealth } from "@/components/dashboard/docker/health/show-health"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogTrigger } from "@/components/ui/dialog"; + +interface Props { + serverId: string; +} + +export const ShowHealthModal = ({ serverId }: Props) => { + const [isOpen, setIsOpen] = useState(false); + + return ( + + + + + +
+ +
+
+
+ ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx index 1fade3f5a..93f4d977e 100644 --- a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx @@ -35,6 +35,7 @@ import { TerminalModal } from "../web-server/terminal-modal"; import { ShowServerActions } from "./actions/show-server-actions"; import { HandleServers } from "./handle-servers"; import { SetupServer } from "./setup-server"; +import { ShowHealthModal } from "./show-health-modal"; import { ShowMonitoringModal } from "./show-monitoring-modal"; import { WelcomeSubscription } from "./welcome-stripe/welcome-subscription"; @@ -324,6 +325,24 @@ export const ShowServers = () => { )} + {permissions?.docker.read && + permissions?.server.read && + server.sshKeyId && + !isBuildServer && ( + + +
+ +
+
+ +

Health

+
+
+ )} +
{permissions?.server.delete && ( diff --git a/apps/dokploy/pages/dashboard/docker.tsx b/apps/dokploy/pages/dashboard/docker.tsx index c73627c77..d2dbba70d 100644 --- a/apps/dokploy/pages/dashboard/docker.tsx +++ b/apps/dokploy/pages/dashboard/docker.tsx @@ -6,6 +6,7 @@ import type { ReactElement } from "react"; import superjson from "superjson"; import { ShowDiskUsage } from "@/components/dashboard/docker/disk-usage/show-disk-usage"; import { ShowDockerEvents } from "@/components/dashboard/docker/events/show-docker-events"; +import { ShowHealth } from "@/components/dashboard/docker/health/show-health"; import { ShowImages } from "@/components/dashboard/docker/images/show-images"; import { ShowContainers } from "@/components/dashboard/docker/show/show-containers"; import { ShowVolumes } from "@/components/dashboard/docker/volumes/show-volumes"; @@ -18,15 +19,25 @@ import { ServerFilter } from "@/components/shared/server-filter"; import { Card } from "@/components/ui/card"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { appRouter } from "@/server/api/root"; +import { api } from "@/utils/api"; const DEFAULT_TAB = "containers"; const DEFAULT_SWARM_TAB = "overview"; const Dashboard = () => { const router = useRouter(); + const { data: isCloud } = api.settings.isCloud.useQuery(); + const { data: permissions } = api.user.getPermissions.useQuery(); + // Host-level diagnostics, so this needs server.read on top of docker.read. + const canSeeHealth = !!permissions?.docker.read && !!permissions?.server.read; - const activeTab = + const queryTab = typeof router.query.tab === "string" ? router.query.tab : DEFAULT_TAB; + const activeTab = + (isCloud && queryTab === "networks") || + (queryTab === "health" && !canSeeHealth) + ? DEFAULT_TAB + : queryTab; const activeSwarmTab = typeof router.query.subtab === "string" @@ -67,9 +78,10 @@ const Dashboard = () => { Swarm Images Volumes - Networks + {!isCloud && Networks} Events Disk Usage + {canSeeHealth && Health} @@ -102,15 +114,22 @@ const Dashboard = () => { - - - + {!isCloud && ( + + + + )} + {canSeeHealth && ( + + + + )} )} diff --git a/apps/dokploy/server/api/routers/deployment.ts b/apps/dokploy/server/api/routers/deployment.ts index dcc74226a..8f19427cf 100644 --- a/apps/dokploy/server/api/routers/deployment.ts +++ b/apps/dokploy/server/api/routers/deployment.ts @@ -6,6 +6,7 @@ import { findAllDeploymentsByServerId, findAllDeploymentsCentralized, findDeploymentById, + findFailedDeploymentsCentralized, findScheduleById, IS_CLOUD, removeDeployment, @@ -14,6 +15,7 @@ import { } from "@dokploy/server"; import { db } from "@dokploy/server/db"; import { + checkPermission, checkServicePermissionAndAccess, findMemberByUserId, } from "@dokploy/server/services/permission"; @@ -27,6 +29,7 @@ import { apiFindAllByCompose, apiFindAllByServer, apiFindAllByType, + apiFindFailedDeployments, deployments, server, } from "@/server/db/schema"; @@ -78,6 +81,23 @@ export const deploymentRouter = createTRPCRouter({ }, ), + // Reads log tails over SSH, so same host-level gate as docker.getServerHealth. + allFailedCentralized: protectedProcedure + .input(apiFindFailedDeployments) + .query(async ({ ctx, input }) => { + await checkPermission(ctx, { + deployment: ["read"], + docker: ["read"], + server: ["read"], + }); + const orgId = ctx.session.activeOrganizationId; + const accessedServices = + ctx.user.role !== "owner" && ctx.user.role !== "admin" + ? (await findMemberByUserId(ctx.user.id, orgId)).accessedServices + : null; + return findFailedDeploymentsCentralized(orgId, accessedServices, input); + }), + queueList: withPermission("deployment", "read").query(async ({ ctx }) => { const orgId = ctx.session.activeOrganizationId; let rows: QueueJobRow[]; diff --git a/apps/dokploy/server/api/routers/docker.ts b/apps/dokploy/server/api/routers/docker.ts index e025b3434..010b7723f 100644 --- a/apps/dokploy/server/api/routers/docker.ts +++ b/apps/dokploy/server/api/routers/docker.ts @@ -11,6 +11,7 @@ import { getContainersByAppLabel, getContainersByAppNameMatch, getDockerEvents, + getServerHealth as getServerHealthData, getServiceContainersByAppName, getStackContainersByAppName, listContainerFiles, @@ -18,11 +19,12 @@ import { uploadFileToContainer, writeContainerFile, } from "@dokploy/server"; +import { checkPermission } from "@dokploy/server/services/permission"; import { TRPCError } from "@trpc/server"; import { z } from "zod"; import { audit } from "@/server/api/utils/audit"; import { uploadFileToContainerSchema } from "@/utils/schema"; -import { createTRPCRouter, withPermission } from "../trpc"; +import { createTRPCRouter, protectedProcedure, withPermission } from "../trpc"; export const containerIdRegex = /^[a-zA-Z0-9.\-_]+$/; @@ -52,6 +54,29 @@ export const dockerRouter = createTRPCRouter({ return await getContainers(input.serverId); }), + // Host-level diagnostics, so this requires docker.read AND server.read. + getServerHealth: protectedProcedure + .input( + z.object({ + serverId: z.string().optional(), + sinceHours: z.number().int().min(1).max(168).optional(), + }), + ) + .query(async ({ input, ctx }) => { + await checkPermission(ctx, { docker: ["read"], server: ["read"] }); + if (input.serverId) { + const server = await findServerById(input.serverId); + if (server.organizationId !== ctx.session?.activeOrganizationId) { + throw new TRPCError({ code: "UNAUTHORIZED" }); + } + } + return await getServerHealthData( + ctx.session.activeOrganizationId, + input.serverId, + input.sinceHours, + ); + }), + restartContainer: withPermission("docker", "read") .input( z.object({ diff --git a/packages/server/src/db/schema/deployment.ts b/packages/server/src/db/schema/deployment.ts index 0c1b547e0..c3aea8345 100644 --- a/packages/server/src/db/schema/deployment.ts +++ b/packages/server/src/db/schema/deployment.ts @@ -233,3 +233,10 @@ export const apiFindAllByType = z.object({ "volumeBackup", ]), }); + +export const apiFindFailedDeployments = z.object({ + dateFrom: z.string().datetime().optional(), + dateTo: z.string().datetime().optional(), + projectId: z.string().optional(), + environmentId: z.string().optional(), +}); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index b2a0c7f12..3c91bca59 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -50,6 +50,7 @@ export * from "./services/rollbacks"; export * from "./services/schedule"; export * from "./services/security"; export * from "./services/server"; +export * from "./services/server-health"; export * from "./services/settings"; export * from "./services/ssh-key"; export * from "./services/user"; @@ -95,6 +96,7 @@ export * from "./utils/builders/utils"; export * from "./utils/cluster/upload"; export * from "./utils/crons/enterprise"; export * from "./utils/databases/rebuild"; +export * from "./utils/deployment-failure-classifier"; export * from "./utils/docker/collision"; export * from "./utils/docker/compose"; export * from "./utils/docker/compose/configs"; diff --git a/packages/server/src/services/deployment.ts b/packages/server/src/services/deployment.ts index fd61fc106..3d5a5915d 100644 --- a/packages/server/src/services/deployment.ts +++ b/packages/server/src/services/deployment.ts @@ -10,12 +10,14 @@ import { type apiCreateDeploymentSchedule, type apiCreateDeploymentServer, type apiCreateDeploymentVolumeBackup, + type apiFindFailedDeployments, applications, compose, deployments, environments, projects, } from "@dokploy/server/db/schema"; +import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; import { removeDirectoryIfExistsContent } from "@dokploy/server/utils/filesystem/directory"; import { execAsync, @@ -23,7 +25,7 @@ import { } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { format } from "date-fns"; -import { and, desc, eq, inArray, or, sql } from "drizzle-orm"; +import { and, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm"; import type { z } from "zod"; import { type Application, @@ -942,6 +944,185 @@ export const findAllDeploymentsCentralized = async ( }); }; +// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg above to avoid +// touching that existing code path; adds projectId/environmentId filtering. +type FailedDeploymentIdFilters = { projectId?: string; environmentId?: string }; + +async function getApplicationIdsForFailedDeployments( + orgId: string, + accessedServices: string[] | null, + filters: FailedDeploymentIdFilters, +): Promise { + const conditions = [ + eq(projects.organizationId, orgId), + ...(accessedServices !== null + ? [inArray(applications.applicationId, accessedServices)] + : []), + ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), + ...(filters.environmentId + ? [eq(environments.environmentId, filters.environmentId)] + : []), + ]; + const rows = await db + .select({ applicationId: applications.applicationId }) + .from(applications) + .innerJoin( + environments, + eq(applications.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where(and(...conditions)); + return rows.map((r) => r.applicationId); +} + +async function getComposeIdsForFailedDeployments( + orgId: string, + accessedServices: string[] | null, + filters: FailedDeploymentIdFilters, +): Promise { + const conditions = [ + eq(projects.organizationId, orgId), + ...(accessedServices !== null + ? [inArray(compose.composeId, accessedServices)] + : []), + ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), + ...(filters.environmentId + ? [eq(environments.environmentId, filters.environmentId)] + : []), + ]; + const rows = await db + .select({ composeId: compose.composeId }) + .from(compose) + .innerJoin( + environments, + eq(compose.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where(and(...conditions)); + return rows.map((r) => r.composeId); +} + +const readLogTail = async ( + logPath: string | null, + serverId: string | null | undefined, +) => { + if (!logPath) return ""; + const command = `tail -c 4096 "${logPath}" 2>/dev/null || echo ""`; + const { stdout } = serverId + ? await execAsyncRemote(serverId, command) + : await execAsync(command); + return stdout; +}; + +// Caps simultaneous SSH connections when classifying a batch of failures. +const LOG_TAIL_CONCURRENCY = 8; + +async function mapWithConcurrency( + items: T[], + limit: number, + fn: (item: T) => Promise, +): Promise { + const results = new Array(items.length); + let nextIndex = 0; + const worker = async () => { + while (nextIndex < items.length) { + const current = nextIndex++; + results[current] = await fn(items[current] as T); + } + }; + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, worker), + ); + return results; +} + +export type FailedDeploymentFilters = z.infer; + +// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed +// deployments, each tagged with a classifyFailure() guess from its log tail. +export const findFailedDeploymentsCentralized = async ( + orgId: string, + accessedServices: string[] | null, + filters: FailedDeploymentFilters = {}, +) => { + if (accessedServices !== null && accessedServices.length === 0) { + return []; + } + + const idFilters: FailedDeploymentIdFilters = { + projectId: filters.projectId, + environmentId: filters.environmentId, + }; + + const [appIds, compIds] = await Promise.all([ + getApplicationIdsForFailedDeployments(orgId, accessedServices, idFilters), + getComposeIdsForFailedDeployments(orgId, accessedServices, idFilters), + ]); + + if (appIds.length === 0 && compIds.length === 0) { + return []; + } + + // createdAt is stored as text via toISOString(); normalize the filters the + // same way so the string range comparison below stays valid. + const dateFrom = filters.dateFrom + ? new Date(filters.dateFrom).toISOString() + : undefined; + const dateTo = filters.dateTo + ? new Date(filters.dateTo).toISOString() + : undefined; + + const serviceConditions = [ + ...(appIds.length > 0 ? [inArray(deployments.applicationId, appIds)] : []), + ...(compIds.length > 0 ? [inArray(deployments.composeId, compIds)] : []), + ]; + + const conditions = [ + eq(deployments.status, "error"), + serviceConditions.length === 1 + ? serviceConditions[0] + : or(...serviceConditions), + ...(dateFrom ? [gte(deployments.createdAt, dateFrom)] : []), + ...(dateTo ? [lte(deployments.createdAt, dateTo)] : []), + ]; + + const failed = await db.query.deployments.findMany({ + where: and(...conditions), + orderBy: desc(deployments.createdAt), + with: centralizedDeploymentsWith, + limit: 200, + }); + + // Skip further attempts to a server once it's timed out once, so a batch + // of failures on one unreachable host doesn't pay the SSH timeout N times. + const deadServers = new Set(); + return mapWithConcurrency( + failed, + LOG_TAIL_CONCURRENCY, + async (deployment) => { + const serverId = + deployment.buildServer?.serverId || + deployment.server?.serverId || + deployment.application?.server?.serverId || + deployment.compose?.server?.serverId || + null; + let logTail = ""; + if (!serverId || !deadServers.has(serverId)) { + try { + logTail = await readLogTail(deployment.logPath, serverId); + } catch { + if (serverId) deadServers.add(serverId); + } + } + const combined = `${deployment.errorMessage ?? ""}\n${logTail}`; + const classification = classifyFailure(combined); + // Tail end only: the actual failure line is almost always near the end of the log. + const rawError = combined.trim().slice(-1000) || null; + return { ...deployment, classification, rawError }; + }, + ); +}; + export const updateDeployment = async ( deploymentId: string, deploymentData: Partial, diff --git a/packages/server/src/services/server-health.ts b/packages/server/src/services/server-health.ts new file mode 100644 index 000000000..5da66a511 --- /dev/null +++ b/packages/server/src/services/server-health.ts @@ -0,0 +1,386 @@ +import { db } from "@dokploy/server/db"; +import { + applications, + compose, + environments, + projects, +} from "@dokploy/server/db/schema"; +import { + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import { TRPCError } from "@trpc/server"; +import { and, eq, isNull } from "drizzle-orm"; +import { IS_CLOUD } from "../constants"; +import { getRemoteDocker } from "../utils/servers/remote-docker"; + +export interface NetworkIpUsage { + name: string; + driver: string; + subnet: string | null; + containersInUse: number; + capacity: number | null; + percentUsed: number | null; +} + +export interface ServerHealthResult { + checkedAt: string; + containers: { + containerCount: number; + serviceCount: number; + }; + resources: { + memTotalBytes: number; + memUsedBytes: number; + cpuCount: number; + }; + inotify: { + maxWatches: number; + maxInstances: number; + maxQueuedEvents: number; + currentInstances: number; + persisted: boolean; + }; + disk: { + totalBytes: number; + usedBytes: number; + }; + dockerNetworks: { + count: number; + addressPools: unknown | null; + /** Per-network IP utilization, including reserved networks like dokploy-network */ + usage: NetworkIpUsage[]; + /** Set when `usage` couldn't be read, so the UI can tell that apart from "no networks" */ + usageError?: string; + }; + daemonErrors: string[]; + /** The window actually passed to `journalctl --since`, computed on the target machine's clock */ + daemonLogsWindow: { fromEpoch: number; toEpoch: number } | null; + reservation: { + memoryReservedBytes: number; + cpuReservedNanoCpus: number; + appCount: number; + unsupportedComposeCount: number; + } | null; + error?: string; +} + +interface RawHealthOutput { + containerCount?: number | string; + serviceCount?: number | string; + memTotalBytes?: number | string; + memUsedBytes?: number | string; + cpuCount?: number | string; + inotifyMaxWatches?: number | string; + inotifyMaxInstances?: number | string; + inotifyMaxQueuedEvents?: number | string; + inotifyCurrentInstances?: number | string; + inotifyPersistedCount?: number | string; + diskTotalBytes?: number | string; + diskUsedBytes?: number | string; + networkCount?: number | string; + daemonConfigBase64?: string; + daemonErrorsBase64?: string; + daemonLogsFromEpoch?: number | string; + daemonLogsToEpoch?: number | string; +} + +const toInt = (value: unknown): number => { + const n = Number.parseInt(String(value ?? "0"), 10); + return Number.isFinite(n) ? n : 0; +}; + +const b64Decode = (value?: string): string => { + if (!value) return ""; + try { + return Buffer.from(value, "base64").toString("utf-8"); + } catch { + return ""; + } +}; + +// Single read-only script, one SSH/exec round-trip; each check falls back to 0/empty on its own. +const buildHealthScript = (sinceHours: number) => ` +containerCount=$(docker ps -a -q 2>/dev/null | wc -l | tr -d ' ') +serviceCount=$(docker service ls --format '{{.Name}}' 2>/dev/null | wc -l | tr -d ' ') + +memTotal=$(free -b 2>/dev/null | awk '/^Mem:/{print $2}'); [ -z "$memTotal" ] && memTotal=0 +memUsed=$(free -b 2>/dev/null | awk '/^Mem:/{print $3}'); [ -z "$memUsed" ] && memUsed=0 +cpuCount=$(nproc 2>/dev/null); [ -z "$cpuCount" ] && cpuCount=0 + +inotifyMaxWatches=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null); [ -z "$inotifyMaxWatches" ] && inotifyMaxWatches=0 +inotifyMaxInstances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null); [ -z "$inotifyMaxInstances" ] && inotifyMaxInstances=0 +inotifyMaxQueued=$(cat /proc/sys/fs/inotify/max_queued_events 2>/dev/null); [ -z "$inotifyMaxQueued" ] && inotifyMaxQueued=0 +inotifyCurrentInstances=$(find /proc/*/fd -lname 'anon_inode:inotify*' 2>/dev/null | wc -l | tr -d ' ') +inotifyPersistedCount=$(grep -rl inotify /etc/sysctl.conf /etc/sysctl.d/ 2>/dev/null | wc -l | tr -d ' ') + +diskTotal=$(df -B1 / 2>/dev/null | awk 'NR==2{print $2}'); [ -z "$diskTotal" ] && diskTotal=0 +diskUsed=$(df -B1 / 2>/dev/null | awk 'NR==2{print $3}'); [ -z "$diskUsed" ] && diskUsed=0 + +networkCount=$(docker network ls -q 2>/dev/null | wc -l | tr -d ' ') +daemonConfigB64=$(cat /etc/docker/daemon.json 2>/dev/null | base64 2>/dev/null | tr -d '\\n') + +daemonLogsToEpoch=$(date +%s 2>/dev/null); [ -z "$daemonLogsToEpoch" ] && daemonLogsToEpoch=0 +daemonLogsFromEpoch=$((daemonLogsToEpoch - ${sinceHours} * 3600)) + +daemonErrorsB64=$(journalctl -u docker --no-pager --since "${sinceHours} hours ago" 2>/dev/null | grep -iE "inotify|too many open|cannot allocate|oom|conntrack|no space|pids.max|fork:|resource temporarily unavailable|could not find an available ip|no available ip|task allocation failure|address already in use" | tail -n 50 | base64 2>/dev/null | tr -d '\\n') + +printf '{"containerCount":%s,"serviceCount":%s,"memTotalBytes":%s,"memUsedBytes":%s,"cpuCount":%s,"inotifyMaxWatches":%s,"inotifyMaxInstances":%s,"inotifyMaxQueuedEvents":%s,"inotifyCurrentInstances":%s,"inotifyPersistedCount":%s,"diskTotalBytes":%s,"diskUsedBytes":%s,"networkCount":%s,"daemonConfigBase64":"%s","daemonErrorsBase64":"%s","daemonLogsFromEpoch":%s,"daemonLogsToEpoch":%s}' "$containerCount" "$serviceCount" "$memTotal" "$memUsed" "$cpuCount" "$inotifyMaxWatches" "$inotifyMaxInstances" "$inotifyMaxQueued" "$inotifyCurrentInstances" "$inotifyPersistedCount" "$diskTotal" "$diskUsed" "$networkCount" "$daemonConfigB64" "$daemonErrorsB64" "$daemonLogsFromEpoch" "$daemonLogsToEpoch" +`; + +const emptyResult = (error: unknown): ServerHealthResult => ({ + checkedAt: new Date().toISOString(), + containers: { containerCount: 0, serviceCount: 0 }, + resources: { memTotalBytes: 0, memUsedBytes: 0, cpuCount: 0 }, + inotify: { + maxWatches: 0, + maxInstances: 0, + maxQueuedEvents: 0, + currentInstances: 0, + persisted: false, + }, + disk: { totalBytes: 0, usedBytes: 0 }, + dockerNetworks: { count: 0, addressPools: null, usage: [] }, + daemonErrors: [], + daemonLogsWindow: null, + reservation: null, + error: + error instanceof Error ? error.message : "Could not read server health", +}); + +/** Usable host addresses in a subnet, excluding network + broadcast. Null if the subnet is missing/invalid. */ +export const getSubnetCapacity = ( + subnet: string | undefined, +): number | null => { + if (!subnet) return null; + const prefix = Number.parseInt(subnet.split("/")[1] ?? "", 10); + if (Number.isNaN(prefix) || prefix < 0 || prefix > 32) return null; + const hostBits = 32 - prefix; + if (hostBits <= 1) return 0; + return 2 ** hostBits - 2; +}; + +type DockerNetworkSummary = { + Name: string; + Driver: string; + IPAM?: { Config?: Array<{ Subnet?: string }> | null }; +}; + +type InspectableNetwork = { + inspect: (opts?: { verbose?: boolean }) => Promise<{ + Containers?: Record; + }>; +}; + +// Includes reserved networks like dokploy-network, which are excluded from the `network` table/UI. +const getNetworksIpUsage = async ( + serverId: string | undefined, +): Promise => { + const docker = await getRemoteDocker(serverId); + const dockerNetworks = + (await docker.listNetworks()) as DockerNetworkSummary[]; + const relevant = dockerNetworks.filter( + (n) => n.Driver === "bridge" || n.Driver === "overlay", + ); + + const rows = await Promise.all( + relevant.map(async (n): Promise => { + // IPv4 entry specifically: dual-stack networks list an IPv6 config too, and it may come first. + const subnet = n.IPAM?.Config?.find( + (c) => c.Subnet && !c.Subnet.includes(":"), + )?.Subnet; + const capacity = getSubnetCapacity(subnet); + let containersInUse = 0; + try { + // Cast the object (not the detached method) so `net.inspect(...)` stays a bound method call. + const net = docker.getNetwork(n.Name) as unknown as InspectableNetwork; + // verbose: true for full swarm-wide container list on overlay networks + const info = await net.inspect( + n.Driver === "overlay" ? { verbose: true } : undefined, + ); + containersInUse = Object.keys(info.Containers ?? {}).length; + } catch { + // Network may have disappeared between list and inspect; report 0 + } + return { + name: n.Name, + driver: n.Driver, + subnet: subnet ?? null, + containersInUse, + capacity, + percentUsed: + capacity && capacity > 0 + ? Math.round((containersInUse / capacity) * 100) + : null, + }; + }), + ); + + return rows.sort((a, b) => (b.percentUsed ?? -1) - (a.percentUsed ?? -1)); +}; + +// Sums Application memoryReservation/cpuReservation (bytes/NanoCPUs, see +// calculateResources in utils/docker/utils.ts). Compose has no such columns. +export const getReservationSummary = async ( + orgId: string, + serverId?: string, +) => { + const appRows = await db + .select({ + memoryReservation: applications.memoryReservation, + cpuReservation: applications.cpuReservation, + }) + .from(applications) + .innerJoin( + environments, + eq(applications.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where( + and( + eq(projects.organizationId, orgId), + serverId + ? eq(applications.serverId, serverId) + : isNull(applications.serverId), + ), + ); + + let memoryReservedBytes = 0; + let cpuReservedNanoCpus = 0; + for (const row of appRows) { + if (row.memoryReservation) { + memoryReservedBytes += toInt(row.memoryReservation); + } + if (row.cpuReservation) { + cpuReservedNanoCpus += toInt(row.cpuReservation); + } + } + + const composeRows = await db + .select({ composeId: compose.composeId }) + .from(compose) + .innerJoin( + environments, + eq(compose.environmentId, environments.environmentId), + ) + .innerJoin(projects, eq(environments.projectId, projects.projectId)) + .where( + and( + eq(projects.organizationId, orgId), + serverId ? eq(compose.serverId, serverId) : isNull(compose.serverId), + ), + ); + + return { + memoryReservedBytes, + cpuReservedNanoCpus, + appCount: appRows.length, + unsupportedComposeCount: composeRows.length, + }; +}; + +export const getServerHealth = async ( + orgId: string, + serverId?: string, + sinceHours = 24, +): Promise => { + if (IS_CLOUD && !serverId) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Server is required", + }); + } + + const hours = Math.max( + 1, + Math.min(168, Math.floor(Number(sinceHours) || 24)), + ); + const script = buildHealthScript(hours); + + let stdout = ""; + try { + const result = serverId + ? await execAsyncRemote(serverId, script) + : await execAsync(script); + stdout = result.stdout; + } catch (error) { + return emptyResult(error); + } + + let parsed: RawHealthOutput; + try { + parsed = JSON.parse(stdout.trim()); + } catch { + return emptyResult(new Error("Could not parse server health output")); + } + + const daemonErrors = b64Decode(parsed.daemonErrorsBase64) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + + let addressPools: unknown = null; + const daemonConfigText = b64Decode(parsed.daemonConfigBase64); + if (daemonConfigText) { + try { + addressPools = + JSON.parse(daemonConfigText)?.["default-address-pools"] ?? null; + } catch { + addressPools = null; + } + } + + // Not part of the shell script above: needs structured per-network data from the Docker API. + const [reservation, networkUsageResult] = await Promise.all([ + getReservationSummary(orgId, serverId).catch(() => null), + getNetworksIpUsage(serverId).then( + (usage) => ({ usage, error: undefined }), + (error: unknown) => ({ + usage: [] as NetworkIpUsage[], + error: + error instanceof Error + ? error.message + : "Failed to read network usage", + }), + ), + ]); + + return { + checkedAt: new Date().toISOString(), + containers: { + containerCount: toInt(parsed.containerCount), + serviceCount: toInt(parsed.serviceCount), + }, + resources: { + memTotalBytes: toInt(parsed.memTotalBytes), + memUsedBytes: toInt(parsed.memUsedBytes), + cpuCount: toInt(parsed.cpuCount), + }, + inotify: { + maxWatches: toInt(parsed.inotifyMaxWatches), + maxInstances: toInt(parsed.inotifyMaxInstances), + maxQueuedEvents: toInt(parsed.inotifyMaxQueuedEvents), + currentInstances: toInt(parsed.inotifyCurrentInstances), + persisted: toInt(parsed.inotifyPersistedCount) > 0, + }, + disk: { + totalBytes: toInt(parsed.diskTotalBytes), + usedBytes: toInt(parsed.diskUsedBytes), + }, + dockerNetworks: { + count: toInt(parsed.networkCount), + addressPools, + usage: networkUsageResult.usage, + usageError: networkUsageResult.error, + }, + daemonErrors, + daemonLogsWindow: + toInt(parsed.daemonLogsToEpoch) > 0 + ? { + fromEpoch: toInt(parsed.daemonLogsFromEpoch), + toEpoch: toInt(parsed.daemonLogsToEpoch), + } + : null, + reservation, + }; +}; diff --git a/packages/server/src/utils/deployment-failure-classifier.ts b/packages/server/src/utils/deployment-failure-classifier.ts new file mode 100644 index 000000000..c2d8af883 --- /dev/null +++ b/packages/server/src/utils/deployment-failure-classifier.ts @@ -0,0 +1,69 @@ +export interface FailurePattern { + id: string; + label: string; + pattern: RegExp; +} + +export const FAILURE_PATTERNS: FailurePattern[] = [ + { + id: "inotify-limit", + label: "Inotify limit reached", + pattern: /max_user_instances|max_user_watches|inotify watch limit reached/i, + }, + { + id: "oom", + label: "Out of memory (OOM)", + pattern: /\boom[-_ ]?kill(er|ed)?\b|killed process|out of memory/i, + }, + { + id: "address-pool-exhausted", + label: "Docker network address pool exhausted", + pattern: + /could not find an available[,]? non-overlapping|available.*address pool/i, + }, + { + id: "network-ip-exhausted", + label: "Network out of IP addresses", + pattern: + /could not find an available ip|no available ip addresses|task allocation failure/i, + }, + { + id: "network-attach-timeout", + label: "Network attach timed out", + pattern: + /attaching to network failed|failed to set up container networking/i, + }, + { + id: "disk-full", + label: "Disk space exhausted", + pattern: /no space left on device/i, + }, + { + id: "conntrack-full", + label: "Connection tracking table full", + pattern: /nf_conntrack:?\s*table full/i, + }, + { + id: "fd-limit", + label: "File descriptor limit reached", + pattern: /too many open files/i, + }, + { + id: "pid-limit", + label: "Process/PID limit reached", + pattern: + /pids\.max|cannot allocate memory|fork:.*resource temporarily unavailable/i, + }, +]; + +export const classifyFailure = ( + text: string | null | undefined, +): { id: string; label: string } | null => { + if (!text) return null; + for (const entry of FAILURE_PATTERNS) { + if (entry.pattern.test(text)) { + return { id: entry.id, label: entry.label }; + } + } + return null; +}; From 795065810d98c42f207fab856730762008f6553d Mon Sep 17 00:00:00 2001 From: Narciso Date: Thu, 13 Aug 2026 10:27:02 -0400 Subject: [PATCH 2/5] chore: trim multi-line comments to single lines Tightens a handful of two-line comments left over from earlier iterations (Health tab / failed-deployment classification) down to one line each, no behavior change. --- packages/server/src/services/deployment.ts | 12 ++++-------- packages/server/src/services/server-health.ts | 3 +-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/server/src/services/deployment.ts b/packages/server/src/services/deployment.ts index 3d5a5915d..cf6b3b7b4 100644 --- a/packages/server/src/services/deployment.ts +++ b/packages/server/src/services/deployment.ts @@ -944,8 +944,7 @@ export const findAllDeploymentsCentralized = async ( }); }; -// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg above to avoid -// touching that existing code path; adds projectId/environmentId filtering. +// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg to avoid touching that path. type FailedDeploymentIdFilters = { projectId?: string; environmentId?: string }; async function getApplicationIdsForFailedDeployments( @@ -1038,8 +1037,7 @@ async function mapWithConcurrency( export type FailedDeploymentFilters = z.infer; -// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed -// deployments, each tagged with a classifyFailure() guess from its log tail. +// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed deployments. export const findFailedDeploymentsCentralized = async ( orgId: string, accessedServices: string[] | null, @@ -1063,8 +1061,7 @@ export const findFailedDeploymentsCentralized = async ( return []; } - // createdAt is stored as text via toISOString(); normalize the filters the - // same way so the string range comparison below stays valid. + // createdAt is stored as text via toISOString(); normalize filters the same way. const dateFrom = filters.dateFrom ? new Date(filters.dateFrom).toISOString() : undefined; @@ -1093,8 +1090,7 @@ export const findFailedDeploymentsCentralized = async ( limit: 200, }); - // Skip further attempts to a server once it's timed out once, so a batch - // of failures on one unreachable host doesn't pay the SSH timeout N times. + // Skip a server again once it's timed out, so one dead host doesn't cost N SSH timeouts. const deadServers = new Set(); return mapWithConcurrency( failed, diff --git a/packages/server/src/services/server-health.ts b/packages/server/src/services/server-health.ts index 5da66a511..a7d5a21a2 100644 --- a/packages/server/src/services/server-health.ts +++ b/packages/server/src/services/server-health.ts @@ -219,8 +219,7 @@ const getNetworksIpUsage = async ( return rows.sort((a, b) => (b.percentUsed ?? -1) - (a.percentUsed ?? -1)); }; -// Sums Application memoryReservation/cpuReservation (bytes/NanoCPUs, see -// calculateResources in utils/docker/utils.ts). Compose has no such columns. +// Sums Application memoryReservation/cpuReservation; Compose has no such columns. export const getReservationSummary = async ( orgId: string, serverId?: string, From c741ff36c097be7786ff5453cfb3466ec5d745a7 Mon Sep 17 00:00:00 2001 From: Narciso Date: Thu, 13 Aug 2026 11:15:12 -0400 Subject: [PATCH 3/5] fix: don't auto-run Health diagnostics on mount MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the mount-only auto-refetch — it still ran host shell, Docker, DB, and SSH log-tail work before the user asked for it, just once instead of on every filter change. Back to fully manual: nothing runs until Check/Re-check is clicked, matching the panel's own copy ("Nothing runs automatically — click to check"). --- .../components/dashboard/docker/health/show-health.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/apps/dokploy/components/dashboard/docker/health/show-health.tsx b/apps/dokploy/components/dashboard/docker/health/show-health.tsx index 6d2a5f3c3..9aa134bdc 100644 --- a/apps/dokploy/components/dashboard/docker/health/show-health.tsx +++ b/apps/dokploy/components/dashboard/docker/health/show-health.tsx @@ -142,12 +142,6 @@ export const ShowHealth = ({ serverId }: Props) => { { enabled: false }, ); - // Mount-only: filter changes require an explicit Check click, not an auto-refetch. - useEffect(() => { - refetch(); - refetchFailed(); - }, []); - // Refreshes the default range's "to" bound to now right before checking. const [checkFailedTick, setCheckFailedTick] = useState(0); useEffect(() => { From 95b1afc2c255eebc72b8b6c22f894408195808c3 Mon Sep 17 00:00:00 2001 From: Narciso Date: Thu, 13 Aug 2026 12:03:38 -0400 Subject: [PATCH 4/5] fix: correct log-tail server resolution, drop unrelated Networks gating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Log tail for failed-deployment classification now checks the app's current buildServer before falling back to its server, matching the buildServerId-then-serverId precedence used everywhere else in the codebase (application.ts, drop.ts, directory.ts, patch-repo.ts). - Removes a stale !isCloud gate on the Networks tab that this branch had reintroduced from before Mauricio removed it in daba2d7fb (Networks is intentionally shown in cloud now) — unrelated to this PR's scope, docker.tsx's Networks tab/content now matches canary. --- apps/dokploy/pages/dashboard/docker.tsx | 16 +++++----------- packages/server/src/services/deployment.ts | 2 ++ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/apps/dokploy/pages/dashboard/docker.tsx b/apps/dokploy/pages/dashboard/docker.tsx index d2dbba70d..27a54373a 100644 --- a/apps/dokploy/pages/dashboard/docker.tsx +++ b/apps/dokploy/pages/dashboard/docker.tsx @@ -26,7 +26,6 @@ const DEFAULT_SWARM_TAB = "overview"; const Dashboard = () => { const router = useRouter(); - const { data: isCloud } = api.settings.isCloud.useQuery(); const { data: permissions } = api.user.getPermissions.useQuery(); // Host-level diagnostics, so this needs server.read on top of docker.read. const canSeeHealth = !!permissions?.docker.read && !!permissions?.server.read; @@ -34,10 +33,7 @@ const Dashboard = () => { const queryTab = typeof router.query.tab === "string" ? router.query.tab : DEFAULT_TAB; const activeTab = - (isCloud && queryTab === "networks") || - (queryTab === "health" && !canSeeHealth) - ? DEFAULT_TAB - : queryTab; + queryTab === "health" && !canSeeHealth ? DEFAULT_TAB : queryTab; const activeSwarmTab = typeof router.query.subtab === "string" @@ -78,7 +74,7 @@ const Dashboard = () => { Swarm Images Volumes - {!isCloud && Networks} + Networks Events Disk Usage {canSeeHealth && Health} @@ -114,11 +110,9 @@ const Dashboard = () => { - {!isCloud && ( - - - - )} + + + diff --git a/packages/server/src/services/deployment.ts b/packages/server/src/services/deployment.ts index cf6b3b7b4..846dc26e0 100644 --- a/packages/server/src/services/deployment.ts +++ b/packages/server/src/services/deployment.ts @@ -1096,8 +1096,10 @@ export const findFailedDeploymentsCentralized = async ( failed, LOG_TAIL_CONCURRENCY, async (deployment) => { + // Same buildServerId-then-serverId precedence used app-wide (application.ts, drop.ts, etc.) const serverId = deployment.buildServer?.serverId || + deployment.application?.buildServer?.serverId || deployment.server?.serverId || deployment.application?.server?.serverId || deployment.compose?.server?.serverId || From 171c70492df8783aab51ad0da19f4747eb1ccb33 Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Thu, 13 Aug 2026 15:03:35 -0600 Subject: [PATCH 5/5] refactor: remove deployment failure classifier and related tests --- .../deployment-failure-classifier.test.ts | 55 ---- .../dashboard/docker/health/show-health.tsx | 279 +----------------- apps/dokploy/server/api/routers/deployment.ts | 20 -- packages/server/src/db/schema/deployment.ts | 7 - packages/server/src/index.ts | 1 - packages/server/src/services/deployment.ts | 181 +----------- .../utils/deployment-failure-classifier.ts | 69 ----- 7 files changed, 11 insertions(+), 601 deletions(-) delete mode 100644 apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts delete mode 100644 packages/server/src/utils/deployment-failure-classifier.ts diff --git a/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts b/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts deleted file mode 100644 index e1272d9b1..000000000 --- a/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; -import { describe, expect, test } from "vitest"; - -describe("classifyFailure", () => { - test("returns null for empty/null/undefined input", () => { - expect(classifyFailure(null)).toBeNull(); - expect(classifyFailure(undefined)).toBeNull(); - expect(classifyFailure("")).toBeNull(); - }); - - test("returns null when nothing matches", () => { - expect(classifyFailure("Build succeeded, deploying now")).toBeNull(); - }); - - test.each([ - [ - "inotify-limit", - "Error: ENOSPC: System limit for number of file watchers reached, inotify watch limit reached", - ], - [ - "oom", - "Container killed process, out of memory: Kill process 1234 (node) score", - ], - [ - "address-pool-exhausted", - "Error response from daemon: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network", - ], - [ - "network-ip-exhausted", - 'time="2026-08-12T19:10:14Z" level=error msg="task allocation failure" error="failed to allocate network IP for task kow0rt7j4nk166qill9vgb6gw network cuz8knvfyobvqsp10yvh39iu7: could not find an available IP"', - ], - [ - "network-attach-timeout", - "Error response from daemon: failed to set up container networking: attaching to network failed, make sure your network options are correct and check manager logs: context deadline exceeded", - ], - ["disk-full", "write /var/lib/docker/tmp/x: no space left on device"], - ["conntrack-full", "nf_conntrack: table full, dropping packet"], - ["fd-limit", "accept4: too many open files"], - ["pid-limit", "fork: retry: Resource temporarily unavailable"], - ])("classifies %s pattern", (expectedId, logLine) => { - expect(classifyFailure(logLine)?.id).toBe(expectedId); - }); - - test("network-ip-exhausted and address-pool-exhausted stay distinct", () => { - const poolExhausted = - "could not find an available, non-overlapping IPv4 address pool"; - const ipExhausted = "could not find an available IP"; - expect(classifyFailure(poolExhausted)?.id).toBe("address-pool-exhausted"); - expect(classifyFailure(ipExhausted)?.id).toBe("network-ip-exhausted"); - }); - - test("matches case-insensitively", () => { - expect(classifyFailure("NO SPACE LEFT ON DEVICE")?.id).toBe("disk-full"); - }); -}); diff --git a/apps/dokploy/components/dashboard/docker/health/show-health.tsx b/apps/dokploy/components/dashboard/docker/health/show-health.tsx index 9aa134bdc..003b21c7f 100644 --- a/apps/dokploy/components/dashboard/docker/health/show-health.tsx +++ b/apps/dokploy/components/dashboard/docker/health/show-health.tsx @@ -1,28 +1,18 @@ -import { format } from "date-fns"; import { AlertTriangle, - Calendar as CalendarIcon, Cpu, Download, HardDrive, - Info, Loader2, Network, RefreshCw, Server as ServerIcon, } from "lucide-react"; -import Link from "next/link"; -import { useEffect, useState } from "react"; +import { useState } from "react"; import { CodeEditor } from "@/components/shared/code-editor"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; -import { Calendar } from "@/components/ui/calendar"; import { Card } from "@/components/ui/card"; -import { - Popover, - PopoverContent, - PopoverTrigger, -} from "@/components/ui/popover"; import { Progress } from "@/components/ui/progress"; import { Select, @@ -39,11 +29,6 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from "@/components/ui/tooltip"; import { api } from "@/utils/api"; interface Props { @@ -62,52 +47,6 @@ const SINCE_HOURS_OPTIONS = [ { label: "Last 7 days", value: 168 }, ]; -const getDefaultDateRange = () => { - const to = new Date(); - const from = new Date(); - from.setDate(from.getDate() - 7); - return { from, to }; -}; - -const getServiceInfo = (deployment: { - application?: { - applicationId: string; - name: string; - environment?: { - environmentId: string; - project?: { projectId: string; name: string }; - }; - } | null; - compose?: { - composeId: string; - name: string; - environment?: { - environmentId: string; - project?: { projectId: string; name: string }; - }; - } | null; -}) => { - const app = deployment.application; - const comp = deployment.compose; - if (app?.environment?.project) { - return { - name: app.name, - projectId: app.environment.project.projectId, - projectName: app.environment.project.name, - href: `/dashboard/project/${app.environment.project.projectId}/environment/${app.environment.environmentId}/services/application/${app.applicationId}`, - }; - } - if (comp?.environment?.project) { - return { - name: comp.name, - projectId: comp.environment.project.projectId, - projectName: comp.environment.project.name, - href: `/dashboard/project/${comp.environment.project.projectId}/environment/${comp.environment.environmentId}/services/compose/${comp.composeId}`, - }; - } - return null; -}; - export const ShowHealth = ({ serverId }: Props) => { const [sinceHours, setSinceHours] = useState(24); const { @@ -117,41 +56,9 @@ export const ShowHealth = ({ serverId }: Props) => { isFetched, } = api.docker.getServerHealth.useQuery( { serverId, sinceHours }, - { enabled: false }, + { refetchOnMount: false, refetchOnWindowFocus: false }, ); - const [dateRange, setDateRange] = useState<{ - from: Date | undefined; - to: Date | undefined; - }>(() => getDefaultDateRange()); - const [hasCustomRange, setHasCustomRange] = useState(false); - const [projectId, setProjectId] = useState(undefined); - const { data: projects } = api.project.all.useQuery(); - - const { - data: failedDeployments, - isFetching: isFetchingFailed, - refetch: refetchFailed, - isFetched: isFetchedFailed, - } = api.deployment.allFailedCentralized.useQuery( - { - dateFrom: dateRange.from?.toISOString(), - dateTo: dateRange.to?.toISOString(), - projectId, - }, - { enabled: false }, - ); - - // Refreshes the default range's "to" bound to now right before checking. - const [checkFailedTick, setCheckFailedTick] = useState(0); - useEffect(() => { - if (checkFailedTick > 0) refetchFailed(); - }, [checkFailedTick]); - const handleCheckFailed = () => { - if (!hasCustomRange) setDateRange(getDefaultDateRange()); - setCheckFailedTick((t) => t + 1); - }; - const memUsedPct = health ? pct(health.resources.memUsedBytes, health.resources.memTotalBytes) : 0; @@ -255,20 +162,6 @@ export const ShowHealth = ({ serverId }: Props) => { lines.push(""); } - lines.push("## Failed deployments"); - if (failedDeployments && failedDeployments.length > 0) { - for (const d of failedDeployments) { - const info = getServiceInfo(d); - const cause = d.classification?.label ?? "Unclassified"; - lines.push( - `- ${format(new Date(d.createdAt), "PPp")} | ${info?.name ?? "—"} | ${info?.projectName ?? "—"} | ${cause}`, - ); - } - } else { - lines.push("(none checked, or none in range)"); - } - lines.push(""); - lines.push("## Docker daemon errors (raw)"); lines.push("```"); lines.push(daemonErrorsText); @@ -298,8 +191,8 @@ export const ShowHealth = ({ serverId }: Props) => {

Server diagnostics

Runs a read-only check over SSH (inotify limits, disk, Docker - network pool, daemon errors). Nothing runs automatically — click - to check. + network pool, daemon errors). Runs automatically when you open + this tab — click Re-check to refresh.

@@ -335,12 +228,10 @@ export const ShowHealth = ({ serverId }: Props) => {
- {!isFetched && !isFetching && ( + {isFetching && !health && (
- - - Nothing checked yet — click "Check server health" above. - + + Checking server health…
)} @@ -351,7 +242,9 @@ export const ShowHealth = ({ serverId }: Props) => { )} {health && !health.error && ( -
+
@@ -556,158 +449,6 @@ export const ShowHealth = ({ serverId }: Props) => { )}
- - -
-
-
-

Failed deployments

-

- Deployments that ended in error, with an automatic cause guess - based on the same patterns used to triage failures manually. -

-
-
- - - - - - - { - setHasCustomRange(true); - setDateRange({ from: range?.from, to: range?.to }); - }} - numberOfMonths={2} - /> - - - -
-
- - {!isFetchedFailed && !isFetchingFailed && ( -
- Nothing checked yet. -
- )} - - {isFetchedFailed && (failedDeployments?.length ?? 0) === 0 && ( -
- No failed deployments in this range. -
- )} - - {(failedDeployments?.length ?? 0) > 0 && ( - - - - Date - Service - Project - Likely cause - - - - {failedDeployments?.map((deployment) => { - const info = getServiceInfo(deployment); - return ( - - - {format(new Date(deployment.createdAt), "PPp")} - - - {info ? ( - - {info.name} - - ) : ( - "—" - )} - - - {info ? ( - - {info.projectName} - - ) : ( - "—" - )} - - - - {deployment.classification ? ( - - {deployment.classification.label} - - ) : ( - - Unclassified - - )} - {deployment.rawError && ( - - - - - - {deployment.rawError} - - - )} - - - - ); - })} - -
- )} -
-
); }; diff --git a/apps/dokploy/server/api/routers/deployment.ts b/apps/dokploy/server/api/routers/deployment.ts index 8f19427cf..dcc74226a 100644 --- a/apps/dokploy/server/api/routers/deployment.ts +++ b/apps/dokploy/server/api/routers/deployment.ts @@ -6,7 +6,6 @@ import { findAllDeploymentsByServerId, findAllDeploymentsCentralized, findDeploymentById, - findFailedDeploymentsCentralized, findScheduleById, IS_CLOUD, removeDeployment, @@ -15,7 +14,6 @@ import { } from "@dokploy/server"; import { db } from "@dokploy/server/db"; import { - checkPermission, checkServicePermissionAndAccess, findMemberByUserId, } from "@dokploy/server/services/permission"; @@ -29,7 +27,6 @@ import { apiFindAllByCompose, apiFindAllByServer, apiFindAllByType, - apiFindFailedDeployments, deployments, server, } from "@/server/db/schema"; @@ -81,23 +78,6 @@ export const deploymentRouter = createTRPCRouter({ }, ), - // Reads log tails over SSH, so same host-level gate as docker.getServerHealth. - allFailedCentralized: protectedProcedure - .input(apiFindFailedDeployments) - .query(async ({ ctx, input }) => { - await checkPermission(ctx, { - deployment: ["read"], - docker: ["read"], - server: ["read"], - }); - const orgId = ctx.session.activeOrganizationId; - const accessedServices = - ctx.user.role !== "owner" && ctx.user.role !== "admin" - ? (await findMemberByUserId(ctx.user.id, orgId)).accessedServices - : null; - return findFailedDeploymentsCentralized(orgId, accessedServices, input); - }), - queueList: withPermission("deployment", "read").query(async ({ ctx }) => { const orgId = ctx.session.activeOrganizationId; let rows: QueueJobRow[]; diff --git a/packages/server/src/db/schema/deployment.ts b/packages/server/src/db/schema/deployment.ts index c3aea8345..0c1b547e0 100644 --- a/packages/server/src/db/schema/deployment.ts +++ b/packages/server/src/db/schema/deployment.ts @@ -233,10 +233,3 @@ export const apiFindAllByType = z.object({ "volumeBackup", ]), }); - -export const apiFindFailedDeployments = z.object({ - dateFrom: z.string().datetime().optional(), - dateTo: z.string().datetime().optional(), - projectId: z.string().optional(), - environmentId: z.string().optional(), -}); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 3c91bca59..d0f843204 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -96,7 +96,6 @@ export * from "./utils/builders/utils"; export * from "./utils/cluster/upload"; export * from "./utils/crons/enterprise"; export * from "./utils/databases/rebuild"; -export * from "./utils/deployment-failure-classifier"; export * from "./utils/docker/collision"; export * from "./utils/docker/compose"; export * from "./utils/docker/compose/configs"; diff --git a/packages/server/src/services/deployment.ts b/packages/server/src/services/deployment.ts index 846dc26e0..fd61fc106 100644 --- a/packages/server/src/services/deployment.ts +++ b/packages/server/src/services/deployment.ts @@ -10,14 +10,12 @@ import { type apiCreateDeploymentSchedule, type apiCreateDeploymentServer, type apiCreateDeploymentVolumeBackup, - type apiFindFailedDeployments, applications, compose, deployments, environments, projects, } from "@dokploy/server/db/schema"; -import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; import { removeDirectoryIfExistsContent } from "@dokploy/server/utils/filesystem/directory"; import { execAsync, @@ -25,7 +23,7 @@ import { } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { format } from "date-fns"; -import { and, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, or, sql } from "drizzle-orm"; import type { z } from "zod"; import { type Application, @@ -944,183 +942,6 @@ export const findAllDeploymentsCentralized = async ( }); }; -// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg to avoid touching that path. -type FailedDeploymentIdFilters = { projectId?: string; environmentId?: string }; - -async function getApplicationIdsForFailedDeployments( - orgId: string, - accessedServices: string[] | null, - filters: FailedDeploymentIdFilters, -): Promise { - const conditions = [ - eq(projects.organizationId, orgId), - ...(accessedServices !== null - ? [inArray(applications.applicationId, accessedServices)] - : []), - ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), - ...(filters.environmentId - ? [eq(environments.environmentId, filters.environmentId)] - : []), - ]; - const rows = await db - .select({ applicationId: applications.applicationId }) - .from(applications) - .innerJoin( - environments, - eq(applications.environmentId, environments.environmentId), - ) - .innerJoin(projects, eq(environments.projectId, projects.projectId)) - .where(and(...conditions)); - return rows.map((r) => r.applicationId); -} - -async function getComposeIdsForFailedDeployments( - orgId: string, - accessedServices: string[] | null, - filters: FailedDeploymentIdFilters, -): Promise { - const conditions = [ - eq(projects.organizationId, orgId), - ...(accessedServices !== null - ? [inArray(compose.composeId, accessedServices)] - : []), - ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), - ...(filters.environmentId - ? [eq(environments.environmentId, filters.environmentId)] - : []), - ]; - const rows = await db - .select({ composeId: compose.composeId }) - .from(compose) - .innerJoin( - environments, - eq(compose.environmentId, environments.environmentId), - ) - .innerJoin(projects, eq(environments.projectId, projects.projectId)) - .where(and(...conditions)); - return rows.map((r) => r.composeId); -} - -const readLogTail = async ( - logPath: string | null, - serverId: string | null | undefined, -) => { - if (!logPath) return ""; - const command = `tail -c 4096 "${logPath}" 2>/dev/null || echo ""`; - const { stdout } = serverId - ? await execAsyncRemote(serverId, command) - : await execAsync(command); - return stdout; -}; - -// Caps simultaneous SSH connections when classifying a batch of failures. -const LOG_TAIL_CONCURRENCY = 8; - -async function mapWithConcurrency( - items: T[], - limit: number, - fn: (item: T) => Promise, -): Promise { - const results = new Array(items.length); - let nextIndex = 0; - const worker = async () => { - while (nextIndex < items.length) { - const current = nextIndex++; - results[current] = await fn(items[current] as T); - } - }; - await Promise.all( - Array.from({ length: Math.min(limit, items.length) }, worker), - ); - return results; -} - -export type FailedDeploymentFilters = z.infer; - -// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed deployments. -export const findFailedDeploymentsCentralized = async ( - orgId: string, - accessedServices: string[] | null, - filters: FailedDeploymentFilters = {}, -) => { - if (accessedServices !== null && accessedServices.length === 0) { - return []; - } - - const idFilters: FailedDeploymentIdFilters = { - projectId: filters.projectId, - environmentId: filters.environmentId, - }; - - const [appIds, compIds] = await Promise.all([ - getApplicationIdsForFailedDeployments(orgId, accessedServices, idFilters), - getComposeIdsForFailedDeployments(orgId, accessedServices, idFilters), - ]); - - if (appIds.length === 0 && compIds.length === 0) { - return []; - } - - // createdAt is stored as text via toISOString(); normalize filters the same way. - const dateFrom = filters.dateFrom - ? new Date(filters.dateFrom).toISOString() - : undefined; - const dateTo = filters.dateTo - ? new Date(filters.dateTo).toISOString() - : undefined; - - const serviceConditions = [ - ...(appIds.length > 0 ? [inArray(deployments.applicationId, appIds)] : []), - ...(compIds.length > 0 ? [inArray(deployments.composeId, compIds)] : []), - ]; - - const conditions = [ - eq(deployments.status, "error"), - serviceConditions.length === 1 - ? serviceConditions[0] - : or(...serviceConditions), - ...(dateFrom ? [gte(deployments.createdAt, dateFrom)] : []), - ...(dateTo ? [lte(deployments.createdAt, dateTo)] : []), - ]; - - const failed = await db.query.deployments.findMany({ - where: and(...conditions), - orderBy: desc(deployments.createdAt), - with: centralizedDeploymentsWith, - limit: 200, - }); - - // Skip a server again once it's timed out, so one dead host doesn't cost N SSH timeouts. - const deadServers = new Set(); - return mapWithConcurrency( - failed, - LOG_TAIL_CONCURRENCY, - async (deployment) => { - // Same buildServerId-then-serverId precedence used app-wide (application.ts, drop.ts, etc.) - const serverId = - deployment.buildServer?.serverId || - deployment.application?.buildServer?.serverId || - deployment.server?.serverId || - deployment.application?.server?.serverId || - deployment.compose?.server?.serverId || - null; - let logTail = ""; - if (!serverId || !deadServers.has(serverId)) { - try { - logTail = await readLogTail(deployment.logPath, serverId); - } catch { - if (serverId) deadServers.add(serverId); - } - } - const combined = `${deployment.errorMessage ?? ""}\n${logTail}`; - const classification = classifyFailure(combined); - // Tail end only: the actual failure line is almost always near the end of the log. - const rawError = combined.trim().slice(-1000) || null; - return { ...deployment, classification, rawError }; - }, - ); -}; - export const updateDeployment = async ( deploymentId: string, deploymentData: Partial, diff --git a/packages/server/src/utils/deployment-failure-classifier.ts b/packages/server/src/utils/deployment-failure-classifier.ts deleted file mode 100644 index c2d8af883..000000000 --- a/packages/server/src/utils/deployment-failure-classifier.ts +++ /dev/null @@ -1,69 +0,0 @@ -export interface FailurePattern { - id: string; - label: string; - pattern: RegExp; -} - -export const FAILURE_PATTERNS: FailurePattern[] = [ - { - id: "inotify-limit", - label: "Inotify limit reached", - pattern: /max_user_instances|max_user_watches|inotify watch limit reached/i, - }, - { - id: "oom", - label: "Out of memory (OOM)", - pattern: /\boom[-_ ]?kill(er|ed)?\b|killed process|out of memory/i, - }, - { - id: "address-pool-exhausted", - label: "Docker network address pool exhausted", - pattern: - /could not find an available[,]? non-overlapping|available.*address pool/i, - }, - { - id: "network-ip-exhausted", - label: "Network out of IP addresses", - pattern: - /could not find an available ip|no available ip addresses|task allocation failure/i, - }, - { - id: "network-attach-timeout", - label: "Network attach timed out", - pattern: - /attaching to network failed|failed to set up container networking/i, - }, - { - id: "disk-full", - label: "Disk space exhausted", - pattern: /no space left on device/i, - }, - { - id: "conntrack-full", - label: "Connection tracking table full", - pattern: /nf_conntrack:?\s*table full/i, - }, - { - id: "fd-limit", - label: "File descriptor limit reached", - pattern: /too many open files/i, - }, - { - id: "pid-limit", - label: "Process/PID limit reached", - pattern: - /pids\.max|cannot allocate memory|fork:.*resource temporarily unavailable/i, - }, -]; - -export const classifyFailure = ( - text: string | null | undefined, -): { id: string; label: string } | null => { - if (!text) return null; - for (const entry of FAILURE_PATTERNS) { - if (entry.pattern.test(text)) { - return { id: entry.id, label: entry.label }; - } - } - return null; -};