diff --git a/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts b/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts deleted file mode 100644 index e1272d9b1..000000000 --- a/apps/dokploy/__test__/utils/deployment-failure-classifier.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; -import { describe, expect, test } from "vitest"; - -describe("classifyFailure", () => { - test("returns null for empty/null/undefined input", () => { - expect(classifyFailure(null)).toBeNull(); - expect(classifyFailure(undefined)).toBeNull(); - expect(classifyFailure("")).toBeNull(); - }); - - test("returns null when nothing matches", () => { - expect(classifyFailure("Build succeeded, deploying now")).toBeNull(); - }); - - test.each([ - [ - "inotify-limit", - "Error: ENOSPC: System limit for number of file watchers reached, inotify watch limit reached", - ], - [ - "oom", - "Container killed process, out of memory: Kill process 1234 (node) score", - ], - [ - "address-pool-exhausted", - "Error response from daemon: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network", - ], - [ - "network-ip-exhausted", - 'time="2026-08-12T19:10:14Z" level=error msg="task allocation failure" error="failed to allocate network IP for task kow0rt7j4nk166qill9vgb6gw network cuz8knvfyobvqsp10yvh39iu7: could not find an available IP"', - ], - [ - "network-attach-timeout", - "Error response from daemon: failed to set up container networking: attaching to network failed, make sure your network options are correct and check manager logs: context deadline exceeded", - ], - ["disk-full", "write /var/lib/docker/tmp/x: no space left on device"], - ["conntrack-full", "nf_conntrack: table full, dropping packet"], - ["fd-limit", "accept4: too many open files"], - ["pid-limit", "fork: retry: Resource temporarily unavailable"], - ])("classifies %s pattern", (expectedId, logLine) => { - expect(classifyFailure(logLine)?.id).toBe(expectedId); - }); - - test("network-ip-exhausted and address-pool-exhausted stay distinct", () => { - const poolExhausted = - "could not find an available, non-overlapping IPv4 address pool"; - const ipExhausted = "could not find an available IP"; - expect(classifyFailure(poolExhausted)?.id).toBe("address-pool-exhausted"); - expect(classifyFailure(ipExhausted)?.id).toBe("network-ip-exhausted"); - }); - - test("matches case-insensitively", () => { - expect(classifyFailure("NO SPACE LEFT ON DEVICE")?.id).toBe("disk-full"); - }); -}); diff --git a/apps/dokploy/components/dashboard/docker/health/show-health.tsx b/apps/dokploy/components/dashboard/docker/health/show-health.tsx index 9aa134bdc..003b21c7f 100644 --- a/apps/dokploy/components/dashboard/docker/health/show-health.tsx +++ b/apps/dokploy/components/dashboard/docker/health/show-health.tsx @@ -1,28 +1,18 @@ -import { format } from "date-fns"; import { AlertTriangle, - Calendar as CalendarIcon, Cpu, Download, HardDrive, - Info, Loader2, Network, RefreshCw, Server as ServerIcon, } from "lucide-react"; -import Link from "next/link"; -import { useEffect, useState } from "react"; +import { useState } from "react"; import { CodeEditor } from "@/components/shared/code-editor"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; -import { Calendar } from "@/components/ui/calendar"; import { Card } from "@/components/ui/card"; -import { - Popover, - PopoverContent, - PopoverTrigger, -} from "@/components/ui/popover"; import { Progress } from "@/components/ui/progress"; import { Select, @@ -39,11 +29,6 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from "@/components/ui/tooltip"; import { api } from "@/utils/api"; interface Props { @@ -62,52 +47,6 @@ const SINCE_HOURS_OPTIONS = [ { label: "Last 7 days", value: 168 }, ]; -const getDefaultDateRange = () => { - const to = new Date(); - const from = new Date(); - from.setDate(from.getDate() - 7); - return { from, to }; -}; - -const getServiceInfo = (deployment: { - application?: { - applicationId: string; - name: string; - environment?: { - environmentId: string; - project?: { projectId: string; name: string }; - }; - } | null; - compose?: { - composeId: string; - name: string; - environment?: { - environmentId: string; - project?: { projectId: string; name: string }; - }; - } | null; -}) => { - const app = deployment.application; - const comp = deployment.compose; - if (app?.environment?.project) { - return { - name: app.name, - projectId: app.environment.project.projectId, - projectName: app.environment.project.name, - href: `/dashboard/project/${app.environment.project.projectId}/environment/${app.environment.environmentId}/services/application/${app.applicationId}`, - }; - } - if (comp?.environment?.project) { - return { - name: comp.name, - projectId: comp.environment.project.projectId, - projectName: comp.environment.project.name, - href: `/dashboard/project/${comp.environment.project.projectId}/environment/${comp.environment.environmentId}/services/compose/${comp.composeId}`, - }; - } - return null; -}; - export const ShowHealth = ({ serverId }: Props) => { const [sinceHours, setSinceHours] = useState(24); const { @@ -117,41 +56,9 @@ export const ShowHealth = ({ serverId }: Props) => { isFetched, } = api.docker.getServerHealth.useQuery( { serverId, sinceHours }, - { enabled: false }, + { refetchOnMount: false, refetchOnWindowFocus: false }, ); - const [dateRange, setDateRange] = useState<{ - from: Date | undefined; - to: Date | undefined; - }>(() => getDefaultDateRange()); - const [hasCustomRange, setHasCustomRange] = useState(false); - const [projectId, setProjectId] = useState(undefined); - const { data: projects } = api.project.all.useQuery(); - - const { - data: failedDeployments, - isFetching: isFetchingFailed, - refetch: refetchFailed, - isFetched: isFetchedFailed, - } = api.deployment.allFailedCentralized.useQuery( - { - dateFrom: dateRange.from?.toISOString(), - dateTo: dateRange.to?.toISOString(), - projectId, - }, - { enabled: false }, - ); - - // Refreshes the default range's "to" bound to now right before checking. - const [checkFailedTick, setCheckFailedTick] = useState(0); - useEffect(() => { - if (checkFailedTick > 0) refetchFailed(); - }, [checkFailedTick]); - const handleCheckFailed = () => { - if (!hasCustomRange) setDateRange(getDefaultDateRange()); - setCheckFailedTick((t) => t + 1); - }; - const memUsedPct = health ? pct(health.resources.memUsedBytes, health.resources.memTotalBytes) : 0; @@ -255,20 +162,6 @@ export const ShowHealth = ({ serverId }: Props) => { lines.push(""); } - lines.push("## Failed deployments"); - if (failedDeployments && failedDeployments.length > 0) { - for (const d of failedDeployments) { - const info = getServiceInfo(d); - const cause = d.classification?.label ?? "Unclassified"; - lines.push( - `- ${format(new Date(d.createdAt), "PPp")} | ${info?.name ?? "—"} | ${info?.projectName ?? "—"} | ${cause}`, - ); - } - } else { - lines.push("(none checked, or none in range)"); - } - lines.push(""); - lines.push("## Docker daemon errors (raw)"); lines.push("```"); lines.push(daemonErrorsText); @@ -298,8 +191,8 @@ export const ShowHealth = ({ serverId }: Props) => {

Server diagnostics

Runs a read-only check over SSH (inotify limits, disk, Docker - network pool, daemon errors). Nothing runs automatically — click - to check. + network pool, daemon errors). Runs automatically when you open + this tab — click Re-check to refresh.

@@ -335,12 +228,10 @@ export const ShowHealth = ({ serverId }: Props) => {
- {!isFetched && !isFetching && ( + {isFetching && !health && (
- - - Nothing checked yet — click "Check server health" above. - + + Checking server health…
)} @@ -351,7 +242,9 @@ export const ShowHealth = ({ serverId }: Props) => { )} {health && !health.error && ( -
+
@@ -556,158 +449,6 @@ export const ShowHealth = ({ serverId }: Props) => { )}
- - -
-
-
-

Failed deployments

-

- Deployments that ended in error, with an automatic cause guess - based on the same patterns used to triage failures manually. -

-
-
- - - - - - - { - setHasCustomRange(true); - setDateRange({ from: range?.from, to: range?.to }); - }} - numberOfMonths={2} - /> - - - -
-
- - {!isFetchedFailed && !isFetchingFailed && ( -
- Nothing checked yet. -
- )} - - {isFetchedFailed && (failedDeployments?.length ?? 0) === 0 && ( -
- No failed deployments in this range. -
- )} - - {(failedDeployments?.length ?? 0) > 0 && ( - - - - Date - Service - Project - Likely cause - - - - {failedDeployments?.map((deployment) => { - const info = getServiceInfo(deployment); - return ( - - - {format(new Date(deployment.createdAt), "PPp")} - - - {info ? ( - - {info.name} - - ) : ( - "—" - )} - - - {info ? ( - - {info.projectName} - - ) : ( - "—" - )} - - - - {deployment.classification ? ( - - {deployment.classification.label} - - ) : ( - - Unclassified - - )} - {deployment.rawError && ( - - - - - - {deployment.rawError} - - - )} - - - - ); - })} - -
- )} -
-
); }; diff --git a/apps/dokploy/server/api/routers/deployment.ts b/apps/dokploy/server/api/routers/deployment.ts index 8f19427cf..dcc74226a 100644 --- a/apps/dokploy/server/api/routers/deployment.ts +++ b/apps/dokploy/server/api/routers/deployment.ts @@ -6,7 +6,6 @@ import { findAllDeploymentsByServerId, findAllDeploymentsCentralized, findDeploymentById, - findFailedDeploymentsCentralized, findScheduleById, IS_CLOUD, removeDeployment, @@ -15,7 +14,6 @@ import { } from "@dokploy/server"; import { db } from "@dokploy/server/db"; import { - checkPermission, checkServicePermissionAndAccess, findMemberByUserId, } from "@dokploy/server/services/permission"; @@ -29,7 +27,6 @@ import { apiFindAllByCompose, apiFindAllByServer, apiFindAllByType, - apiFindFailedDeployments, deployments, server, } from "@/server/db/schema"; @@ -81,23 +78,6 @@ export const deploymentRouter = createTRPCRouter({ }, ), - // Reads log tails over SSH, so same host-level gate as docker.getServerHealth. - allFailedCentralized: protectedProcedure - .input(apiFindFailedDeployments) - .query(async ({ ctx, input }) => { - await checkPermission(ctx, { - deployment: ["read"], - docker: ["read"], - server: ["read"], - }); - const orgId = ctx.session.activeOrganizationId; - const accessedServices = - ctx.user.role !== "owner" && ctx.user.role !== "admin" - ? (await findMemberByUserId(ctx.user.id, orgId)).accessedServices - : null; - return findFailedDeploymentsCentralized(orgId, accessedServices, input); - }), - queueList: withPermission("deployment", "read").query(async ({ ctx }) => { const orgId = ctx.session.activeOrganizationId; let rows: QueueJobRow[]; diff --git a/packages/server/src/db/schema/deployment.ts b/packages/server/src/db/schema/deployment.ts index c3aea8345..0c1b547e0 100644 --- a/packages/server/src/db/schema/deployment.ts +++ b/packages/server/src/db/schema/deployment.ts @@ -233,10 +233,3 @@ export const apiFindAllByType = z.object({ "volumeBackup", ]), }); - -export const apiFindFailedDeployments = z.object({ - dateFrom: z.string().datetime().optional(), - dateTo: z.string().datetime().optional(), - projectId: z.string().optional(), - environmentId: z.string().optional(), -}); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 3c91bca59..d0f843204 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -96,7 +96,6 @@ export * from "./utils/builders/utils"; export * from "./utils/cluster/upload"; export * from "./utils/crons/enterprise"; export * from "./utils/databases/rebuild"; -export * from "./utils/deployment-failure-classifier"; export * from "./utils/docker/collision"; export * from "./utils/docker/compose"; export * from "./utils/docker/compose/configs"; diff --git a/packages/server/src/services/deployment.ts b/packages/server/src/services/deployment.ts index 846dc26e0..fd61fc106 100644 --- a/packages/server/src/services/deployment.ts +++ b/packages/server/src/services/deployment.ts @@ -10,14 +10,12 @@ import { type apiCreateDeploymentSchedule, type apiCreateDeploymentServer, type apiCreateDeploymentVolumeBackup, - type apiFindFailedDeployments, applications, compose, deployments, environments, projects, } from "@dokploy/server/db/schema"; -import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier"; import { removeDirectoryIfExistsContent } from "@dokploy/server/utils/filesystem/directory"; import { execAsync, @@ -25,7 +23,7 @@ import { } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { format } from "date-fns"; -import { and, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, or, sql } from "drizzle-orm"; import type { z } from "zod"; import { type Application, @@ -944,183 +942,6 @@ export const findAllDeploymentsCentralized = async ( }); }; -// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg to avoid touching that path. -type FailedDeploymentIdFilters = { projectId?: string; environmentId?: string }; - -async function getApplicationIdsForFailedDeployments( - orgId: string, - accessedServices: string[] | null, - filters: FailedDeploymentIdFilters, -): Promise { - const conditions = [ - eq(projects.organizationId, orgId), - ...(accessedServices !== null - ? [inArray(applications.applicationId, accessedServices)] - : []), - ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), - ...(filters.environmentId - ? [eq(environments.environmentId, filters.environmentId)] - : []), - ]; - const rows = await db - .select({ applicationId: applications.applicationId }) - .from(applications) - .innerJoin( - environments, - eq(applications.environmentId, environments.environmentId), - ) - .innerJoin(projects, eq(environments.projectId, projects.projectId)) - .where(and(...conditions)); - return rows.map((r) => r.applicationId); -} - -async function getComposeIdsForFailedDeployments( - orgId: string, - accessedServices: string[] | null, - filters: FailedDeploymentIdFilters, -): Promise { - const conditions = [ - eq(projects.organizationId, orgId), - ...(accessedServices !== null - ? [inArray(compose.composeId, accessedServices)] - : []), - ...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []), - ...(filters.environmentId - ? [eq(environments.environmentId, filters.environmentId)] - : []), - ]; - const rows = await db - .select({ composeId: compose.composeId }) - .from(compose) - .innerJoin( - environments, - eq(compose.environmentId, environments.environmentId), - ) - .innerJoin(projects, eq(environments.projectId, projects.projectId)) - .where(and(...conditions)); - return rows.map((r) => r.composeId); -} - -const readLogTail = async ( - logPath: string | null, - serverId: string | null | undefined, -) => { - if (!logPath) return ""; - const command = `tail -c 4096 "${logPath}" 2>/dev/null || echo ""`; - const { stdout } = serverId - ? await execAsyncRemote(serverId, command) - : await execAsync(command); - return stdout; -}; - -// Caps simultaneous SSH connections when classifying a batch of failures. -const LOG_TAIL_CONCURRENCY = 8; - -async function mapWithConcurrency( - items: T[], - limit: number, - fn: (item: T) => Promise, -): Promise { - const results = new Array(items.length); - let nextIndex = 0; - const worker = async () => { - while (nextIndex < items.length) { - const current = nextIndex++; - results[current] = await fn(items[current] as T); - } - }; - await Promise.all( - Array.from({ length: Math.min(limit, items.length) }, worker), - ); - return results; -} - -export type FailedDeploymentFilters = z.infer; - -// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed deployments. -export const findFailedDeploymentsCentralized = async ( - orgId: string, - accessedServices: string[] | null, - filters: FailedDeploymentFilters = {}, -) => { - if (accessedServices !== null && accessedServices.length === 0) { - return []; - } - - const idFilters: FailedDeploymentIdFilters = { - projectId: filters.projectId, - environmentId: filters.environmentId, - }; - - const [appIds, compIds] = await Promise.all([ - getApplicationIdsForFailedDeployments(orgId, accessedServices, idFilters), - getComposeIdsForFailedDeployments(orgId, accessedServices, idFilters), - ]); - - if (appIds.length === 0 && compIds.length === 0) { - return []; - } - - // createdAt is stored as text via toISOString(); normalize filters the same way. - const dateFrom = filters.dateFrom - ? new Date(filters.dateFrom).toISOString() - : undefined; - const dateTo = filters.dateTo - ? new Date(filters.dateTo).toISOString() - : undefined; - - const serviceConditions = [ - ...(appIds.length > 0 ? [inArray(deployments.applicationId, appIds)] : []), - ...(compIds.length > 0 ? [inArray(deployments.composeId, compIds)] : []), - ]; - - const conditions = [ - eq(deployments.status, "error"), - serviceConditions.length === 1 - ? serviceConditions[0] - : or(...serviceConditions), - ...(dateFrom ? [gte(deployments.createdAt, dateFrom)] : []), - ...(dateTo ? [lte(deployments.createdAt, dateTo)] : []), - ]; - - const failed = await db.query.deployments.findMany({ - where: and(...conditions), - orderBy: desc(deployments.createdAt), - with: centralizedDeploymentsWith, - limit: 200, - }); - - // Skip a server again once it's timed out, so one dead host doesn't cost N SSH timeouts. - const deadServers = new Set(); - return mapWithConcurrency( - failed, - LOG_TAIL_CONCURRENCY, - async (deployment) => { - // Same buildServerId-then-serverId precedence used app-wide (application.ts, drop.ts, etc.) - const serverId = - deployment.buildServer?.serverId || - deployment.application?.buildServer?.serverId || - deployment.server?.serverId || - deployment.application?.server?.serverId || - deployment.compose?.server?.serverId || - null; - let logTail = ""; - if (!serverId || !deadServers.has(serverId)) { - try { - logTail = await readLogTail(deployment.logPath, serverId); - } catch { - if (serverId) deadServers.add(serverId); - } - } - const combined = `${deployment.errorMessage ?? ""}\n${logTail}`; - const classification = classifyFailure(combined); - // Tail end only: the actual failure line is almost always near the end of the log. - const rawError = combined.trim().slice(-1000) || null; - return { ...deployment, classification, rawError }; - }, - ); -}; - export const updateDeployment = async ( deploymentId: string, deploymentData: Partial, diff --git a/packages/server/src/utils/deployment-failure-classifier.ts b/packages/server/src/utils/deployment-failure-classifier.ts deleted file mode 100644 index c2d8af883..000000000 --- a/packages/server/src/utils/deployment-failure-classifier.ts +++ /dev/null @@ -1,69 +0,0 @@ -export interface FailurePattern { - id: string; - label: string; - pattern: RegExp; -} - -export const FAILURE_PATTERNS: FailurePattern[] = [ - { - id: "inotify-limit", - label: "Inotify limit reached", - pattern: /max_user_instances|max_user_watches|inotify watch limit reached/i, - }, - { - id: "oom", - label: "Out of memory (OOM)", - pattern: /\boom[-_ ]?kill(er|ed)?\b|killed process|out of memory/i, - }, - { - id: "address-pool-exhausted", - label: "Docker network address pool exhausted", - pattern: - /could not find an available[,]? non-overlapping|available.*address pool/i, - }, - { - id: "network-ip-exhausted", - label: "Network out of IP addresses", - pattern: - /could not find an available ip|no available ip addresses|task allocation failure/i, - }, - { - id: "network-attach-timeout", - label: "Network attach timed out", - pattern: - /attaching to network failed|failed to set up container networking/i, - }, - { - id: "disk-full", - label: "Disk space exhausted", - pattern: /no space left on device/i, - }, - { - id: "conntrack-full", - label: "Connection tracking table full", - pattern: /nf_conntrack:?\s*table full/i, - }, - { - id: "fd-limit", - label: "File descriptor limit reached", - pattern: /too many open files/i, - }, - { - id: "pid-limit", - label: "Process/PID limit reached", - pattern: - /pids\.max|cannot allocate memory|fork:.*resource temporarily unavailable/i, - }, -]; - -export const classifyFailure = ( - text: string | null | undefined, -): { id: string; label: string } | null => { - if (!text) return null; - for (const entry of FAILURE_PATTERNS) { - if (entry.pattern.test(text)) { - return { id: entry.id, label: entry.label }; - } - } - return null; -};