mirror of
https://github.com/Dokploy/dokploy.git
synced 2026-10-15 13:21:11 +00:00
refactor: remove deployment failure classifier and related tests
This commit is contained in:
Mauricio Siu
@@ -1,55 +0,0 @@
|
||||
import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
describe("classifyFailure", () => {
|
||||
test("returns null for empty/null/undefined input", () => {
|
||||
expect(classifyFailure(null)).toBeNull();
|
||||
expect(classifyFailure(undefined)).toBeNull();
|
||||
expect(classifyFailure("")).toBeNull();
|
||||
});
|
||||
|
||||
test("returns null when nothing matches", () => {
|
||||
expect(classifyFailure("Build succeeded, deploying now")).toBeNull();
|
||||
});
|
||||
|
||||
test.each([
|
||||
[
|
||||
"inotify-limit",
|
||||
"Error: ENOSPC: System limit for number of file watchers reached, inotify watch limit reached",
|
||||
],
|
||||
[
|
||||
"oom",
|
||||
"Container killed process, out of memory: Kill process 1234 (node) score",
|
||||
],
|
||||
[
|
||||
"address-pool-exhausted",
|
||||
"Error response from daemon: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network",
|
||||
],
|
||||
[
|
||||
"network-ip-exhausted",
|
||||
'time="2026-08-12T19:10:14Z" level=error msg="task allocation failure" error="failed to allocate network IP for task kow0rt7j4nk166qill9vgb6gw network cuz8knvfyobvqsp10yvh39iu7: could not find an available IP"',
|
||||
],
|
||||
[
|
||||
"network-attach-timeout",
|
||||
"Error response from daemon: failed to set up container networking: attaching to network failed, make sure your network options are correct and check manager logs: context deadline exceeded",
|
||||
],
|
||||
["disk-full", "write /var/lib/docker/tmp/x: no space left on device"],
|
||||
["conntrack-full", "nf_conntrack: table full, dropping packet"],
|
||||
["fd-limit", "accept4: too many open files"],
|
||||
["pid-limit", "fork: retry: Resource temporarily unavailable"],
|
||||
])("classifies %s pattern", (expectedId, logLine) => {
|
||||
expect(classifyFailure(logLine)?.id).toBe(expectedId);
|
||||
});
|
||||
|
||||
test("network-ip-exhausted and address-pool-exhausted stay distinct", () => {
|
||||
const poolExhausted =
|
||||
"could not find an available, non-overlapping IPv4 address pool";
|
||||
const ipExhausted = "could not find an available IP";
|
||||
expect(classifyFailure(poolExhausted)?.id).toBe("address-pool-exhausted");
|
||||
expect(classifyFailure(ipExhausted)?.id).toBe("network-ip-exhausted");
|
||||
});
|
||||
|
||||
test("matches case-insensitively", () => {
|
||||
expect(classifyFailure("NO SPACE LEFT ON DEVICE")?.id).toBe("disk-full");
|
||||
});
|
||||
});
|
||||
@@ -1,28 +1,18 @@
|
||||
import { format } from "date-fns";
|
||||
import {
|
||||
AlertTriangle,
|
||||
Calendar as CalendarIcon,
|
||||
Cpu,
|
||||
Download,
|
||||
HardDrive,
|
||||
Info,
|
||||
Loader2,
|
||||
Network,
|
||||
RefreshCw,
|
||||
Server as ServerIcon,
|
||||
} from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { useEffect, useState } from "react";
|
||||
import { useState } from "react";
|
||||
import { CodeEditor } from "@/components/shared/code-editor";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Calendar } from "@/components/ui/calendar";
|
||||
import { Card } from "@/components/ui/card";
|
||||
import {
|
||||
Popover,
|
||||
PopoverContent,
|
||||
PopoverTrigger,
|
||||
} from "@/components/ui/popover";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import {
|
||||
Select,
|
||||
@@ -39,11 +29,6 @@ import {
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { api } from "@/utils/api";
|
||||
|
||||
interface Props {
|
||||
@@ -62,52 +47,6 @@ const SINCE_HOURS_OPTIONS = [
|
||||
{ label: "Last 7 days", value: 168 },
|
||||
];
|
||||
|
||||
const getDefaultDateRange = () => {
|
||||
const to = new Date();
|
||||
const from = new Date();
|
||||
from.setDate(from.getDate() - 7);
|
||||
return { from, to };
|
||||
};
|
||||
|
||||
const getServiceInfo = (deployment: {
|
||||
application?: {
|
||||
applicationId: string;
|
||||
name: string;
|
||||
environment?: {
|
||||
environmentId: string;
|
||||
project?: { projectId: string; name: string };
|
||||
};
|
||||
} | null;
|
||||
compose?: {
|
||||
composeId: string;
|
||||
name: string;
|
||||
environment?: {
|
||||
environmentId: string;
|
||||
project?: { projectId: string; name: string };
|
||||
};
|
||||
} | null;
|
||||
}) => {
|
||||
const app = deployment.application;
|
||||
const comp = deployment.compose;
|
||||
if (app?.environment?.project) {
|
||||
return {
|
||||
name: app.name,
|
||||
projectId: app.environment.project.projectId,
|
||||
projectName: app.environment.project.name,
|
||||
href: `/dashboard/project/${app.environment.project.projectId}/environment/${app.environment.environmentId}/services/application/${app.applicationId}`,
|
||||
};
|
||||
}
|
||||
if (comp?.environment?.project) {
|
||||
return {
|
||||
name: comp.name,
|
||||
projectId: comp.environment.project.projectId,
|
||||
projectName: comp.environment.project.name,
|
||||
href: `/dashboard/project/${comp.environment.project.projectId}/environment/${comp.environment.environmentId}/services/compose/${comp.composeId}`,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
export const ShowHealth = ({ serverId }: Props) => {
|
||||
const [sinceHours, setSinceHours] = useState(24);
|
||||
const {
|
||||
@@ -117,41 +56,9 @@ export const ShowHealth = ({ serverId }: Props) => {
|
||||
isFetched,
|
||||
} = api.docker.getServerHealth.useQuery(
|
||||
{ serverId, sinceHours },
|
||||
{ enabled: false },
|
||||
{ refetchOnMount: false, refetchOnWindowFocus: false },
|
||||
);
|
||||
|
||||
const [dateRange, setDateRange] = useState<{
|
||||
from: Date | undefined;
|
||||
to: Date | undefined;
|
||||
}>(() => getDefaultDateRange());
|
||||
const [hasCustomRange, setHasCustomRange] = useState(false);
|
||||
const [projectId, setProjectId] = useState<string | undefined>(undefined);
|
||||
const { data: projects } = api.project.all.useQuery();
|
||||
|
||||
const {
|
||||
data: failedDeployments,
|
||||
isFetching: isFetchingFailed,
|
||||
refetch: refetchFailed,
|
||||
isFetched: isFetchedFailed,
|
||||
} = api.deployment.allFailedCentralized.useQuery(
|
||||
{
|
||||
dateFrom: dateRange.from?.toISOString(),
|
||||
dateTo: dateRange.to?.toISOString(),
|
||||
projectId,
|
||||
},
|
||||
{ enabled: false },
|
||||
);
|
||||
|
||||
// Refreshes the default range's "to" bound to now right before checking.
|
||||
const [checkFailedTick, setCheckFailedTick] = useState(0);
|
||||
useEffect(() => {
|
||||
if (checkFailedTick > 0) refetchFailed();
|
||||
}, [checkFailedTick]);
|
||||
const handleCheckFailed = () => {
|
||||
if (!hasCustomRange) setDateRange(getDefaultDateRange());
|
||||
setCheckFailedTick((t) => t + 1);
|
||||
};
|
||||
|
||||
const memUsedPct = health
|
||||
? pct(health.resources.memUsedBytes, health.resources.memTotalBytes)
|
||||
: 0;
|
||||
@@ -255,20 +162,6 @@ export const ShowHealth = ({ serverId }: Props) => {
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
lines.push("## Failed deployments");
|
||||
if (failedDeployments && failedDeployments.length > 0) {
|
||||
for (const d of failedDeployments) {
|
||||
const info = getServiceInfo(d);
|
||||
const cause = d.classification?.label ?? "Unclassified";
|
||||
lines.push(
|
||||
`- ${format(new Date(d.createdAt), "PPp")} | ${info?.name ?? "—"} | ${info?.projectName ?? "—"} | ${cause}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
lines.push("(none checked, or none in range)");
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
lines.push("## Docker daemon errors (raw)");
|
||||
lines.push("```");
|
||||
lines.push(daemonErrorsText);
|
||||
@@ -298,8 +191,8 @@ export const ShowHealth = ({ serverId }: Props) => {
|
||||
<h3 className="text-lg font-medium">Server diagnostics</h3>
|
||||
<p className="text-sm text-muted-foreground max-w-xl">
|
||||
Runs a read-only check over SSH (inotify limits, disk, Docker
|
||||
network pool, daemon errors). Nothing runs automatically — click
|
||||
to check.
|
||||
network pool, daemon errors). Runs automatically when you open
|
||||
this tab — click Re-check to refresh.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
@@ -335,12 +228,10 @@ export const ShowHealth = ({ serverId }: Props) => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!isFetched && !isFetching && (
|
||||
{isFetching && !health && (
|
||||
<div className="flex flex-col items-center justify-center gap-2 py-12 text-muted-foreground border border-dashed rounded-xl">
|
||||
<ServerIcon className="size-8" />
|
||||
<span>
|
||||
Nothing checked yet — click "Check server health" above.
|
||||
</span>
|
||||
<Loader2 className="size-8 animate-spin" />
|
||||
<span>Checking server health…</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -351,7 +242,9 @@ export const ShowHealth = ({ serverId }: Props) => {
|
||||
)}
|
||||
|
||||
{health && !health.error && (
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||
<div
|
||||
className={`grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4 transition-opacity ${isFetching ? "opacity-50" : ""}`}
|
||||
>
|
||||
<Card className="p-4">
|
||||
<div className="flex items-center gap-2 text-sm font-medium mb-2">
|
||||
<ServerIcon className="size-4 text-muted-foreground" />
|
||||
@@ -556,158 +449,6 @@ export const ShowHealth = ({ serverId }: Props) => {
|
||||
)}
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
<Card className="bg-sidebar p-2.5 rounded-xl w-full">
|
||||
<div className="rounded-xl bg-background shadow-md p-6 flex flex-col gap-4">
|
||||
<div className="flex items-center justify-between gap-4 flex-wrap">
|
||||
<div>
|
||||
<h3 className="text-lg font-medium">Failed deployments</h3>
|
||||
<p className="text-sm text-muted-foreground max-w-xl">
|
||||
Deployments that ended in error, with an automatic cause guess
|
||||
based on the same patterns used to triage failures manually.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<Select
|
||||
value={projectId ?? "all"}
|
||||
onValueChange={(v) => setProjectId(v === "all" ? undefined : v)}
|
||||
>
|
||||
<SelectTrigger className="w-[180px]">
|
||||
<SelectValue placeholder="All projects" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="all">All projects</SelectItem>
|
||||
{projects?.map((project) => (
|
||||
<SelectItem
|
||||
key={project.projectId}
|
||||
value={project.projectId}
|
||||
>
|
||||
{project.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Popover>
|
||||
<PopoverTrigger asChild>
|
||||
<Button
|
||||
variant="outline"
|
||||
className="justify-start text-left font-normal"
|
||||
>
|
||||
<CalendarIcon className="mr-2 h-4 w-4" />
|
||||
{dateRange.from && dateRange.to
|
||||
? `${format(dateRange.from, "LLL dd, y")} - ${format(dateRange.to, "LLL dd, y")}`
|
||||
: "Pick a date range"}
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-auto p-0" align="end">
|
||||
<Calendar
|
||||
mode="range"
|
||||
defaultMonth={dateRange.from}
|
||||
selected={{ from: dateRange.from, to: dateRange.to }}
|
||||
onSelect={(range) => {
|
||||
setHasCustomRange(true);
|
||||
setDateRange({ from: range?.from, to: range?.to });
|
||||
}}
|
||||
numberOfMonths={2}
|
||||
/>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
<Button onClick={handleCheckFailed} disabled={isFetchingFailed}>
|
||||
{isFetchingFailed ? (
|
||||
<Loader2 className="size-4 animate-spin mr-2" />
|
||||
) : (
|
||||
<RefreshCw className="size-4 mr-2" />
|
||||
)}
|
||||
{isFetchedFailed ? "Re-check" : "Check"}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!isFetchedFailed && !isFetchingFailed && (
|
||||
<div className="text-sm text-muted-foreground py-4 text-center border border-dashed rounded-xl">
|
||||
Nothing checked yet.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isFetchedFailed && (failedDeployments?.length ?? 0) === 0 && (
|
||||
<div className="text-sm text-muted-foreground py-4 text-center">
|
||||
No failed deployments in this range.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(failedDeployments?.length ?? 0) > 0 && (
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Date</TableHead>
|
||||
<TableHead>Service</TableHead>
|
||||
<TableHead>Project</TableHead>
|
||||
<TableHead>Likely cause</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{failedDeployments?.map((deployment) => {
|
||||
const info = getServiceInfo(deployment);
|
||||
return (
|
||||
<TableRow key={deployment.deploymentId}>
|
||||
<TableCell className="whitespace-nowrap">
|
||||
{format(new Date(deployment.createdAt), "PPp")}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{info ? (
|
||||
<Link
|
||||
href={info.href}
|
||||
className="text-primary hover:underline"
|
||||
>
|
||||
{info.name}
|
||||
</Link>
|
||||
) : (
|
||||
"—"
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{info ? (
|
||||
<Link
|
||||
href={`/dashboard/project/${info.projectId}`}
|
||||
className="text-muted-foreground hover:underline"
|
||||
>
|
||||
{info.projectName}
|
||||
</Link>
|
||||
) : (
|
||||
"—"
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<span className="flex items-center gap-1.5">
|
||||
{deployment.classification ? (
|
||||
<Badge variant="destructive">
|
||||
{deployment.classification.label}
|
||||
</Badge>
|
||||
) : (
|
||||
<span className="text-muted-foreground text-xs">
|
||||
Unclassified
|
||||
</span>
|
||||
)}
|
||||
{deployment.rawError && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger>
|
||||
<Info className="size-3.5 text-muted-foreground" />
|
||||
</TooltipTrigger>
|
||||
<TooltipContent className="max-w-md whitespace-pre-wrap break-all font-mono text-xs">
|
||||
{deployment.rawError}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
</span>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
);
|
||||
})}
|
||||
</TableBody>
|
||||
</Table>
|
||||
)}
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
findAllDeploymentsByServerId,
|
||||
findAllDeploymentsCentralized,
|
||||
findDeploymentById,
|
||||
findFailedDeploymentsCentralized,
|
||||
findScheduleById,
|
||||
IS_CLOUD,
|
||||
removeDeployment,
|
||||
@@ -15,7 +14,6 @@ import {
|
||||
} from "@dokploy/server";
|
||||
import { db } from "@dokploy/server/db";
|
||||
import {
|
||||
checkPermission,
|
||||
checkServicePermissionAndAccess,
|
||||
findMemberByUserId,
|
||||
} from "@dokploy/server/services/permission";
|
||||
@@ -29,7 +27,6 @@ import {
|
||||
apiFindAllByCompose,
|
||||
apiFindAllByServer,
|
||||
apiFindAllByType,
|
||||
apiFindFailedDeployments,
|
||||
deployments,
|
||||
server,
|
||||
} from "@/server/db/schema";
|
||||
@@ -81,23 +78,6 @@ export const deploymentRouter = createTRPCRouter({
|
||||
},
|
||||
),
|
||||
|
||||
// Reads log tails over SSH, so same host-level gate as docker.getServerHealth.
|
||||
allFailedCentralized: protectedProcedure
|
||||
.input(apiFindFailedDeployments)
|
||||
.query(async ({ ctx, input }) => {
|
||||
await checkPermission(ctx, {
|
||||
deployment: ["read"],
|
||||
docker: ["read"],
|
||||
server: ["read"],
|
||||
});
|
||||
const orgId = ctx.session.activeOrganizationId;
|
||||
const accessedServices =
|
||||
ctx.user.role !== "owner" && ctx.user.role !== "admin"
|
||||
? (await findMemberByUserId(ctx.user.id, orgId)).accessedServices
|
||||
: null;
|
||||
return findFailedDeploymentsCentralized(orgId, accessedServices, input);
|
||||
}),
|
||||
|
||||
queueList: withPermission("deployment", "read").query(async ({ ctx }) => {
|
||||
const orgId = ctx.session.activeOrganizationId;
|
||||
let rows: QueueJobRow[];
|
||||
|
||||
@@ -233,10 +233,3 @@ export const apiFindAllByType = z.object({
|
||||
"volumeBackup",
|
||||
]),
|
||||
});
|
||||
|
||||
export const apiFindFailedDeployments = z.object({
|
||||
dateFrom: z.string().datetime().optional(),
|
||||
dateTo: z.string().datetime().optional(),
|
||||
projectId: z.string().optional(),
|
||||
environmentId: z.string().optional(),
|
||||
});
|
||||
|
||||
@@ -96,7 +96,6 @@ export * from "./utils/builders/utils";
|
||||
export * from "./utils/cluster/upload";
|
||||
export * from "./utils/crons/enterprise";
|
||||
export * from "./utils/databases/rebuild";
|
||||
export * from "./utils/deployment-failure-classifier";
|
||||
export * from "./utils/docker/collision";
|
||||
export * from "./utils/docker/compose";
|
||||
export * from "./utils/docker/compose/configs";
|
||||
|
||||
@@ -10,14 +10,12 @@ import {
|
||||
type apiCreateDeploymentSchedule,
|
||||
type apiCreateDeploymentServer,
|
||||
type apiCreateDeploymentVolumeBackup,
|
||||
type apiFindFailedDeployments,
|
||||
applications,
|
||||
compose,
|
||||
deployments,
|
||||
environments,
|
||||
projects,
|
||||
} from "@dokploy/server/db/schema";
|
||||
import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier";
|
||||
import { removeDirectoryIfExistsContent } from "@dokploy/server/utils/filesystem/directory";
|
||||
import {
|
||||
execAsync,
|
||||
@@ -25,7 +23,7 @@ import {
|
||||
} from "@dokploy/server/utils/process/execAsync";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { format } from "date-fns";
|
||||
import { and, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm";
|
||||
import { and, desc, eq, inArray, or, sql } from "drizzle-orm";
|
||||
import type { z } from "zod";
|
||||
import {
|
||||
type Application,
|
||||
@@ -944,183 +942,6 @@ export const findAllDeploymentsCentralized = async (
|
||||
});
|
||||
};
|
||||
|
||||
// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg to avoid touching that path.
|
||||
type FailedDeploymentIdFilters = { projectId?: string; environmentId?: string };
|
||||
|
||||
async function getApplicationIdsForFailedDeployments(
|
||||
orgId: string,
|
||||
accessedServices: string[] | null,
|
||||
filters: FailedDeploymentIdFilters,
|
||||
): Promise<string[]> {
|
||||
const conditions = [
|
||||
eq(projects.organizationId, orgId),
|
||||
...(accessedServices !== null
|
||||
? [inArray(applications.applicationId, accessedServices)]
|
||||
: []),
|
||||
...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []),
|
||||
...(filters.environmentId
|
||||
? [eq(environments.environmentId, filters.environmentId)]
|
||||
: []),
|
||||
];
|
||||
const rows = await db
|
||||
.select({ applicationId: applications.applicationId })
|
||||
.from(applications)
|
||||
.innerJoin(
|
||||
environments,
|
||||
eq(applications.environmentId, environments.environmentId),
|
||||
)
|
||||
.innerJoin(projects, eq(environments.projectId, projects.projectId))
|
||||
.where(and(...conditions));
|
||||
return rows.map((r) => r.applicationId);
|
||||
}
|
||||
|
||||
async function getComposeIdsForFailedDeployments(
|
||||
orgId: string,
|
||||
accessedServices: string[] | null,
|
||||
filters: FailedDeploymentIdFilters,
|
||||
): Promise<string[]> {
|
||||
const conditions = [
|
||||
eq(projects.organizationId, orgId),
|
||||
...(accessedServices !== null
|
||||
? [inArray(compose.composeId, accessedServices)]
|
||||
: []),
|
||||
...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []),
|
||||
...(filters.environmentId
|
||||
? [eq(environments.environmentId, filters.environmentId)]
|
||||
: []),
|
||||
];
|
||||
const rows = await db
|
||||
.select({ composeId: compose.composeId })
|
||||
.from(compose)
|
||||
.innerJoin(
|
||||
environments,
|
||||
eq(compose.environmentId, environments.environmentId),
|
||||
)
|
||||
.innerJoin(projects, eq(environments.projectId, projects.projectId))
|
||||
.where(and(...conditions));
|
||||
return rows.map((r) => r.composeId);
|
||||
}
|
||||
|
||||
const readLogTail = async (
|
||||
logPath: string | null,
|
||||
serverId: string | null | undefined,
|
||||
) => {
|
||||
if (!logPath) return "";
|
||||
const command = `tail -c 4096 "${logPath}" 2>/dev/null || echo ""`;
|
||||
const { stdout } = serverId
|
||||
? await execAsyncRemote(serverId, command)
|
||||
: await execAsync(command);
|
||||
return stdout;
|
||||
};
|
||||
|
||||
// Caps simultaneous SSH connections when classifying a batch of failures.
|
||||
const LOG_TAIL_CONCURRENCY = 8;
|
||||
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results = new Array<R>(items.length);
|
||||
let nextIndex = 0;
|
||||
const worker = async () => {
|
||||
while (nextIndex < items.length) {
|
||||
const current = nextIndex++;
|
||||
results[current] = await fn(items[current] as T);
|
||||
}
|
||||
};
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(limit, items.length) }, worker),
|
||||
);
|
||||
return results;
|
||||
}
|
||||
|
||||
export type FailedDeploymentFilters = z.infer<typeof apiFindFailedDeployments>;
|
||||
|
||||
// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed deployments.
|
||||
export const findFailedDeploymentsCentralized = async (
|
||||
orgId: string,
|
||||
accessedServices: string[] | null,
|
||||
filters: FailedDeploymentFilters = {},
|
||||
) => {
|
||||
if (accessedServices !== null && accessedServices.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const idFilters: FailedDeploymentIdFilters = {
|
||||
projectId: filters.projectId,
|
||||
environmentId: filters.environmentId,
|
||||
};
|
||||
|
||||
const [appIds, compIds] = await Promise.all([
|
||||
getApplicationIdsForFailedDeployments(orgId, accessedServices, idFilters),
|
||||
getComposeIdsForFailedDeployments(orgId, accessedServices, idFilters),
|
||||
]);
|
||||
|
||||
if (appIds.length === 0 && compIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// createdAt is stored as text via toISOString(); normalize filters the same way.
|
||||
const dateFrom = filters.dateFrom
|
||||
? new Date(filters.dateFrom).toISOString()
|
||||
: undefined;
|
||||
const dateTo = filters.dateTo
|
||||
? new Date(filters.dateTo).toISOString()
|
||||
: undefined;
|
||||
|
||||
const serviceConditions = [
|
||||
...(appIds.length > 0 ? [inArray(deployments.applicationId, appIds)] : []),
|
||||
...(compIds.length > 0 ? [inArray(deployments.composeId, compIds)] : []),
|
||||
];
|
||||
|
||||
const conditions = [
|
||||
eq(deployments.status, "error"),
|
||||
serviceConditions.length === 1
|
||||
? serviceConditions[0]
|
||||
: or(...serviceConditions),
|
||||
...(dateFrom ? [gte(deployments.createdAt, dateFrom)] : []),
|
||||
...(dateTo ? [lte(deployments.createdAt, dateTo)] : []),
|
||||
];
|
||||
|
||||
const failed = await db.query.deployments.findMany({
|
||||
where: and(...conditions),
|
||||
orderBy: desc(deployments.createdAt),
|
||||
with: centralizedDeploymentsWith,
|
||||
limit: 200,
|
||||
});
|
||||
|
||||
// Skip a server again once it's timed out, so one dead host doesn't cost N SSH timeouts.
|
||||
const deadServers = new Set<string>();
|
||||
return mapWithConcurrency(
|
||||
failed,
|
||||
LOG_TAIL_CONCURRENCY,
|
||||
async (deployment) => {
|
||||
// Same buildServerId-then-serverId precedence used app-wide (application.ts, drop.ts, etc.)
|
||||
const serverId =
|
||||
deployment.buildServer?.serverId ||
|
||||
deployment.application?.buildServer?.serverId ||
|
||||
deployment.server?.serverId ||
|
||||
deployment.application?.server?.serverId ||
|
||||
deployment.compose?.server?.serverId ||
|
||||
null;
|
||||
let logTail = "";
|
||||
if (!serverId || !deadServers.has(serverId)) {
|
||||
try {
|
||||
logTail = await readLogTail(deployment.logPath, serverId);
|
||||
} catch {
|
||||
if (serverId) deadServers.add(serverId);
|
||||
}
|
||||
}
|
||||
const combined = `${deployment.errorMessage ?? ""}\n${logTail}`;
|
||||
const classification = classifyFailure(combined);
|
||||
// Tail end only: the actual failure line is almost always near the end of the log.
|
||||
const rawError = combined.trim().slice(-1000) || null;
|
||||
return { ...deployment, classification, rawError };
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
export const updateDeployment = async (
|
||||
deploymentId: string,
|
||||
deploymentData: Partial<Deployment>,
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
export interface FailurePattern {
|
||||
id: string;
|
||||
label: string;
|
||||
pattern: RegExp;
|
||||
}
|
||||
|
||||
export const FAILURE_PATTERNS: FailurePattern[] = [
|
||||
{
|
||||
id: "inotify-limit",
|
||||
label: "Inotify limit reached",
|
||||
pattern: /max_user_instances|max_user_watches|inotify watch limit reached/i,
|
||||
},
|
||||
{
|
||||
id: "oom",
|
||||
label: "Out of memory (OOM)",
|
||||
pattern: /\boom[-_ ]?kill(er|ed)?\b|killed process|out of memory/i,
|
||||
},
|
||||
{
|
||||
id: "address-pool-exhausted",
|
||||
label: "Docker network address pool exhausted",
|
||||
pattern:
|
||||
/could not find an available[,]? non-overlapping|available.*address pool/i,
|
||||
},
|
||||
{
|
||||
id: "network-ip-exhausted",
|
||||
label: "Network out of IP addresses",
|
||||
pattern:
|
||||
/could not find an available ip|no available ip addresses|task allocation failure/i,
|
||||
},
|
||||
{
|
||||
id: "network-attach-timeout",
|
||||
label: "Network attach timed out",
|
||||
pattern:
|
||||
/attaching to network failed|failed to set up container networking/i,
|
||||
},
|
||||
{
|
||||
id: "disk-full",
|
||||
label: "Disk space exhausted",
|
||||
pattern: /no space left on device/i,
|
||||
},
|
||||
{
|
||||
id: "conntrack-full",
|
||||
label: "Connection tracking table full",
|
||||
pattern: /nf_conntrack:?\s*table full/i,
|
||||
},
|
||||
{
|
||||
id: "fd-limit",
|
||||
label: "File descriptor limit reached",
|
||||
pattern: /too many open files/i,
|
||||
},
|
||||
{
|
||||
id: "pid-limit",
|
||||
label: "Process/PID limit reached",
|
||||
pattern:
|
||||
/pids\.max|cannot allocate memory|fork:.*resource temporarily unavailable/i,
|
||||
},
|
||||
];
|
||||
|
||||
export const classifyFailure = (
|
||||
text: string | null | undefined,
|
||||
): { id: string; label: string } | null => {
|
||||
if (!text) return null;
|
||||
for (const entry of FAILURE_PATTERNS) {
|
||||
if (entry.pattern.test(text)) {
|
||||
return { id: entry.id, label: entry.label };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
Reference in New Issue
Block a user