mirror of
https://github.com/Dokploy/dokploy.git
synced 2026-10-15 21:31:11 +00:00
feat: add Docker Health diagnostics tab
Server diagnostics tab (Docker dashboard) with:
- Per-network IP usage: subnet capacity vs. containers in use, including
reserved networks like dokploy-network which are excluded from the
managed Networks tab. Surfaces read errors instead of silently showing
zeroes when the Docker API call fails.
- Broadened daemon-error detection and failure classification for Swarm
network failures: IP pool exhaustion on an existing network
("could not find an available IP", "task allocation failure") and
network attach timeouts ("attaching to network failed", "context
deadline exceeded") — both previously fell through as Unclassified.
Verified end-to-end against a real systemd-managed dockerd.
- Failed-deployment rows now carry the raw log tail used for
classification, shown on hover and linked to their service/project,
so a guess (or lack of one) can be verified against the actual error
and followed straight to the app.
- Daemon-error panel is a read-only code editor, always visible (not
hidden when empty), prefixed with the actual "Logs from X to Y"
window passed to journalctl so the --since window can be verified.
- "Download report" exports everything on screen (metrics + raw logs)
as Markdown, so a customer can send one file instead of a screen-share.
- Health check now IS_CLOUD-gated the same way network sync already is:
a cloud org can no longer omit serverId and have the check silently
run against the shared platform host instead of their own server.
Includes an inotify diagnostics card (current vs. max_user_instances/
watches/queued_events) and a failed-deployments log with automatic
cause classification based on daemon/log error patterns.
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
import { getSubnetCapacity } from "@dokploy/server/services/server-health";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
describe("getSubnetCapacity", () => {
|
||||
test("returns null for missing/invalid input", () => {
|
||||
expect(getSubnetCapacity(undefined)).toBeNull();
|
||||
expect(getSubnetCapacity("")).toBeNull();
|
||||
expect(getSubnetCapacity("10.0.0.0")).toBeNull();
|
||||
expect(getSubnetCapacity("not-a-subnet")).toBeNull();
|
||||
expect(getSubnetCapacity("10.0.0.0/33")).toBeNull();
|
||||
expect(getSubnetCapacity("10.0.0.0/-1")).toBeNull();
|
||||
});
|
||||
|
||||
test("excludes network and broadcast addresses", () => {
|
||||
expect(getSubnetCapacity("10.0.1.0/24")).toBe(254);
|
||||
expect(getSubnetCapacity("10.0.0.0/16")).toBe(65534);
|
||||
expect(getSubnetCapacity("10.99.99.0/30")).toBe(2);
|
||||
});
|
||||
|
||||
test("returns 0 for subnets too small to hold a host", () => {
|
||||
expect(getSubnetCapacity("10.0.0.0/31")).toBe(0);
|
||||
expect(getSubnetCapacity("10.0.0.0/32")).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
describe("classifyFailure", () => {
|
||||
test("returns null for empty/null/undefined input", () => {
|
||||
expect(classifyFailure(null)).toBeNull();
|
||||
expect(classifyFailure(undefined)).toBeNull();
|
||||
expect(classifyFailure("")).toBeNull();
|
||||
});
|
||||
|
||||
test("returns null when nothing matches", () => {
|
||||
expect(classifyFailure("Build succeeded, deploying now")).toBeNull();
|
||||
});
|
||||
|
||||
test.each([
|
||||
[
|
||||
"inotify-limit",
|
||||
"Error: ENOSPC: System limit for number of file watchers reached, inotify watch limit reached",
|
||||
],
|
||||
[
|
||||
"oom",
|
||||
"Container killed process, out of memory: Kill process 1234 (node) score",
|
||||
],
|
||||
[
|
||||
"address-pool-exhausted",
|
||||
"Error response from daemon: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network",
|
||||
],
|
||||
[
|
||||
"network-ip-exhausted",
|
||||
'time="2026-08-12T19:10:14Z" level=error msg="task allocation failure" error="failed to allocate network IP for task kow0rt7j4nk166qill9vgb6gw network cuz8knvfyobvqsp10yvh39iu7: could not find an available IP"',
|
||||
],
|
||||
[
|
||||
"network-attach-timeout",
|
||||
"Error response from daemon: failed to set up container networking: attaching to network failed, make sure your network options are correct and check manager logs: context deadline exceeded",
|
||||
],
|
||||
["disk-full", "write /var/lib/docker/tmp/x: no space left on device"],
|
||||
["conntrack-full", "nf_conntrack: table full, dropping packet"],
|
||||
["fd-limit", "accept4: too many open files"],
|
||||
["pid-limit", "fork: retry: Resource temporarily unavailable"],
|
||||
])("classifies %s pattern", (expectedId, logLine) => {
|
||||
expect(classifyFailure(logLine)?.id).toBe(expectedId);
|
||||
});
|
||||
|
||||
test("network-ip-exhausted and address-pool-exhausted stay distinct", () => {
|
||||
const poolExhausted =
|
||||
"could not find an available, non-overlapping IPv4 address pool";
|
||||
const ipExhausted = "could not find an available IP";
|
||||
expect(classifyFailure(poolExhausted)?.id).toBe("address-pool-exhausted");
|
||||
expect(classifyFailure(ipExhausted)?.id).toBe("network-ip-exhausted");
|
||||
});
|
||||
|
||||
test("matches case-insensitively", () => {
|
||||
expect(classifyFailure("NO SPACE LEFT ON DEVICE")?.id).toBe("disk-full");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,719 @@
|
||||
import { format } from "date-fns";
|
||||
import {
|
||||
AlertTriangle,
|
||||
Calendar as CalendarIcon,
|
||||
Cpu,
|
||||
Download,
|
||||
HardDrive,
|
||||
Info,
|
||||
Loader2,
|
||||
Network,
|
||||
RefreshCw,
|
||||
Server as ServerIcon,
|
||||
} from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { useEffect, useState } from "react";
|
||||
import { CodeEditor } from "@/components/shared/code-editor";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Calendar } from "@/components/ui/calendar";
|
||||
import { Card } from "@/components/ui/card";
|
||||
import {
|
||||
Popover,
|
||||
PopoverContent,
|
||||
PopoverTrigger,
|
||||
} from "@/components/ui/popover";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { api } from "@/utils/api";
|
||||
|
||||
interface Props {
|
||||
serverId?: string;
|
||||
}
|
||||
|
||||
const bytesToGb = (bytes: number) => (bytes / 1024 ** 3).toFixed(1);
|
||||
const nanoCpusToCores = (nanoCpus: number) => (nanoCpus / 1e9).toFixed(2);
|
||||
const pct = (used: number, total: number) =>
|
||||
Math.min(100, Math.round((used / (total || 1)) * 100));
|
||||
|
||||
const SINCE_HOURS_OPTIONS = [
|
||||
{ label: "Last hour", value: 1 },
|
||||
{ label: "Last 6 hours", value: 6 },
|
||||
{ label: "Last 24 hours", value: 24 },
|
||||
{ label: "Last 7 days", value: 168 },
|
||||
];
|
||||
|
||||
const getDefaultDateRange = () => {
|
||||
const to = new Date();
|
||||
const from = new Date();
|
||||
from.setDate(from.getDate() - 7);
|
||||
return { from, to };
|
||||
};
|
||||
|
||||
const getServiceInfo = (deployment: {
|
||||
application?: {
|
||||
applicationId: string;
|
||||
name: string;
|
||||
environment?: {
|
||||
environmentId: string;
|
||||
project?: { projectId: string; name: string };
|
||||
};
|
||||
} | null;
|
||||
compose?: {
|
||||
composeId: string;
|
||||
name: string;
|
||||
environment?: {
|
||||
environmentId: string;
|
||||
project?: { projectId: string; name: string };
|
||||
};
|
||||
} | null;
|
||||
}) => {
|
||||
const app = deployment.application;
|
||||
const comp = deployment.compose;
|
||||
if (app?.environment?.project) {
|
||||
return {
|
||||
name: app.name,
|
||||
projectId: app.environment.project.projectId,
|
||||
projectName: app.environment.project.name,
|
||||
href: `/dashboard/project/${app.environment.project.projectId}/environment/${app.environment.environmentId}/services/application/${app.applicationId}`,
|
||||
};
|
||||
}
|
||||
if (comp?.environment?.project) {
|
||||
return {
|
||||
name: comp.name,
|
||||
projectId: comp.environment.project.projectId,
|
||||
projectName: comp.environment.project.name,
|
||||
href: `/dashboard/project/${comp.environment.project.projectId}/environment/${comp.environment.environmentId}/services/compose/${comp.composeId}`,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
export const ShowHealth = ({ serverId }: Props) => {
|
||||
const [sinceHours, setSinceHours] = useState(24);
|
||||
const {
|
||||
data: health,
|
||||
isFetching,
|
||||
refetch,
|
||||
isFetched,
|
||||
} = api.docker.getServerHealth.useQuery(
|
||||
{ serverId, sinceHours },
|
||||
{ enabled: false },
|
||||
);
|
||||
|
||||
const [dateRange, setDateRange] = useState<{
|
||||
from: Date | undefined;
|
||||
to: Date | undefined;
|
||||
}>(() => getDefaultDateRange());
|
||||
const [hasCustomRange, setHasCustomRange] = useState(false);
|
||||
const [projectId, setProjectId] = useState<string | undefined>(undefined);
|
||||
const { data: projects } = api.project.all.useQuery();
|
||||
|
||||
const {
|
||||
data: failedDeployments,
|
||||
isFetching: isFetchingFailed,
|
||||
refetch: refetchFailed,
|
||||
isFetched: isFetchedFailed,
|
||||
} = api.deployment.allFailedCentralized.useQuery(
|
||||
{
|
||||
dateFrom: dateRange.from?.toISOString(),
|
||||
dateTo: dateRange.to?.toISOString(),
|
||||
projectId,
|
||||
},
|
||||
{ enabled: false },
|
||||
);
|
||||
|
||||
// Mount-only: filter changes require an explicit Check click, not an auto-refetch.
|
||||
useEffect(() => {
|
||||
refetch();
|
||||
refetchFailed();
|
||||
}, []);
|
||||
|
||||
// Refreshes the default range's "to" bound to now right before checking.
|
||||
const [checkFailedTick, setCheckFailedTick] = useState(0);
|
||||
useEffect(() => {
|
||||
if (checkFailedTick > 0) refetchFailed();
|
||||
}, [checkFailedTick]);
|
||||
const handleCheckFailed = () => {
|
||||
if (!hasCustomRange) setDateRange(getDefaultDateRange());
|
||||
setCheckFailedTick((t) => t + 1);
|
||||
};
|
||||
|
||||
const memUsedPct = health
|
||||
? pct(health.resources.memUsedBytes, health.resources.memTotalBytes)
|
||||
: 0;
|
||||
const diskUsedPct = health
|
||||
? pct(health.disk.usedBytes, health.disk.totalBytes)
|
||||
: 0;
|
||||
const inotifyUsedPct = health
|
||||
? pct(health.inotify.currentInstances, health.inotify.maxInstances)
|
||||
: 0;
|
||||
|
||||
const daemonLogsWindowText = health?.daemonLogsWindow
|
||||
? `Logs from ${new Date(health.daemonLogsWindow.fromEpoch * 1000).toLocaleString()} to ${new Date(health.daemonLogsWindow.toEpoch * 1000).toLocaleString()}`
|
||||
: null;
|
||||
const daemonErrorsText = [
|
||||
daemonLogsWindowText,
|
||||
health && health.daemonErrors.length > 0
|
||||
? health.daemonErrors.join("\n")
|
||||
: "(no daemon errors matched in this window)",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
|
||||
const buildHealthReport = () => {
|
||||
if (!health) return "";
|
||||
const lines: string[] = [];
|
||||
lines.push("# Dokploy server health report");
|
||||
lines.push(`Generated: ${new Date(health.checkedAt).toLocaleString()}`);
|
||||
lines.push(
|
||||
`Window: ${SINCE_HOURS_OPTIONS.find((o) => o.value === sinceHours)?.label ?? `${sinceHours}h`}`,
|
||||
);
|
||||
lines.push("");
|
||||
|
||||
lines.push("## Containers & services");
|
||||
lines.push(`Containers: ${health.containers.containerCount}`);
|
||||
lines.push(`Swarm services: ${health.containers.serviceCount}`);
|
||||
lines.push("");
|
||||
|
||||
lines.push("## Host resources");
|
||||
lines.push(
|
||||
`Memory: ${bytesToGb(health.resources.memUsedBytes)} / ${bytesToGb(health.resources.memTotalBytes)} GB`,
|
||||
);
|
||||
lines.push(`CPU cores: ${health.resources.cpuCount}`);
|
||||
lines.push("");
|
||||
|
||||
lines.push("## Disk (/)");
|
||||
lines.push(
|
||||
`${bytesToGb(health.disk.usedBytes)} / ${bytesToGb(health.disk.totalBytes)} GB`,
|
||||
);
|
||||
lines.push("");
|
||||
|
||||
lines.push("## Inotify");
|
||||
lines.push(
|
||||
`max_user_instances: ${health.inotify.currentInstances} / ${health.inotify.maxInstances}`,
|
||||
);
|
||||
lines.push(`max_user_watches: ${health.inotify.maxWatches}`);
|
||||
lines.push(`max_queued_events: ${health.inotify.maxQueuedEvents}`);
|
||||
lines.push(
|
||||
`Persisted in sysctl: ${health.inotify.persisted ? "yes" : "no"}`,
|
||||
);
|
||||
lines.push("");
|
||||
|
||||
lines.push("## Docker networks");
|
||||
lines.push(`Total networks: ${health.dockerNetworks.count}`);
|
||||
if (health.dockerNetworks.addressPools) {
|
||||
lines.push(
|
||||
`default-address-pools: ${JSON.stringify(health.dockerNetworks.addressPools)}`,
|
||||
);
|
||||
}
|
||||
if (health.dockerNetworks.usageError) {
|
||||
lines.push(
|
||||
`Could not read network IP usage: ${health.dockerNetworks.usageError}`,
|
||||
);
|
||||
} else if (health.dockerNetworks.usage.length > 0) {
|
||||
lines.push("");
|
||||
lines.push(
|
||||
"| Network | Driver | Subnet | IPs in use | Capacity | Usage |",
|
||||
);
|
||||
lines.push("|---|---|---|---|---|---|");
|
||||
for (const n of health.dockerNetworks.usage) {
|
||||
lines.push(
|
||||
`| ${n.name} | ${n.driver} | ${n.subnet ?? "auto"} | ${n.containersInUse} | ${n.capacity ?? "—"} | ${n.percentUsed ?? "—"}% |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
if (health.reservation) {
|
||||
lines.push("## Memory/CPU reservation");
|
||||
lines.push(
|
||||
`Reserved memory: ${bytesToGb(health.reservation.memoryReservedBytes)} GB`,
|
||||
);
|
||||
lines.push(
|
||||
`Reserved CPU: ${nanoCpusToCores(health.reservation.cpuReservedNanoCpus)} cores`,
|
||||
);
|
||||
lines.push(`Across ${health.reservation.appCount} Application services`);
|
||||
if (health.reservation.unsupportedComposeCount > 0) {
|
||||
lines.push(
|
||||
`${health.reservation.unsupportedComposeCount} Compose service(s) not included (not tracked per-service)`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
lines.push("## Failed deployments");
|
||||
if (failedDeployments && failedDeployments.length > 0) {
|
||||
for (const d of failedDeployments) {
|
||||
const info = getServiceInfo(d);
|
||||
const cause = d.classification?.label ?? "Unclassified";
|
||||
lines.push(
|
||||
`- ${format(new Date(d.createdAt), "PPp")} | ${info?.name ?? "—"} | ${info?.projectName ?? "—"} | ${cause}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
lines.push("(none checked, or none in range)");
|
||||
}
|
||||
lines.push("");
|
||||
|
||||
lines.push("## Docker daemon errors (raw)");
|
||||
lines.push("```");
|
||||
lines.push(daemonErrorsText);
|
||||
lines.push("```");
|
||||
|
||||
return lines.join("\n");
|
||||
};
|
||||
|
||||
const handleDownload = () => {
|
||||
const report = buildHealthReport();
|
||||
if (!report) return;
|
||||
const blob = new Blob([report], { type: "text/markdown" });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = `dokploy-health-report-${new Date().toISOString().replace(/[:.]/g, "-")}.md`;
|
||||
a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-4 w-full">
|
||||
<Card className="bg-sidebar p-2.5 rounded-xl w-full">
|
||||
<div className="rounded-xl bg-background shadow-md p-6 flex flex-col gap-4">
|
||||
<div className="flex items-center justify-between gap-4 flex-wrap">
|
||||
<div>
|
||||
<h3 className="text-lg font-medium">Server diagnostics</h3>
|
||||
<p className="text-sm text-muted-foreground max-w-xl">
|
||||
Runs a read-only check over SSH (inotify limits, disk, Docker
|
||||
network pool, daemon errors). Nothing runs automatically — click
|
||||
to check.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Select
|
||||
value={String(sinceHours)}
|
||||
onValueChange={(v) => setSinceHours(Number(v))}
|
||||
>
|
||||
<SelectTrigger className="w-[160px]">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{SINCE_HOURS_OPTIONS.map((opt) => (
|
||||
<SelectItem key={opt.value} value={String(opt.value)}>
|
||||
{opt.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Button onClick={() => refetch()} disabled={isFetching}>
|
||||
{isFetching ? (
|
||||
<Loader2 className="size-4 animate-spin mr-2" />
|
||||
) : (
|
||||
<RefreshCw className="size-4 mr-2" />
|
||||
)}
|
||||
{isFetched ? "Re-check" : "Check server health"}
|
||||
</Button>
|
||||
{health && !health.error && (
|
||||
<Button variant="outline" onClick={handleDownload}>
|
||||
<Download className="size-4 mr-2" />
|
||||
Download report
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!isFetched && !isFetching && (
|
||||
<div className="flex flex-col items-center justify-center gap-2 py-12 text-muted-foreground border border-dashed rounded-xl">
|
||||
<ServerIcon className="size-8" />
|
||||
<span>
|
||||
Nothing checked yet — click "Check server health" above.
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{health?.error && (
|
||||
<div className="text-sm text-destructive border border-destructive/30 bg-destructive/10 rounded-lg p-3">
|
||||
Couldn't read server health: {health.error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{health && !health.error && (
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||
<Card className="p-4">
|
||||
<div className="flex items-center gap-2 text-sm font-medium mb-2">
|
||||
<ServerIcon className="size-4 text-muted-foreground" />
|
||||
Containers & services
|
||||
</div>
|
||||
<div className="text-2xl font-semibold">
|
||||
{health.containers.containerCount}
|
||||
</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
containers · {health.containers.serviceCount} swarm services
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
<Card className="p-4">
|
||||
<div className="flex items-center gap-2 text-sm font-medium mb-2">
|
||||
<Cpu className="size-4 text-muted-foreground" />
|
||||
Host resources
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
Memory: {bytesToGb(health.resources.memUsedBytes)} /{" "}
|
||||
{bytesToGb(health.resources.memTotalBytes)} GB
|
||||
</div>
|
||||
<Progress value={memUsedPct} className="h-2 mt-1" />
|
||||
<div className="text-xs text-muted-foreground mt-2">
|
||||
{health.resources.cpuCount} CPU cores
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
<Card className="p-4">
|
||||
<div className="flex items-center gap-2 text-sm font-medium mb-2">
|
||||
<HardDrive className="size-4 text-muted-foreground" />
|
||||
Disk (/)
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
{bytesToGb(health.disk.usedBytes)} /{" "}
|
||||
{bytesToGb(health.disk.totalBytes)} GB
|
||||
</div>
|
||||
<Progress value={diskUsedPct} className="h-2 mt-1" />
|
||||
</Card>
|
||||
|
||||
<Card className="p-4">
|
||||
<div className="flex items-center justify-between gap-2 text-sm font-medium mb-2">
|
||||
<span className="flex items-center gap-2">
|
||||
<AlertTriangle className="size-4 text-muted-foreground" />
|
||||
Inotify
|
||||
</span>
|
||||
<Badge
|
||||
variant={health.inotify.persisted ? "default" : "secondary"}
|
||||
>
|
||||
{health.inotify.persisted ? "persisted" : "runtime only"}
|
||||
</Badge>
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
max_user_instances: {health.inotify.currentInstances} /{" "}
|
||||
{health.inotify.maxInstances.toLocaleString()} (
|
||||
{inotifyUsedPct}%)
|
||||
</div>
|
||||
<Progress value={inotifyUsedPct} className="h-2 mt-1" />
|
||||
<div className="text-xs text-muted-foreground mt-2">
|
||||
max_user_watches: {health.inotify.maxWatches.toLocaleString()}{" "}
|
||||
· max_queued_events:{" "}
|
||||
{health.inotify.maxQueuedEvents.toLocaleString()}
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
<Card className="p-4">
|
||||
<div className="flex items-center gap-2 text-sm font-medium mb-2">
|
||||
<Network className="size-4 text-muted-foreground" />
|
||||
Docker networks
|
||||
</div>
|
||||
<div className="text-2xl font-semibold">
|
||||
{health.dockerNetworks.count}
|
||||
</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
The default Docker address pool is exhausted around ~30
|
||||
networks unless <code>default-address-pools</code> is
|
||||
configured in <code>/etc/docker/daemon.json</code>.
|
||||
{health.dockerNetworks.addressPools
|
||||
? " Custom pool detected on this server."
|
||||
: ""}
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
{health.reservation && (
|
||||
<Card className="p-4">
|
||||
<div className="flex items-center gap-2 text-sm font-medium mb-2">
|
||||
<Cpu className="size-4 text-muted-foreground" />
|
||||
Memory/CPU reservation
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
{bytesToGb(health.reservation.memoryReservedBytes)} GB
|
||||
reserved ·{" "}
|
||||
{nanoCpusToCores(health.reservation.cpuReservedNanoCpus)}{" "}
|
||||
CPUs reserved
|
||||
</div>
|
||||
<div className="text-xs text-muted-foreground mt-2">
|
||||
Across {health.reservation.appCount} Application services.
|
||||
{health.reservation.unsupportedComposeCount > 0
|
||||
? ` ${health.reservation.unsupportedComposeCount} Compose service(s) on this server aren't included — reservations aren't tracked per-service for Compose.`
|
||||
: ""}
|
||||
</div>
|
||||
</Card>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{health &&
|
||||
!health.error &&
|
||||
(health.dockerNetworks.usage.length > 0 ||
|
||||
health.dockerNetworks.usageError) && (
|
||||
<div className="flex flex-col gap-2">
|
||||
<h4 className="text-sm font-medium flex items-center gap-2">
|
||||
<Network className="size-4 text-muted-foreground" />
|
||||
Network IP usage
|
||||
</h4>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
IPs assigned to containers vs. the subnet's usable capacity,
|
||||
per network — including reserved networks like{" "}
|
||||
<code>dokploy-network</code>. A network stuck near 100% blocks
|
||||
new containers from starting even when the host has plenty of
|
||||
other resources free.
|
||||
</p>
|
||||
{health.dockerNetworks.usageError ? (
|
||||
<div className="text-sm text-destructive border border-destructive/30 bg-destructive/10 rounded-lg p-3">
|
||||
Couldn't read network IP usage:{" "}
|
||||
{health.dockerNetworks.usageError}
|
||||
</div>
|
||||
) : (
|
||||
<div className="rounded-lg border overflow-x-auto">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Network</TableHead>
|
||||
<TableHead>Driver</TableHead>
|
||||
<TableHead>Subnet</TableHead>
|
||||
<TableHead>IPs in use</TableHead>
|
||||
<TableHead>Usage</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{health.dockerNetworks.usage.map((n) => (
|
||||
<TableRow key={n.name}>
|
||||
<TableCell className="font-medium">
|
||||
{n.name}
|
||||
</TableCell>
|
||||
<TableCell className="text-muted-foreground">
|
||||
{n.driver}
|
||||
</TableCell>
|
||||
<TableCell className="text-muted-foreground">
|
||||
{n.subnet ?? "Auto"}
|
||||
</TableCell>
|
||||
<TableCell className="text-muted-foreground">
|
||||
{n.capacity !== null
|
||||
? `${n.containersInUse} / ${n.capacity}`
|
||||
: n.containersInUse}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{n.percentUsed !== null ? (
|
||||
<Badge
|
||||
variant={
|
||||
n.percentUsed >= 90
|
||||
? "red"
|
||||
: n.percentUsed >= 70
|
||||
? "yellow"
|
||||
: "green"
|
||||
}
|
||||
>
|
||||
{n.percentUsed}%
|
||||
</Badge>
|
||||
) : (
|
||||
<span className="text-muted-foreground text-xs">
|
||||
—
|
||||
</span>
|
||||
)}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{health && !health.error && (
|
||||
<div className="flex flex-col gap-2">
|
||||
<h4 className="text-sm font-medium">
|
||||
Docker daemon errors (
|
||||
{SINCE_HOURS_OPTIONS.find(
|
||||
(o) => o.value === sinceHours,
|
||||
)?.label.toLowerCase()}
|
||||
)
|
||||
</h4>
|
||||
<CodeEditor
|
||||
value={daemonErrorsText}
|
||||
language="shell"
|
||||
disabled
|
||||
lineWrapping
|
||||
wrapperClassName="h-64 rounded-lg border"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
<Card className="bg-sidebar p-2.5 rounded-xl w-full">
|
||||
<div className="rounded-xl bg-background shadow-md p-6 flex flex-col gap-4">
|
||||
<div className="flex items-center justify-between gap-4 flex-wrap">
|
||||
<div>
|
||||
<h3 className="text-lg font-medium">Failed deployments</h3>
|
||||
<p className="text-sm text-muted-foreground max-w-xl">
|
||||
Deployments that ended in error, with an automatic cause guess
|
||||
based on the same patterns used to triage failures manually.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<Select
|
||||
value={projectId ?? "all"}
|
||||
onValueChange={(v) => setProjectId(v === "all" ? undefined : v)}
|
||||
>
|
||||
<SelectTrigger className="w-[180px]">
|
||||
<SelectValue placeholder="All projects" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="all">All projects</SelectItem>
|
||||
{projects?.map((project) => (
|
||||
<SelectItem
|
||||
key={project.projectId}
|
||||
value={project.projectId}
|
||||
>
|
||||
{project.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Popover>
|
||||
<PopoverTrigger asChild>
|
||||
<Button
|
||||
variant="outline"
|
||||
className="justify-start text-left font-normal"
|
||||
>
|
||||
<CalendarIcon className="mr-2 h-4 w-4" />
|
||||
{dateRange.from && dateRange.to
|
||||
? `${format(dateRange.from, "LLL dd, y")} - ${format(dateRange.to, "LLL dd, y")}`
|
||||
: "Pick a date range"}
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-auto p-0" align="end">
|
||||
<Calendar
|
||||
mode="range"
|
||||
defaultMonth={dateRange.from}
|
||||
selected={{ from: dateRange.from, to: dateRange.to }}
|
||||
onSelect={(range) => {
|
||||
setHasCustomRange(true);
|
||||
setDateRange({ from: range?.from, to: range?.to });
|
||||
}}
|
||||
numberOfMonths={2}
|
||||
/>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
<Button onClick={handleCheckFailed} disabled={isFetchingFailed}>
|
||||
{isFetchingFailed ? (
|
||||
<Loader2 className="size-4 animate-spin mr-2" />
|
||||
) : (
|
||||
<RefreshCw className="size-4 mr-2" />
|
||||
)}
|
||||
{isFetchedFailed ? "Re-check" : "Check"}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!isFetchedFailed && !isFetchingFailed && (
|
||||
<div className="text-sm text-muted-foreground py-4 text-center border border-dashed rounded-xl">
|
||||
Nothing checked yet.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isFetchedFailed && (failedDeployments?.length ?? 0) === 0 && (
|
||||
<div className="text-sm text-muted-foreground py-4 text-center">
|
||||
No failed deployments in this range.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(failedDeployments?.length ?? 0) > 0 && (
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Date</TableHead>
|
||||
<TableHead>Service</TableHead>
|
||||
<TableHead>Project</TableHead>
|
||||
<TableHead>Likely cause</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{failedDeployments?.map((deployment) => {
|
||||
const info = getServiceInfo(deployment);
|
||||
return (
|
||||
<TableRow key={deployment.deploymentId}>
|
||||
<TableCell className="whitespace-nowrap">
|
||||
{format(new Date(deployment.createdAt), "PPp")}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{info ? (
|
||||
<Link
|
||||
href={info.href}
|
||||
className="text-primary hover:underline"
|
||||
>
|
||||
{info.name}
|
||||
</Link>
|
||||
) : (
|
||||
"—"
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{info ? (
|
||||
<Link
|
||||
href={`/dashboard/project/${info.projectId}`}
|
||||
className="text-muted-foreground hover:underline"
|
||||
>
|
||||
{info.projectName}
|
||||
</Link>
|
||||
) : (
|
||||
"—"
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<span className="flex items-center gap-1.5">
|
||||
{deployment.classification ? (
|
||||
<Badge variant="destructive">
|
||||
{deployment.classification.label}
|
||||
</Badge>
|
||||
) : (
|
||||
<span className="text-muted-foreground text-xs">
|
||||
Unclassified
|
||||
</span>
|
||||
)}
|
||||
{deployment.rawError && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger>
|
||||
<Info className="size-3.5 text-muted-foreground" />
|
||||
</TooltipTrigger>
|
||||
<TooltipContent className="max-w-md whitespace-pre-wrap break-all font-mono text-xs">
|
||||
{deployment.rawError}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
</span>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
);
|
||||
})}
|
||||
</TableBody>
|
||||
</Table>
|
||||
)}
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Stethoscope } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import { ShowHealth } from "@/components/dashboard/docker/health/show-health";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Dialog, DialogContent, DialogTrigger } from "@/components/ui/dialog";
|
||||
|
||||
interface Props {
|
||||
serverId: string;
|
||||
}
|
||||
|
||||
export const ShowHealthModal = ({ serverId }: Props) => {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
|
||||
return (
|
||||
<Dialog open={isOpen} onOpenChange={setIsOpen}>
|
||||
<DialogTrigger asChild>
|
||||
<Button variant="outline" size="icon" className="h-9 w-9">
|
||||
<Stethoscope className="h-4 w-4" />
|
||||
</Button>
|
||||
</DialogTrigger>
|
||||
<DialogContent className="sm:max-w-5xl max-h-[85vh] overflow-y-auto">
|
||||
<div className="flex gap-4 py-4 w-full">
|
||||
<ShowHealth serverId={serverId} />
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
};
|
||||
@@ -35,6 +35,7 @@ import { TerminalModal } from "../web-server/terminal-modal";
|
||||
import { ShowServerActions } from "./actions/show-server-actions";
|
||||
import { HandleServers } from "./handle-servers";
|
||||
import { SetupServer } from "./setup-server";
|
||||
import { ShowHealthModal } from "./show-health-modal";
|
||||
import { ShowMonitoringModal } from "./show-monitoring-modal";
|
||||
import { WelcomeSubscription } from "./welcome-stripe/welcome-subscription";
|
||||
|
||||
@@ -324,6 +325,24 @@ export const ShowServers = () => {
|
||||
</Tooltip>
|
||||
)}
|
||||
|
||||
{permissions?.docker.read &&
|
||||
permissions?.server.read &&
|
||||
server.sshKeyId &&
|
||||
!isBuildServer && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<div>
|
||||
<ShowHealthModal
|
||||
serverId={server.serverId}
|
||||
/>
|
||||
</div>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p>Health</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
|
||||
<div className="flex-1" />
|
||||
|
||||
{permissions?.server.delete && (
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { ReactElement } from "react";
|
||||
import superjson from "superjson";
|
||||
import { ShowDiskUsage } from "@/components/dashboard/docker/disk-usage/show-disk-usage";
|
||||
import { ShowDockerEvents } from "@/components/dashboard/docker/events/show-docker-events";
|
||||
import { ShowHealth } from "@/components/dashboard/docker/health/show-health";
|
||||
import { ShowImages } from "@/components/dashboard/docker/images/show-images";
|
||||
import { ShowContainers } from "@/components/dashboard/docker/show/show-containers";
|
||||
import { ShowVolumes } from "@/components/dashboard/docker/volumes/show-volumes";
|
||||
@@ -18,15 +19,25 @@ import { ServerFilter } from "@/components/shared/server-filter";
|
||||
import { Card } from "@/components/ui/card";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { appRouter } from "@/server/api/root";
|
||||
import { api } from "@/utils/api";
|
||||
|
||||
const DEFAULT_TAB = "containers";
|
||||
const DEFAULT_SWARM_TAB = "overview";
|
||||
|
||||
const Dashboard = () => {
|
||||
const router = useRouter();
|
||||
const { data: isCloud } = api.settings.isCloud.useQuery();
|
||||
const { data: permissions } = api.user.getPermissions.useQuery();
|
||||
// Host-level diagnostics, so this needs server.read on top of docker.read.
|
||||
const canSeeHealth = !!permissions?.docker.read && !!permissions?.server.read;
|
||||
|
||||
const activeTab =
|
||||
const queryTab =
|
||||
typeof router.query.tab === "string" ? router.query.tab : DEFAULT_TAB;
|
||||
const activeTab =
|
||||
(isCloud && queryTab === "networks") ||
|
||||
(queryTab === "health" && !canSeeHealth)
|
||||
? DEFAULT_TAB
|
||||
: queryTab;
|
||||
|
||||
const activeSwarmTab =
|
||||
typeof router.query.subtab === "string"
|
||||
@@ -67,9 +78,10 @@ const Dashboard = () => {
|
||||
<TabsTrigger value="swarm">Swarm</TabsTrigger>
|
||||
<TabsTrigger value="images">Images</TabsTrigger>
|
||||
<TabsTrigger value="volumes">Volumes</TabsTrigger>
|
||||
<TabsTrigger value="networks">Networks</TabsTrigger>
|
||||
{!isCloud && <TabsTrigger value="networks">Networks</TabsTrigger>}
|
||||
<TabsTrigger value="events">Events</TabsTrigger>
|
||||
<TabsTrigger value="disk-usage">Disk Usage</TabsTrigger>
|
||||
{canSeeHealth && <TabsTrigger value="health">Health</TabsTrigger>}
|
||||
</TabsList>
|
||||
<TabsContent value="containers">
|
||||
<ShowContainers serverId={serverId} />
|
||||
@@ -102,15 +114,22 @@ const Dashboard = () => {
|
||||
<TabsContent value="volumes">
|
||||
<ShowVolumes serverId={serverId} />
|
||||
</TabsContent>
|
||||
<TabsContent value="networks">
|
||||
<ShowNetworks serverId={serverId} />
|
||||
</TabsContent>
|
||||
{!isCloud && (
|
||||
<TabsContent value="networks">
|
||||
<ShowNetworks serverId={serverId} />
|
||||
</TabsContent>
|
||||
)}
|
||||
<TabsContent value="events">
|
||||
<ShowDockerEvents serverId={serverId} />
|
||||
</TabsContent>
|
||||
<TabsContent value="disk-usage">
|
||||
<ShowDiskUsage serverId={serverId} />
|
||||
</TabsContent>
|
||||
{canSeeHealth && (
|
||||
<TabsContent value="health">
|
||||
<ShowHealth serverId={serverId} />
|
||||
</TabsContent>
|
||||
)}
|
||||
</Tabs>
|
||||
)}
|
||||
</ServerFilter>
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
findAllDeploymentsByServerId,
|
||||
findAllDeploymentsCentralized,
|
||||
findDeploymentById,
|
||||
findFailedDeploymentsCentralized,
|
||||
findScheduleById,
|
||||
IS_CLOUD,
|
||||
removeDeployment,
|
||||
@@ -14,6 +15,7 @@ import {
|
||||
} from "@dokploy/server";
|
||||
import { db } from "@dokploy/server/db";
|
||||
import {
|
||||
checkPermission,
|
||||
checkServicePermissionAndAccess,
|
||||
findMemberByUserId,
|
||||
} from "@dokploy/server/services/permission";
|
||||
@@ -27,6 +29,7 @@ import {
|
||||
apiFindAllByCompose,
|
||||
apiFindAllByServer,
|
||||
apiFindAllByType,
|
||||
apiFindFailedDeployments,
|
||||
deployments,
|
||||
server,
|
||||
} from "@/server/db/schema";
|
||||
@@ -78,6 +81,23 @@ export const deploymentRouter = createTRPCRouter({
|
||||
},
|
||||
),
|
||||
|
||||
// Reads log tails over SSH, so same host-level gate as docker.getServerHealth.
|
||||
allFailedCentralized: protectedProcedure
|
||||
.input(apiFindFailedDeployments)
|
||||
.query(async ({ ctx, input }) => {
|
||||
await checkPermission(ctx, {
|
||||
deployment: ["read"],
|
||||
docker: ["read"],
|
||||
server: ["read"],
|
||||
});
|
||||
const orgId = ctx.session.activeOrganizationId;
|
||||
const accessedServices =
|
||||
ctx.user.role !== "owner" && ctx.user.role !== "admin"
|
||||
? (await findMemberByUserId(ctx.user.id, orgId)).accessedServices
|
||||
: null;
|
||||
return findFailedDeploymentsCentralized(orgId, accessedServices, input);
|
||||
}),
|
||||
|
||||
queueList: withPermission("deployment", "read").query(async ({ ctx }) => {
|
||||
const orgId = ctx.session.activeOrganizationId;
|
||||
let rows: QueueJobRow[];
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
getContainersByAppLabel,
|
||||
getContainersByAppNameMatch,
|
||||
getDockerEvents,
|
||||
getServerHealth as getServerHealthData,
|
||||
getServiceContainersByAppName,
|
||||
getStackContainersByAppName,
|
||||
listContainerFiles,
|
||||
@@ -18,11 +19,12 @@ import {
|
||||
uploadFileToContainer,
|
||||
writeContainerFile,
|
||||
} from "@dokploy/server";
|
||||
import { checkPermission } from "@dokploy/server/services/permission";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { z } from "zod";
|
||||
import { audit } from "@/server/api/utils/audit";
|
||||
import { uploadFileToContainerSchema } from "@/utils/schema";
|
||||
import { createTRPCRouter, withPermission } from "../trpc";
|
||||
import { createTRPCRouter, protectedProcedure, withPermission } from "../trpc";
|
||||
|
||||
export const containerIdRegex = /^[a-zA-Z0-9.\-_]+$/;
|
||||
|
||||
@@ -52,6 +54,29 @@ export const dockerRouter = createTRPCRouter({
|
||||
return await getContainers(input.serverId);
|
||||
}),
|
||||
|
||||
// Host-level diagnostics, so this requires docker.read AND server.read.
|
||||
getServerHealth: protectedProcedure
|
||||
.input(
|
||||
z.object({
|
||||
serverId: z.string().optional(),
|
||||
sinceHours: z.number().int().min(1).max(168).optional(),
|
||||
}),
|
||||
)
|
||||
.query(async ({ input, ctx }) => {
|
||||
await checkPermission(ctx, { docker: ["read"], server: ["read"] });
|
||||
if (input.serverId) {
|
||||
const server = await findServerById(input.serverId);
|
||||
if (server.organizationId !== ctx.session?.activeOrganizationId) {
|
||||
throw new TRPCError({ code: "UNAUTHORIZED" });
|
||||
}
|
||||
}
|
||||
return await getServerHealthData(
|
||||
ctx.session.activeOrganizationId,
|
||||
input.serverId,
|
||||
input.sinceHours,
|
||||
);
|
||||
}),
|
||||
|
||||
restartContainer: withPermission("docker", "read")
|
||||
.input(
|
||||
z.object({
|
||||
|
||||
@@ -233,3 +233,10 @@ export const apiFindAllByType = z.object({
|
||||
"volumeBackup",
|
||||
]),
|
||||
});
|
||||
|
||||
export const apiFindFailedDeployments = z.object({
|
||||
dateFrom: z.string().datetime().optional(),
|
||||
dateTo: z.string().datetime().optional(),
|
||||
projectId: z.string().optional(),
|
||||
environmentId: z.string().optional(),
|
||||
});
|
||||
|
||||
@@ -50,6 +50,7 @@ export * from "./services/rollbacks";
|
||||
export * from "./services/schedule";
|
||||
export * from "./services/security";
|
||||
export * from "./services/server";
|
||||
export * from "./services/server-health";
|
||||
export * from "./services/settings";
|
||||
export * from "./services/ssh-key";
|
||||
export * from "./services/user";
|
||||
@@ -95,6 +96,7 @@ export * from "./utils/builders/utils";
|
||||
export * from "./utils/cluster/upload";
|
||||
export * from "./utils/crons/enterprise";
|
||||
export * from "./utils/databases/rebuild";
|
||||
export * from "./utils/deployment-failure-classifier";
|
||||
export * from "./utils/docker/collision";
|
||||
export * from "./utils/docker/compose";
|
||||
export * from "./utils/docker/compose/configs";
|
||||
|
||||
@@ -10,12 +10,14 @@ import {
|
||||
type apiCreateDeploymentSchedule,
|
||||
type apiCreateDeploymentServer,
|
||||
type apiCreateDeploymentVolumeBackup,
|
||||
type apiFindFailedDeployments,
|
||||
applications,
|
||||
compose,
|
||||
deployments,
|
||||
environments,
|
||||
projects,
|
||||
} from "@dokploy/server/db/schema";
|
||||
import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier";
|
||||
import { removeDirectoryIfExistsContent } from "@dokploy/server/utils/filesystem/directory";
|
||||
import {
|
||||
execAsync,
|
||||
@@ -23,7 +25,7 @@ import {
|
||||
} from "@dokploy/server/utils/process/execAsync";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { format } from "date-fns";
|
||||
import { and, desc, eq, inArray, or, sql } from "drizzle-orm";
|
||||
import { and, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm";
|
||||
import type { z } from "zod";
|
||||
import {
|
||||
type Application,
|
||||
@@ -942,6 +944,185 @@ export const findAllDeploymentsCentralized = async (
|
||||
});
|
||||
};
|
||||
|
||||
// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg above to avoid
|
||||
// touching that existing code path; adds projectId/environmentId filtering.
|
||||
type FailedDeploymentIdFilters = { projectId?: string; environmentId?: string };
|
||||
|
||||
async function getApplicationIdsForFailedDeployments(
|
||||
orgId: string,
|
||||
accessedServices: string[] | null,
|
||||
filters: FailedDeploymentIdFilters,
|
||||
): Promise<string[]> {
|
||||
const conditions = [
|
||||
eq(projects.organizationId, orgId),
|
||||
...(accessedServices !== null
|
||||
? [inArray(applications.applicationId, accessedServices)]
|
||||
: []),
|
||||
...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []),
|
||||
...(filters.environmentId
|
||||
? [eq(environments.environmentId, filters.environmentId)]
|
||||
: []),
|
||||
];
|
||||
const rows = await db
|
||||
.select({ applicationId: applications.applicationId })
|
||||
.from(applications)
|
||||
.innerJoin(
|
||||
environments,
|
||||
eq(applications.environmentId, environments.environmentId),
|
||||
)
|
||||
.innerJoin(projects, eq(environments.projectId, projects.projectId))
|
||||
.where(and(...conditions));
|
||||
return rows.map((r) => r.applicationId);
|
||||
}
|
||||
|
||||
async function getComposeIdsForFailedDeployments(
|
||||
orgId: string,
|
||||
accessedServices: string[] | null,
|
||||
filters: FailedDeploymentIdFilters,
|
||||
): Promise<string[]> {
|
||||
const conditions = [
|
||||
eq(projects.organizationId, orgId),
|
||||
...(accessedServices !== null
|
||||
? [inArray(compose.composeId, accessedServices)]
|
||||
: []),
|
||||
...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []),
|
||||
...(filters.environmentId
|
||||
? [eq(environments.environmentId, filters.environmentId)]
|
||||
: []),
|
||||
];
|
||||
const rows = await db
|
||||
.select({ composeId: compose.composeId })
|
||||
.from(compose)
|
||||
.innerJoin(
|
||||
environments,
|
||||
eq(compose.environmentId, environments.environmentId),
|
||||
)
|
||||
.innerJoin(projects, eq(environments.projectId, projects.projectId))
|
||||
.where(and(...conditions));
|
||||
return rows.map((r) => r.composeId);
|
||||
}
|
||||
|
||||
const readLogTail = async (
|
||||
logPath: string | null,
|
||||
serverId: string | null | undefined,
|
||||
) => {
|
||||
if (!logPath) return "";
|
||||
const command = `tail -c 4096 "${logPath}" 2>/dev/null || echo ""`;
|
||||
const { stdout } = serverId
|
||||
? await execAsyncRemote(serverId, command)
|
||||
: await execAsync(command);
|
||||
return stdout;
|
||||
};
|
||||
|
||||
// Caps simultaneous SSH connections when classifying a batch of failures.
|
||||
const LOG_TAIL_CONCURRENCY = 8;
|
||||
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results = new Array<R>(items.length);
|
||||
let nextIndex = 0;
|
||||
const worker = async () => {
|
||||
while (nextIndex < items.length) {
|
||||
const current = nextIndex++;
|
||||
results[current] = await fn(items[current] as T);
|
||||
}
|
||||
};
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(limit, items.length) }, worker),
|
||||
);
|
||||
return results;
|
||||
}
|
||||
|
||||
export type FailedDeploymentFilters = z.infer<typeof apiFindFailedDeployments>;
|
||||
|
||||
// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed
|
||||
// deployments, each tagged with a classifyFailure() guess from its log tail.
|
||||
export const findFailedDeploymentsCentralized = async (
|
||||
orgId: string,
|
||||
accessedServices: string[] | null,
|
||||
filters: FailedDeploymentFilters = {},
|
||||
) => {
|
||||
if (accessedServices !== null && accessedServices.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const idFilters: FailedDeploymentIdFilters = {
|
||||
projectId: filters.projectId,
|
||||
environmentId: filters.environmentId,
|
||||
};
|
||||
|
||||
const [appIds, compIds] = await Promise.all([
|
||||
getApplicationIdsForFailedDeployments(orgId, accessedServices, idFilters),
|
||||
getComposeIdsForFailedDeployments(orgId, accessedServices, idFilters),
|
||||
]);
|
||||
|
||||
if (appIds.length === 0 && compIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// createdAt is stored as text via toISOString(); normalize the filters the
|
||||
// same way so the string range comparison below stays valid.
|
||||
const dateFrom = filters.dateFrom
|
||||
? new Date(filters.dateFrom).toISOString()
|
||||
: undefined;
|
||||
const dateTo = filters.dateTo
|
||||
? new Date(filters.dateTo).toISOString()
|
||||
: undefined;
|
||||
|
||||
const serviceConditions = [
|
||||
...(appIds.length > 0 ? [inArray(deployments.applicationId, appIds)] : []),
|
||||
...(compIds.length > 0 ? [inArray(deployments.composeId, compIds)] : []),
|
||||
];
|
||||
|
||||
const conditions = [
|
||||
eq(deployments.status, "error"),
|
||||
serviceConditions.length === 1
|
||||
? serviceConditions[0]
|
||||
: or(...serviceConditions),
|
||||
...(dateFrom ? [gte(deployments.createdAt, dateFrom)] : []),
|
||||
...(dateTo ? [lte(deployments.createdAt, dateTo)] : []),
|
||||
];
|
||||
|
||||
const failed = await db.query.deployments.findMany({
|
||||
where: and(...conditions),
|
||||
orderBy: desc(deployments.createdAt),
|
||||
with: centralizedDeploymentsWith,
|
||||
limit: 200,
|
||||
});
|
||||
|
||||
// Skip further attempts to a server once it's timed out once, so a batch
|
||||
// of failures on one unreachable host doesn't pay the SSH timeout N times.
|
||||
const deadServers = new Set<string>();
|
||||
return mapWithConcurrency(
|
||||
failed,
|
||||
LOG_TAIL_CONCURRENCY,
|
||||
async (deployment) => {
|
||||
const serverId =
|
||||
deployment.buildServer?.serverId ||
|
||||
deployment.server?.serverId ||
|
||||
deployment.application?.server?.serverId ||
|
||||
deployment.compose?.server?.serverId ||
|
||||
null;
|
||||
let logTail = "";
|
||||
if (!serverId || !deadServers.has(serverId)) {
|
||||
try {
|
||||
logTail = await readLogTail(deployment.logPath, serverId);
|
||||
} catch {
|
||||
if (serverId) deadServers.add(serverId);
|
||||
}
|
||||
}
|
||||
const combined = `${deployment.errorMessage ?? ""}\n${logTail}`;
|
||||
const classification = classifyFailure(combined);
|
||||
// Tail end only: the actual failure line is almost always near the end of the log.
|
||||
const rawError = combined.trim().slice(-1000) || null;
|
||||
return { ...deployment, classification, rawError };
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
export const updateDeployment = async (
|
||||
deploymentId: string,
|
||||
deploymentData: Partial<Deployment>,
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
import { db } from "@dokploy/server/db";
|
||||
import {
|
||||
applications,
|
||||
compose,
|
||||
environments,
|
||||
projects,
|
||||
} from "@dokploy/server/db/schema";
|
||||
import {
|
||||
execAsync,
|
||||
execAsyncRemote,
|
||||
} from "@dokploy/server/utils/process/execAsync";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { and, eq, isNull } from "drizzle-orm";
|
||||
import { IS_CLOUD } from "../constants";
|
||||
import { getRemoteDocker } from "../utils/servers/remote-docker";
|
||||
|
||||
export interface NetworkIpUsage {
|
||||
name: string;
|
||||
driver: string;
|
||||
subnet: string | null;
|
||||
containersInUse: number;
|
||||
capacity: number | null;
|
||||
percentUsed: number | null;
|
||||
}
|
||||
|
||||
export interface ServerHealthResult {
|
||||
checkedAt: string;
|
||||
containers: {
|
||||
containerCount: number;
|
||||
serviceCount: number;
|
||||
};
|
||||
resources: {
|
||||
memTotalBytes: number;
|
||||
memUsedBytes: number;
|
||||
cpuCount: number;
|
||||
};
|
||||
inotify: {
|
||||
maxWatches: number;
|
||||
maxInstances: number;
|
||||
maxQueuedEvents: number;
|
||||
currentInstances: number;
|
||||
persisted: boolean;
|
||||
};
|
||||
disk: {
|
||||
totalBytes: number;
|
||||
usedBytes: number;
|
||||
};
|
||||
dockerNetworks: {
|
||||
count: number;
|
||||
addressPools: unknown | null;
|
||||
/** Per-network IP utilization, including reserved networks like dokploy-network */
|
||||
usage: NetworkIpUsage[];
|
||||
/** Set when `usage` couldn't be read, so the UI can tell that apart from "no networks" */
|
||||
usageError?: string;
|
||||
};
|
||||
daemonErrors: string[];
|
||||
/** The window actually passed to `journalctl --since`, computed on the target machine's clock */
|
||||
daemonLogsWindow: { fromEpoch: number; toEpoch: number } | null;
|
||||
reservation: {
|
||||
memoryReservedBytes: number;
|
||||
cpuReservedNanoCpus: number;
|
||||
appCount: number;
|
||||
unsupportedComposeCount: number;
|
||||
} | null;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface RawHealthOutput {
|
||||
containerCount?: number | string;
|
||||
serviceCount?: number | string;
|
||||
memTotalBytes?: number | string;
|
||||
memUsedBytes?: number | string;
|
||||
cpuCount?: number | string;
|
||||
inotifyMaxWatches?: number | string;
|
||||
inotifyMaxInstances?: number | string;
|
||||
inotifyMaxQueuedEvents?: number | string;
|
||||
inotifyCurrentInstances?: number | string;
|
||||
inotifyPersistedCount?: number | string;
|
||||
diskTotalBytes?: number | string;
|
||||
diskUsedBytes?: number | string;
|
||||
networkCount?: number | string;
|
||||
daemonConfigBase64?: string;
|
||||
daemonErrorsBase64?: string;
|
||||
daemonLogsFromEpoch?: number | string;
|
||||
daemonLogsToEpoch?: number | string;
|
||||
}
|
||||
|
||||
const toInt = (value: unknown): number => {
|
||||
const n = Number.parseInt(String(value ?? "0"), 10);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
};
|
||||
|
||||
const b64Decode = (value?: string): string => {
|
||||
if (!value) return "";
|
||||
try {
|
||||
return Buffer.from(value, "base64").toString("utf-8");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
};
|
||||
|
||||
// Single read-only script, one SSH/exec round-trip; each check falls back to 0/empty on its own.
|
||||
const buildHealthScript = (sinceHours: number) => `
|
||||
containerCount=$(docker ps -a -q 2>/dev/null | wc -l | tr -d ' ')
|
||||
serviceCount=$(docker service ls --format '{{.Name}}' 2>/dev/null | wc -l | tr -d ' ')
|
||||
|
||||
memTotal=$(free -b 2>/dev/null | awk '/^Mem:/{print $2}'); [ -z "$memTotal" ] && memTotal=0
|
||||
memUsed=$(free -b 2>/dev/null | awk '/^Mem:/{print $3}'); [ -z "$memUsed" ] && memUsed=0
|
||||
cpuCount=$(nproc 2>/dev/null); [ -z "$cpuCount" ] && cpuCount=0
|
||||
|
||||
inotifyMaxWatches=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null); [ -z "$inotifyMaxWatches" ] && inotifyMaxWatches=0
|
||||
inotifyMaxInstances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null); [ -z "$inotifyMaxInstances" ] && inotifyMaxInstances=0
|
||||
inotifyMaxQueued=$(cat /proc/sys/fs/inotify/max_queued_events 2>/dev/null); [ -z "$inotifyMaxQueued" ] && inotifyMaxQueued=0
|
||||
inotifyCurrentInstances=$(find /proc/*/fd -lname 'anon_inode:inotify*' 2>/dev/null | wc -l | tr -d ' ')
|
||||
inotifyPersistedCount=$(grep -rl inotify /etc/sysctl.conf /etc/sysctl.d/ 2>/dev/null | wc -l | tr -d ' ')
|
||||
|
||||
diskTotal=$(df -B1 / 2>/dev/null | awk 'NR==2{print $2}'); [ -z "$diskTotal" ] && diskTotal=0
|
||||
diskUsed=$(df -B1 / 2>/dev/null | awk 'NR==2{print $3}'); [ -z "$diskUsed" ] && diskUsed=0
|
||||
|
||||
networkCount=$(docker network ls -q 2>/dev/null | wc -l | tr -d ' ')
|
||||
daemonConfigB64=$(cat /etc/docker/daemon.json 2>/dev/null | base64 2>/dev/null | tr -d '\\n')
|
||||
|
||||
daemonLogsToEpoch=$(date +%s 2>/dev/null); [ -z "$daemonLogsToEpoch" ] && daemonLogsToEpoch=0
|
||||
daemonLogsFromEpoch=$((daemonLogsToEpoch - ${sinceHours} * 3600))
|
||||
|
||||
daemonErrorsB64=$(journalctl -u docker --no-pager --since "${sinceHours} hours ago" 2>/dev/null | grep -iE "inotify|too many open|cannot allocate|oom|conntrack|no space|pids.max|fork:|resource temporarily unavailable|could not find an available ip|no available ip|task allocation failure|address already in use" | tail -n 50 | base64 2>/dev/null | tr -d '\\n')
|
||||
|
||||
printf '{"containerCount":%s,"serviceCount":%s,"memTotalBytes":%s,"memUsedBytes":%s,"cpuCount":%s,"inotifyMaxWatches":%s,"inotifyMaxInstances":%s,"inotifyMaxQueuedEvents":%s,"inotifyCurrentInstances":%s,"inotifyPersistedCount":%s,"diskTotalBytes":%s,"diskUsedBytes":%s,"networkCount":%s,"daemonConfigBase64":"%s","daemonErrorsBase64":"%s","daemonLogsFromEpoch":%s,"daemonLogsToEpoch":%s}' "$containerCount" "$serviceCount" "$memTotal" "$memUsed" "$cpuCount" "$inotifyMaxWatches" "$inotifyMaxInstances" "$inotifyMaxQueued" "$inotifyCurrentInstances" "$inotifyPersistedCount" "$diskTotal" "$diskUsed" "$networkCount" "$daemonConfigB64" "$daemonErrorsB64" "$daemonLogsFromEpoch" "$daemonLogsToEpoch"
|
||||
`;
|
||||
|
||||
const emptyResult = (error: unknown): ServerHealthResult => ({
|
||||
checkedAt: new Date().toISOString(),
|
||||
containers: { containerCount: 0, serviceCount: 0 },
|
||||
resources: { memTotalBytes: 0, memUsedBytes: 0, cpuCount: 0 },
|
||||
inotify: {
|
||||
maxWatches: 0,
|
||||
maxInstances: 0,
|
||||
maxQueuedEvents: 0,
|
||||
currentInstances: 0,
|
||||
persisted: false,
|
||||
},
|
||||
disk: { totalBytes: 0, usedBytes: 0 },
|
||||
dockerNetworks: { count: 0, addressPools: null, usage: [] },
|
||||
daemonErrors: [],
|
||||
daemonLogsWindow: null,
|
||||
reservation: null,
|
||||
error:
|
||||
error instanceof Error ? error.message : "Could not read server health",
|
||||
});
|
||||
|
||||
/** Usable host addresses in a subnet, excluding network + broadcast. Null if the subnet is missing/invalid. */
|
||||
export const getSubnetCapacity = (
|
||||
subnet: string | undefined,
|
||||
): number | null => {
|
||||
if (!subnet) return null;
|
||||
const prefix = Number.parseInt(subnet.split("/")[1] ?? "", 10);
|
||||
if (Number.isNaN(prefix) || prefix < 0 || prefix > 32) return null;
|
||||
const hostBits = 32 - prefix;
|
||||
if (hostBits <= 1) return 0;
|
||||
return 2 ** hostBits - 2;
|
||||
};
|
||||
|
||||
type DockerNetworkSummary = {
|
||||
Name: string;
|
||||
Driver: string;
|
||||
IPAM?: { Config?: Array<{ Subnet?: string }> | null };
|
||||
};
|
||||
|
||||
type InspectableNetwork = {
|
||||
inspect: (opts?: { verbose?: boolean }) => Promise<{
|
||||
Containers?: Record<string, unknown>;
|
||||
}>;
|
||||
};
|
||||
|
||||
// Includes reserved networks like dokploy-network, which are excluded from the `network` table/UI.
|
||||
const getNetworksIpUsage = async (
|
||||
serverId: string | undefined,
|
||||
): Promise<NetworkIpUsage[]> => {
|
||||
const docker = await getRemoteDocker(serverId);
|
||||
const dockerNetworks =
|
||||
(await docker.listNetworks()) as DockerNetworkSummary[];
|
||||
const relevant = dockerNetworks.filter(
|
||||
(n) => n.Driver === "bridge" || n.Driver === "overlay",
|
||||
);
|
||||
|
||||
const rows = await Promise.all(
|
||||
relevant.map(async (n): Promise<NetworkIpUsage> => {
|
||||
// IPv4 entry specifically: dual-stack networks list an IPv6 config too, and it may come first.
|
||||
const subnet = n.IPAM?.Config?.find(
|
||||
(c) => c.Subnet && !c.Subnet.includes(":"),
|
||||
)?.Subnet;
|
||||
const capacity = getSubnetCapacity(subnet);
|
||||
let containersInUse = 0;
|
||||
try {
|
||||
// Cast the object (not the detached method) so `net.inspect(...)` stays a bound method call.
|
||||
const net = docker.getNetwork(n.Name) as unknown as InspectableNetwork;
|
||||
// verbose: true for full swarm-wide container list on overlay networks
|
||||
const info = await net.inspect(
|
||||
n.Driver === "overlay" ? { verbose: true } : undefined,
|
||||
);
|
||||
containersInUse = Object.keys(info.Containers ?? {}).length;
|
||||
} catch {
|
||||
// Network may have disappeared between list and inspect; report 0
|
||||
}
|
||||
return {
|
||||
name: n.Name,
|
||||
driver: n.Driver,
|
||||
subnet: subnet ?? null,
|
||||
containersInUse,
|
||||
capacity,
|
||||
percentUsed:
|
||||
capacity && capacity > 0
|
||||
? Math.round((containersInUse / capacity) * 100)
|
||||
: null,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
return rows.sort((a, b) => (b.percentUsed ?? -1) - (a.percentUsed ?? -1));
|
||||
};
|
||||
|
||||
// Sums Application memoryReservation/cpuReservation (bytes/NanoCPUs, see
|
||||
// calculateResources in utils/docker/utils.ts). Compose has no such columns.
|
||||
export const getReservationSummary = async (
|
||||
orgId: string,
|
||||
serverId?: string,
|
||||
) => {
|
||||
const appRows = await db
|
||||
.select({
|
||||
memoryReservation: applications.memoryReservation,
|
||||
cpuReservation: applications.cpuReservation,
|
||||
})
|
||||
.from(applications)
|
||||
.innerJoin(
|
||||
environments,
|
||||
eq(applications.environmentId, environments.environmentId),
|
||||
)
|
||||
.innerJoin(projects, eq(environments.projectId, projects.projectId))
|
||||
.where(
|
||||
and(
|
||||
eq(projects.organizationId, orgId),
|
||||
serverId
|
||||
? eq(applications.serverId, serverId)
|
||||
: isNull(applications.serverId),
|
||||
),
|
||||
);
|
||||
|
||||
let memoryReservedBytes = 0;
|
||||
let cpuReservedNanoCpus = 0;
|
||||
for (const row of appRows) {
|
||||
if (row.memoryReservation) {
|
||||
memoryReservedBytes += toInt(row.memoryReservation);
|
||||
}
|
||||
if (row.cpuReservation) {
|
||||
cpuReservedNanoCpus += toInt(row.cpuReservation);
|
||||
}
|
||||
}
|
||||
|
||||
const composeRows = await db
|
||||
.select({ composeId: compose.composeId })
|
||||
.from(compose)
|
||||
.innerJoin(
|
||||
environments,
|
||||
eq(compose.environmentId, environments.environmentId),
|
||||
)
|
||||
.innerJoin(projects, eq(environments.projectId, projects.projectId))
|
||||
.where(
|
||||
and(
|
||||
eq(projects.organizationId, orgId),
|
||||
serverId ? eq(compose.serverId, serverId) : isNull(compose.serverId),
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
memoryReservedBytes,
|
||||
cpuReservedNanoCpus,
|
||||
appCount: appRows.length,
|
||||
unsupportedComposeCount: composeRows.length,
|
||||
};
|
||||
};
|
||||
|
||||
export const getServerHealth = async (
|
||||
orgId: string,
|
||||
serverId?: string,
|
||||
sinceHours = 24,
|
||||
): Promise<ServerHealthResult> => {
|
||||
if (IS_CLOUD && !serverId) {
|
||||
throw new TRPCError({
|
||||
code: "BAD_REQUEST",
|
||||
message: "Server is required",
|
||||
});
|
||||
}
|
||||
|
||||
const hours = Math.max(
|
||||
1,
|
||||
Math.min(168, Math.floor(Number(sinceHours) || 24)),
|
||||
);
|
||||
const script = buildHealthScript(hours);
|
||||
|
||||
let stdout = "";
|
||||
try {
|
||||
const result = serverId
|
||||
? await execAsyncRemote(serverId, script)
|
||||
: await execAsync(script);
|
||||
stdout = result.stdout;
|
||||
} catch (error) {
|
||||
return emptyResult(error);
|
||||
}
|
||||
|
||||
let parsed: RawHealthOutput;
|
||||
try {
|
||||
parsed = JSON.parse(stdout.trim());
|
||||
} catch {
|
||||
return emptyResult(new Error("Could not parse server health output"));
|
||||
}
|
||||
|
||||
const daemonErrors = b64Decode(parsed.daemonErrorsBase64)
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
let addressPools: unknown = null;
|
||||
const daemonConfigText = b64Decode(parsed.daemonConfigBase64);
|
||||
if (daemonConfigText) {
|
||||
try {
|
||||
addressPools =
|
||||
JSON.parse(daemonConfigText)?.["default-address-pools"] ?? null;
|
||||
} catch {
|
||||
addressPools = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Not part of the shell script above: needs structured per-network data from the Docker API.
|
||||
const [reservation, networkUsageResult] = await Promise.all([
|
||||
getReservationSummary(orgId, serverId).catch(() => null),
|
||||
getNetworksIpUsage(serverId).then(
|
||||
(usage) => ({ usage, error: undefined }),
|
||||
(error: unknown) => ({
|
||||
usage: [] as NetworkIpUsage[],
|
||||
error:
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: "Failed to read network usage",
|
||||
}),
|
||||
),
|
||||
]);
|
||||
|
||||
return {
|
||||
checkedAt: new Date().toISOString(),
|
||||
containers: {
|
||||
containerCount: toInt(parsed.containerCount),
|
||||
serviceCount: toInt(parsed.serviceCount),
|
||||
},
|
||||
resources: {
|
||||
memTotalBytes: toInt(parsed.memTotalBytes),
|
||||
memUsedBytes: toInt(parsed.memUsedBytes),
|
||||
cpuCount: toInt(parsed.cpuCount),
|
||||
},
|
||||
inotify: {
|
||||
maxWatches: toInt(parsed.inotifyMaxWatches),
|
||||
maxInstances: toInt(parsed.inotifyMaxInstances),
|
||||
maxQueuedEvents: toInt(parsed.inotifyMaxQueuedEvents),
|
||||
currentInstances: toInt(parsed.inotifyCurrentInstances),
|
||||
persisted: toInt(parsed.inotifyPersistedCount) > 0,
|
||||
},
|
||||
disk: {
|
||||
totalBytes: toInt(parsed.diskTotalBytes),
|
||||
usedBytes: toInt(parsed.diskUsedBytes),
|
||||
},
|
||||
dockerNetworks: {
|
||||
count: toInt(parsed.networkCount),
|
||||
addressPools,
|
||||
usage: networkUsageResult.usage,
|
||||
usageError: networkUsageResult.error,
|
||||
},
|
||||
daemonErrors,
|
||||
daemonLogsWindow:
|
||||
toInt(parsed.daemonLogsToEpoch) > 0
|
||||
? {
|
||||
fromEpoch: toInt(parsed.daemonLogsFromEpoch),
|
||||
toEpoch: toInt(parsed.daemonLogsToEpoch),
|
||||
}
|
||||
: null,
|
||||
reservation,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,69 @@
|
||||
export interface FailurePattern {
|
||||
id: string;
|
||||
label: string;
|
||||
pattern: RegExp;
|
||||
}
|
||||
|
||||
export const FAILURE_PATTERNS: FailurePattern[] = [
|
||||
{
|
||||
id: "inotify-limit",
|
||||
label: "Inotify limit reached",
|
||||
pattern: /max_user_instances|max_user_watches|inotify watch limit reached/i,
|
||||
},
|
||||
{
|
||||
id: "oom",
|
||||
label: "Out of memory (OOM)",
|
||||
pattern: /\boom[-_ ]?kill(er|ed)?\b|killed process|out of memory/i,
|
||||
},
|
||||
{
|
||||
id: "address-pool-exhausted",
|
||||
label: "Docker network address pool exhausted",
|
||||
pattern:
|
||||
/could not find an available[,]? non-overlapping|available.*address pool/i,
|
||||
},
|
||||
{
|
||||
id: "network-ip-exhausted",
|
||||
label: "Network out of IP addresses",
|
||||
pattern:
|
||||
/could not find an available ip|no available ip addresses|task allocation failure/i,
|
||||
},
|
||||
{
|
||||
id: "network-attach-timeout",
|
||||
label: "Network attach timed out",
|
||||
pattern:
|
||||
/attaching to network failed|failed to set up container networking/i,
|
||||
},
|
||||
{
|
||||
id: "disk-full",
|
||||
label: "Disk space exhausted",
|
||||
pattern: /no space left on device/i,
|
||||
},
|
||||
{
|
||||
id: "conntrack-full",
|
||||
label: "Connection tracking table full",
|
||||
pattern: /nf_conntrack:?\s*table full/i,
|
||||
},
|
||||
{
|
||||
id: "fd-limit",
|
||||
label: "File descriptor limit reached",
|
||||
pattern: /too many open files/i,
|
||||
},
|
||||
{
|
||||
id: "pid-limit",
|
||||
label: "Process/PID limit reached",
|
||||
pattern:
|
||||
/pids\.max|cannot allocate memory|fork:.*resource temporarily unavailable/i,
|
||||
},
|
||||
];
|
||||
|
||||
export const classifyFailure = (
|
||||
text: string | null | undefined,
|
||||
): { id: string; label: string } | null => {
|
||||
if (!text) return null;
|
||||
for (const entry of FAILURE_PATTERNS) {
|
||||
if (entry.pattern.test(text)) {
|
||||
return { id: entry.id, label: entry.label };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
Reference in New Issue
Block a user