gsadmin/dokploy
1
0
Fork 0
mirror of https://github.com/Dokploy/dokploy.git synced 2026-10-19 23:21:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: enhance log management and organization handling

Browse Source 
- Updated API link conditions to allow skipping batch processing based on context.
- Removed unused VECTOR_RESYNC_CRON_JOB constant from server constants.
- Introduced logProviderIds field in server and webServerSettings schemas to manage log providers.
- Added organization service to handle organization deletions and vector agent removals.
- Enhanced server service to find servers with log management enabled and update log providers.
- Updated vector setup to sync vector configurations based on organization and server log provider IDs.
- Implemented Docker labels for resources to improve tracking and management.
- Refactored vector agent setup and removal processes to streamline operations.
This commit is contained in:
Mauricio Siu
2026-09-24 01:23:20 -06:00
parent 9e8c5cde31
commit ae0cdc6456
31 changed files with 10149 additions and 1697 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/dokploy/__test__/log-management/vector-config.test.ts
+55 -224
View File
@@ -1,71 +1,26 @@
import { describe, expect, it, vi } from "vitest";
import { buildVectorConfigYaml } from "@dokploy/server/setup/vector-setup";
import { describe, expect, it } from "vitest";
import { parse } from "yaml";
const mocks = vi.hoisted(() => ({
findManyProjects: vi.fn(),
findEnabledLogProvidersByOrganization: vi.fn(),
}));
vi.mock("@dokploy/server/db", () => ({
db: {
query: {
projects: {
findMany: mocks.findManyProjects,
},
},
},
}));
vi.mock(
"@dokploy/server/services/log-management/service",
async (importOriginal) => ({
...(await importOriginal<
typeof import("@dokploy/server/services/log-management/service")
>()),
findEnabledLogProvidersByOrganization:
mocks.findEnabledLogProvidersByOrganization,
}),
);
const { buildVectorConfigYaml } = await import(
"@dokploy/server/setup/vector-setup"
);
const provider = (overrides: Record<string, unknown>) =>
({
apiKey: null,
apiSecret: null,
endpoint: null,
extraConfig: null,
...overrides,
}) as any;
describe("buildVectorConfigYaml", () => {
it("generates a source, a dokploy_scope transform and one sink per enabled provider", async () => {
mocks.findManyProjects.mockResolvedValue([
{
projectId: "project-1",
name: "My Project",
environments: [
{
environmentId: "env-1",
name: "production",
applications: [
{
appName: "app-my-app-abc123",
applicationId: "application-1",
name: "my-app",
},
],
compose: [],
},
],
},
]);
mocks.findEnabledLogProvidersByOrganization.mockResolvedValue([
{
const yamlStr = await buildVectorConfigYaml([
provider({
logProviderId: "log-provider-1",
name: "loki-prod",
providerType: "loki",
endpoint: "https://loki.example.com",
apiKey: null,
apiSecret: null,
extraConfig: null,
},
}),
]);
const yamlStr = await buildVectorConfigYaml("org-1");
const config = parse(yamlStr) as any;
expect(config.data_dir).toBe("/var/lib/vector");
@@ -74,21 +29,6 @@ describe("buildVectorConfigYaml", () => {
expect(config.transforms.dokploy_scope.inputs).toEqual([
"docker_logs_source",
]);
expect(config.transforms.dokploy_scope.source).toContain(
'.dokploy_organization = "org-1"',
);
expect(config.transforms.dokploy_scope.source).toContain(
'app_name == "app-my-app-abc123"',
);
expect(config.transforms.dokploy_scope.source).toContain(
'.dokploy_project = "My Project"',
);
expect(config.transforms.dokploy_scope.source).toContain(
'.dokploy_environment = "production"',
);
expect(config.transforms.dokploy_scope.source).toContain(
'.dokploy_application = "my-app"',
);
const sinkIds = Object.keys(config.sinks);
expect(sinkIds).toEqual(["sink_log-provider-1"]);
@@ -99,195 +39,89 @@ describe("buildVectorConfigYaml", () => {
});
it("chains scope -> transform -> sink for betterstack/datadog, and scope -> sink direct for loki", async () => {
mocks.findManyProjects.mockResolvedValue([]);
mocks.findEnabledLogProvidersByOrganization.mockResolvedValue([
{
const yamlStr = await buildVectorConfigYaml([
provider({
logProviderId: "loki-1",
name: "loki",
providerType: "loki",
endpoint: "https://loki.example.com",
apiKey: null,
apiSecret: null,
extraConfig: null,
},
{
}),
provider({
logProviderId: "bs-1",
name: "betterstack",
providerType: "betterstack",
endpoint: "https://in.logs.betterstack.com",
apiKey: "token",
apiSecret: null,
extraConfig: null,
},
{
}),
provider({
logProviderId: "dd-1",
name: "datadog",
providerType: "datadog",
endpoint: null,
apiKey: "dd-key",
apiSecret: null,
extraConfig: null,
},
}),
]);
const yamlStr = await buildVectorConfigYaml("org-1");
const config = parse(yamlStr) as any;
expect(config.sinks["sink_loki-1"].inputs).toEqual(["dokploy_scope"]);
expect(config.transforms["transform_bs-1"]).toBeDefined();
expect(config.transforms["transform_bs-1"].inputs).toEqual([
"dokploy_scope",
]);
expect(config.sinks["sink_bs-1"].inputs).toEqual(["transform_bs-1"]);
expect(config.transforms["transform_dd-1"]).toBeDefined();
expect(config.transforms["transform_dd-1"].inputs).toEqual([
"dokploy_scope",
]);
expect(config.sinks["sink_dd-1"].inputs).toEqual(["transform_dd-1"]);
});
it("escapes double quotes in project/application names to avoid breaking the VRL string literal", async () => {
mocks.findManyProjects.mockResolvedValue([
{
projectId: "project-1",
name: 'My "Project"',
environments: [
{
environmentId: "env-1",
name: "production",
applications: [
{
appName: "app-abc123",
applicationId: "application-1",
name: "app",
},
],
compose: [],
},
],
},
]);
mocks.findEnabledLogProvidersByOrganization.mockResolvedValue([]);
const yamlStr = await buildVectorConfigYaml("org-1");
const config = parse(yamlStr) as any;
expect(config.transforms.dokploy_scope.source).toContain(
'.dokploy_project = "My \\"Project\\""',
);
});
it("replaces every control character in a project name, not just newlines, before embedding it in VRL", async () => {
mocks.findManyProjects.mockResolvedValue([
{
projectId: "project-1",
name: "Weird\r\x00Name",
environments: [
{
environmentId: "env-1",
name: "production",
applications: [
{
appName: "app-abc123",
applicationId: "application-1",
name: "app",
},
],
compose: [],
},
],
},
]);
mocks.findEnabledLogProvidersByOrganization.mockResolvedValue([]);
const yamlStr = await buildVectorConfigYaml("org-1");
const config = parse(yamlStr) as any;
const source = config.transforms.dokploy_scope.source as string;
expect(source).toContain('.dokploy_project = "Weird Name"');
});
it("keeps events without a matching appName unscoped (no crash, no filtering), with the scoping fields defaulted to empty string", async () => {
mocks.findManyProjects.mockResolvedValue([]);
mocks.findEnabledLogProvidersByOrganization.mockResolvedValue([]);
const yamlStr = await buildVectorConfigYaml("org-1");
it("reads the Dokploy metadata straight off the container labels, with empty defaults for containers Dokploy did not deploy", async () => {
const yamlStr = await buildVectorConfigYaml([]);
const config = parse(yamlStr) as any;
expect(config.transforms.dokploy_scope.source).toBe(
[
'.dokploy_organization = "org-1"',
'.dokploy_project = ""',
'.dokploy_project_id = ""',
'.dokploy_environment = ""',
'.dokploy_environment_id = ""',
'.dokploy_application = ""',
'.dokploy_application_id = ""',
'.dokploy_organization = .label."dokploy.organization.id" || ""',
'.dokploy_project = .label."dokploy.project" || ""',
'.dokploy_project_id = .label."dokploy.project.id" || ""',
'.dokploy_environment = .label."dokploy.environment" || ""',
'.dokploy_environment_id = .label."dokploy.environment.id" || ""',
'.dokploy_application = .label."dokploy.application" || ""',
'.dokploy_application_id = .label."dokploy.application.id" || ""',
'.dokploy_service = .label."dokploy.service" || ""',
].join("\n"),
);
});
it("uses preloaded org data instead of querying, when given (fan-out reuse)", async () => {
mocks.findManyProjects.mockClear();
mocks.findEnabledLogProvidersByOrganization.mockClear();
const yamlStr = await buildVectorConfigYaml("org-1", {
providers: [],
lookup: {},
});
expect(mocks.findManyProjects).not.toHaveBeenCalled();
expect(mocks.findEnabledLogProvidersByOrganization).not.toHaveBeenCalled();
const config = parse(yamlStr) as any;
expect(config.sinks).toEqual({});
});
it("skips a provider whose config an adapter rejects at build time, instead of failing the whole org's config", async () => {
const yamlStr = await buildVectorConfigYaml("org-1", {
providers: [
{
logProviderId: "broken-es",
name: "es-broken",
providerType: "elasticsearch",
endpoint: "https://es.example.com:9200",
apiKey: null,
apiSecret: null,
extraConfig: { username: "elastic" },
},
{
logProviderId: "good-loki",
name: "loki-ok",
providerType: "loki",
endpoint: "https://loki.example.com",
apiKey: null,
apiSecret: null,
extraConfig: null,
},
] as any,
lookup: {},
});
const yamlStr = await buildVectorConfigYaml([
provider({
logProviderId: "broken-es",
name: "es-broken",
providerType: "elasticsearch",
endpoint: "https://es.example.com:9200",
extraConfig: { username: "elastic" },
}),
provider({
logProviderId: "good-loki",
name: "loki-ok",
providerType: "loki",
endpoint: "https://loki.example.com",
}),
]);
const config = parse(yamlStr) as any;
expect(Object.keys(config.sinks)).toEqual(["sink_good-loki"]);
});
it("dropUnmatched (local host only) filters out events with no matching Dokploy app before any sink", async () => {
it("drops containers Dokploy did not deploy on the local host, which also runs unrelated containers", async () => {
const yamlStr = await buildVectorConfigYaml(
"org-1",
{
providers: [
{
logProviderId: "loki-1",
name: "loki",
providerType: "loki",
endpoint: "https://loki.example.com",
apiKey: null,
apiSecret: null,
extraConfig: null,
},
] as any,
lookup: {},
},
[
provider({
logProviderId: "loki-1",
name: "loki",
providerType: "loki",
endpoint: "https://loki.example.com",
}),
],
{ dropUnmatched: true },
);
@@ -303,10 +137,7 @@ describe("buildVectorConfigYaml", () => {
});
it("does not add the drop-unmatched filter for the per-server (non-local) path", async () => {
mocks.findManyProjects.mockResolvedValue([]);
mocks.findEnabledLogProvidersByOrganization.mockResolvedValue([]);
const yamlStr = await buildVectorConfigYaml("org-1");
const yamlStr = await buildVectorConfigYaml([]);
const config = parse(yamlStr) as any;
expect(config.transforms.dokploy_scope_local_only).toBeUndefined();
apps/dokploy/__test__/log-management/vector-setup-fixes.test.ts
+4 -176
View File
@@ -1,8 +1,4 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
reserve: vi.fn(),
}));
import { describe, expect, it, vi } from "vitest";
vi.mock("@dokploy/server/db", () => ({
db: {
@@ -10,87 +6,9 @@ vi.mock("@dokploy/server/db", () => ({
},
}));
vi.mock("postgres", () => ({
default: vi.fn(() => ({ reserve: mocks.reserve })),
}));
const makeReservedMock = (locked: boolean) => {
const tag: any = vi
.fn()
.mockImplementation((strings: TemplateStringsArray) => {
const text = strings.join("");
if (text.includes("pg_try_advisory_lock")) {
return Promise.resolve([{ locked }]);
}
return Promise.resolve([]);
});
tag.release = vi.fn();
return tag;
};
const {
withConfigWriteLock,
collectSecretValues,
redactSecrets,
stripImageDigest,
vectorServiceSpecUnchanged,
buildServiceSettings,
} = await import("@dokploy/server/setup/vector-setup");
describe("withConfigWriteLock — cross-process guard (session scoped pg_advisory_lock)", () => {
beforeEach(() => {
mocks.reserve.mockReset();
});
it("runs fn and returns its result when the advisory lock is acquired", async () => {
const reserved = makeReservedMock(true);
mocks.reserve.mockResolvedValue(reserved);
const fn = vi.fn().mockResolvedValue("ok");
await expect(withConfigWriteLock("server-1", fn)).resolves.toBe("ok");
expect(fn).toHaveBeenCalledTimes(1);
const joinedCalls = reserved.mock.calls.map((call: any[]) =>
(call[0] as string[]).join(""),
);
expect(
joinedCalls.some((text: string) => text.includes("pg_advisory_unlock")),
).toBe(true);
expect(reserved.release).toHaveBeenCalledTimes(1);
});
it("retries, then throws without ever calling fn, when another process keeps holding the lock", async () => {
mocks.reserve.mockImplementation(async () => makeReservedMock(false));
const fn = vi.fn();
await expect(withConfigWriteLock("server-1", fn)).rejects.toThrow(
/already in progress/i,
);
expect(fn).not.toHaveBeenCalled();
expect(mocks.reserve).toHaveBeenCalledTimes(3);
}, 10_000);
it("still serializes two calls for the same serverId within this process", async () => {
mocks.reserve.mockImplementation(async () => makeReservedMock(true));
const order: string[] = [];
const slow = async (label: string, ms: number) => {
order.push(`${label}:start`);
await new Promise((resolve) => setTimeout(resolve, ms));
order.push(`${label}:end`);
return label;
};
const first = withConfigWriteLock("server-a", () => slow("first", 20));
const second = withConfigWriteLock("server-a", () => slow("second", 1));
await Promise.all([first, second]);
expect(order).toEqual([
"first:start",
"first:end",
"second:start",
"second:end",
]);
});
});
const { collectSecretValues, redactSecrets } = await import(
"@dokploy/server/setup/vector-setup"
);
describe("collectSecretValues / redactSecrets", () => {
it("collects endpoint/apiKey/apiSecret and string extraConfig values across all providers", () => {
@@ -152,93 +70,3 @@ describe("collectSecretValues / redactSecrets", () => {
);
});
});
describe("stripImageDigest", () => {
it("drops an @sha256 digest suffix", () => {
expect(
stripImageDigest("timberio/vector:latest-alpine@sha256:abc123"),
).toBe("timberio/vector:latest-alpine");
});
it("returns the image unchanged when there's no digest", () => {
expect(stripImageDigest("timberio/vector:latest-alpine")).toBe(
"timberio/vector:latest-alpine",
);
});
});
describe("vectorServiceSpecUnchanged", () => {
const settings = {
TaskTemplate: {
ContainerSpec: {
Image: "timberio/vector:latest-alpine",
Args: ["--config", "/etc/vector/vector.yaml", "--watch-config"],
Mounts: [{ Type: "bind", Source: "/a", Target: "/b" }],
},
Networks: [{ Target: "host" }],
},
Mode: { Replicated: { Replicas: 1 } },
} as any;
it("is unchanged when inspect matches settings modulo an image digest suffix and a resolved network ID", () => {
const inspect = {
Spec: {
TaskTemplate: {
ContainerSpec: {
Image: "timberio/vector:latest-alpine@sha256:deadbeef",
Args: ["--config", "/etc/vector/vector.yaml", "--watch-config"],
Mounts: [{ Type: "bind", Source: "/a", Target: "/b" }],
},
Networks: [{ Target: "k1vw5h40twado5nnk9n2hi8nt" }],
},
Mode: { Replicated: { Replicas: 1 } },
},
} as any;
expect(vectorServiceSpecUnchanged(inspect, settings)).toBe(true);
});
it("is changed when a bind mount source differs", () => {
const inspect = {
Spec: {
TaskTemplate: {
ContainerSpec: {
Image: "timberio/vector:latest-alpine",
Args: ["--config", "/etc/vector/vector.yaml", "--watch-config"],
Mounts: [{ Type: "bind", Source: "/old-path", Target: "/b" }],
},
Networks: [{ Target: "host" }],
},
Mode: { Replicated: { Replicas: 1 } },
},
} as any;
expect(vectorServiceSpecUnchanged(inspect, settings)).toBe(false);
});
it("is changed when Mode differs", () => {
const inspect = {
Spec: {
TaskTemplate: settings.TaskTemplate,
Mode: { Global: {} },
},
} as any;
expect(vectorServiceSpecUnchanged(inspect, settings)).toBe(false);
});
});
describe("buildServiceSettings", () => {
it("bind-mounts the config directory, not the vector.yaml file itself", () => {
const settings = buildServiceSettings("/etc/dokploy/vector") as any;
const mounts = settings.TaskTemplate.ContainerSpec.Mounts as Array<{
Source: string;
Target: string;
}>;
const configMount = mounts.find((m) => m.Target === "/etc/vector");
expect(configMount?.Source).toBe("/etc/dokploy/vector");
expect(mounts.some((m) => m.Target === "/etc/vector/vector.yaml")).toBe(
false,
);
expect(settings.TaskTemplate.ContainerSpec.Args).toContain(
"/etc/vector/vector.yaml",
);
});
});
apps/dokploy/__test__/log-management/vector-setup-web.test.ts
-96
View File
@@ -1,96 +0,0 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
getWebServerSettings: vi.fn(),
hasEnabledLogProvider: vi.fn(),
getService: vi.fn(),
remove: vi.fn(),
}));
vi.mock("@dokploy/server/services/web-server-settings", () => ({
getWebServerSettings: mocks.getWebServerSettings,
}));
vi.mock(
"@dokploy/server/services/log-management/service",
async (importOriginal) => ({
...(await importOriginal<
typeof import("@dokploy/server/services/log-management/service")
>()),
hasEnabledLogProvider: mocks.hasEnabledLogProvider,
}),
);
vi.mock("@dokploy/server/utils/servers/remote-docker", () => ({
getRemoteDocker: vi.fn().mockResolvedValue({
getService: mocks.getService,
}),
}));
const { syncWebVectorAgent } = await import(
"@dokploy/server/setup/vector-setup"
);
describe("syncWebVectorAgent — install condition", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("tears down (never installs) when the local toggle is off", async () => {
mocks.getWebServerSettings.mockResolvedValue({
enableLogManagement: false,
logManagementOrganizationId: "org-1",
});
mocks.remove.mockResolvedValue(undefined);
mocks.getService.mockReturnValue({ remove: mocks.remove });
const result = await syncWebVectorAgent();
expect(result.installed).toBe(false);
expect(mocks.getService).toHaveBeenCalledWith("dokploy-vector");
expect(mocks.remove).toHaveBeenCalled();
});
it("tears down when no organization has claimed the local agent yet", async () => {
mocks.getWebServerSettings.mockResolvedValue({
enableLogManagement: true,
logManagementOrganizationId: null,
});
mocks.remove.mockResolvedValue(undefined);
mocks.getService.mockReturnValue({ remove: mocks.remove });
const result = await syncWebVectorAgent();
expect(result.installed).toBe(false);
expect(mocks.hasEnabledLogProvider).not.toHaveBeenCalled();
});
it("tears down when the owning organization has no enabled log provider", async () => {
mocks.getWebServerSettings.mockResolvedValue({
enableLogManagement: true,
logManagementOrganizationId: "org-1",
});
mocks.hasEnabledLogProvider.mockResolvedValue(false);
mocks.remove.mockResolvedValue(undefined);
mocks.getService.mockReturnValue({ remove: mocks.remove });
const result = await syncWebVectorAgent();
expect(result.installed).toBe(false);
expect(mocks.hasEnabledLogProvider).toHaveBeenCalledWith("org-1");
});
it("skips the hasEnabledLogProvider query when providers are already preloaded", async () => {
mocks.getWebServerSettings.mockResolvedValue({
enableLogManagement: true,
logManagementOrganizationId: "org-1",
});
mocks.remove.mockResolvedValue(undefined);
mocks.getService.mockReturnValue({ remove: mocks.remove });
const result = await syncWebVectorAgent({ providers: [], lookup: {} });
expect(result.installed).toBe(false);
expect(mocks.hasEnabledLogProvider).not.toHaveBeenCalled();
});
});
apps/dokploy/__test__/traefik/server/update-server-config.test.ts
+1 -1
View File
@@ -25,7 +25,7 @@ const baseSettings: WebServerSettings = {
letsEncryptEmail: null,
sshPrivateKey: null,
enableDockerCleanup: false,
enableLogManagement: false,
logProviderIds: [],
logManagementOrganizationId: null,
buildsConcurrency: 1,
logCleanupCron: null,
apps/dokploy/components/dashboard/settings/log-management/handle-log-provider.tsx
+1 -6
View File
@@ -179,16 +179,11 @@ export const HandleLogProvider = ({ logProviderId }: Props) => {
...(logProviderId ? { logProviderId } : {}),
...payload,
} as any)
.then((result: any) => {
.then(() => {
utils.logProvider.all.invalidate();
toast.success(
logProviderId ? "Log provider updated" : "Log provider added",
);
if (result?.syncErrors && result.syncErrors.length > 0) {
toast.error(
`Failed to sync ${result.syncErrors.length} server(s) — they may still be shipping with the old config`,
);
}
setIsOpen(false);
})
.catch(() => {
apps/dokploy/components/dashboard/settings/log-management/show-log-management-servers.tsx
+257 -51
View File
@@ -1,5 +1,11 @@
import { Loader2, ScrollText, Server } from "lucide-react";
import Link from "next/link";
import { Copy, Loader2, ScrollText, Server } from "lucide-react";
import { useState } from "react";
import { toast } from "sonner";
import { logProviderIcons } from "@/components/icons/log-provider-icons";
import { AlertBlock } from "@/components/shared/alert-block";
import { DialogAction } from "@/components/shared/dialog-action";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
Card,
CardContent,
@@ -7,18 +13,110 @@ import {
CardHeader,
CardTitle,
} from "@/components/ui/card";
import { Checkbox } from "@/components/ui/checkbox";
import { Label } from "@/components/ui/label";
import { api } from "@/utils/api";
import { ToggleLogManagement } from "../servers/actions/toggle-log-management";
type Status = "running" | "stopped" | "unknown";
const statusBadge = (status: Status) => {
if (status === "running") {
return <Badge variant="green">Running</Badge>;
}
if (status === "stopped") {
return <Badge variant="secondary">Not deployed</Badge>;
}
return <Badge variant="destructive">Unreachable</Badge>;
};
export const ShowLogManagementServers = () => {
const { data: providers, isPending: isPendingProviders } =
api.logProvider.all.useQuery();
const { data: isCloud } = api.settings.isCloud.useQuery();
const { data: servers, isPending: isPendingServers } =
api.server.all.useQuery();
const utils = api.useUtils();
const [selection, setSelection] = useState<Record<string, string[]>>({});
const [pendingTarget, setPendingTarget] = useState<string | null>(null);
const [errors, setErrors] = useState<Record<string, string>>({});
const hasAnyProvider = (providers?.length ?? 0) > 0;
const isPending = isPendingProviders || isPendingServers;
const { data: providers } = api.logProvider.all.useQuery();
const { data: targets, isPending } = api.logProvider.serverStatus.useQuery(
undefined,
{
refetchOnWindowFocus: false,
staleTime: 30_000,
trpc: { context: { skipBatch: true } },
},
);
const { mutateAsync: deployOnServer } =
api.logProvider.deployOnServer.useMutation();
const { mutateAsync: removeOnServer } =
api.logProvider.removeOnServer.useMutation();
const keyOf = (serverId: string | null) => serverId ?? "local";
const idsFor = (serverId: string | null, saved: string[]) =>
selection[keyOf(serverId)] ?? saved;
const toggleProvider = (
serverId: string | null,
saved: string[],
logProviderId: string,
) => {
const key = keyOf(serverId);
const current = selection[key] ?? saved;
setSelection({
...selection,
[key]: current.includes(logProviderId)
? current.filter((id) => id !== logProviderId)
: [...current, logProviderId],
});
};
const setError = (serverId: string | null, message?: string) =>
setErrors((prev) => {
const next = { ...prev };
if (message) {
next[keyOf(serverId)] = message;
} else {
delete next[keyOf(serverId)];
}
return next;
});
const handleDeploy = async (serverId: string | null, ids: string[]) => {
setPendingTarget(keyOf(serverId));
setError(serverId);
await deployOnServer({ serverId, logProviderIds: ids })
.then(() => {
toast.success("Vector agent deployed");
utils.logProvider.serverStatus.invalidate();
})
.catch((error) => {
setError(
serverId,
error instanceof Error ? error.message : "Failed to deploy the agent",
);
})
.finally(() => setPendingTarget(null));
};
const handleRemove = async (serverId: string | null) => {
setPendingTarget(keyOf(serverId));
setError(serverId);
await removeOnServer({ serverId })
.then(() => {
toast.success("Vector agent removed");
setSelection((prev) => ({ ...prev, [keyOf(serverId)]: [] }));
utils.logProvider.serverStatus.invalidate();
})
.catch((error) => {
setError(
serverId,
error instanceof Error ? error.message : "Failed to remove the agent",
);
})
.finally(() => setPendingTarget(null));
};
const hasProviders = (providers?.length ?? 0) > 0;
return (
<div className="w-full">
@@ -30,17 +128,18 @@ export const ShowLogManagementServers = () => {
Servers
</CardTitle>
<CardDescription>
Pick which servers ship their container logs to the providers
above. Each one runs a Vector agent while it's enabled.
Pick which providers each server ships to, then deploy its Vector
agent. Changing a provider only takes effect on a server after you
redeploy it.
</CardDescription>
</CardHeader>
<CardContent className="space-y-2 py-8 border-t">
{isPending ? (
<div className="flex flex-row gap-2 items-center justify-center text-sm text-muted-foreground min-h-[15vh]">
<span>Loading...</span>
<span>Checking servers...</span>
<Loader2 className="animate-spin size-4" />
</div>
) : !hasAnyProvider ? (
) : !hasProviders ? (
<div className="flex flex-col items-center gap-3 min-h-[15vh] justify-center">
<ScrollText className="size-8 self-center text-muted-foreground" />
<span className="text-base text-muted-foreground text-center">
@@ -49,51 +148,158 @@ export const ShowLogManagementServers = () => {
</div>
) : (
<div className="flex flex-col gap-4">
{!isCloud && (
<div className="flex items-center justify-between bg-sidebar p-1 w-full rounded-lg">
<div className="flex items-center justify-between p-3.5 rounded-lg bg-background border w-full">
<div className="flex gap-2 flex-col">
<span className="text-sm font-medium">Web Server</span>
<span className="text-xs text-muted-foreground">
The machine running Dokploy
</span>
</div>
<ToggleLogManagement />
</div>
</div>
)}
{targets?.map((target) => {
const key = keyOf(target.serverId);
const ids = idsFor(target.serverId, target.logProviderIds);
const isBusy = pendingTarget === key;
const isDeployed = target.status === "running";
const hasEnabledSelected = ids.some(
(id) =>
providers?.find((p) => p.logProviderId === id)?.enabled,
);
{servers?.map((server) => (
<div
key={server.serverId}
className="flex items-center justify-between bg-sidebar p-1 w-full rounded-lg"
>
<div className="flex items-center justify-between p-3.5 rounded-lg bg-background border w-full">
<div className="flex gap-2 flex-col">
<span className="text-sm font-medium">
{server.name}
</span>
<span className="text-xs text-muted-foreground">
{server.ipAddress}
</span>
return (
<div
key={key}
className="flex flex-col gap-3 rounded-lg border p-4"
>
<div className="flex items-center justify-between">
<div className="flex flex-col">
<span className="text-sm font-medium">
{target.name}
</span>
{target.ipAddress && (
<span className="text-xs text-muted-foreground">
{target.ipAddress}
</span>
)}
</div>
<div className="flex items-center gap-2">
{statusBadge(target.status as Status)}
{isDeployed && (
<DialogAction
title="Remove the Vector agent"
description="This server will stop shipping container logs until you deploy it again. Continue?"
type="destructive"
onClick={() => handleRemove(target.serverId)}
>
<Button
variant="ghost"
size="sm"
isLoading={isBusy}
>
Remove
</Button>
</DialogAction>
)}
</div>
</div>
<ToggleLogManagement serverId={server.serverId} />
</div>
</div>
))}
{isCloud && servers?.length === 0 && (
<div className="flex flex-col gap-2">
<span className="text-xs font-medium">
Log providers
</span>
<div className="flex flex-col gap-2">
{providers?.map((provider) => {
const ProviderIcon =
logProviderIcons[
provider.providerType as keyof typeof logProviderIcons
];
return (
<div
key={provider.logProviderId}
className={`flex flex-row items-center gap-2 text-sm ${
provider.enabled ? "" : "opacity-60"
}`}
>
<Checkbox
id={`${key}-${provider.logProviderId}`}
disabled={!provider.enabled}
checked={ids.includes(provider.logProviderId)}
onCheckedChange={() =>
toggleProvider(
target.serverId,
target.logProviderIds,
provider.logProviderId,
)
}
/>
{ProviderIcon && (
<ProviderIcon className="size-4 shrink-0" />
)}
<Label
htmlFor={`${key}-${provider.logProviderId}`}
className={`font-normal ${
provider.enabled
? "cursor-pointer"
: "cursor-not-allowed"
}`}
>
{provider.name}
</Label>
{!provider.enabled && (
<Badge variant="secondary">Disabled</Badge>
)}
</div>
);
})}
</div>
</div>
{errors[key] && (
<AlertBlock type="error" className="items-start">
<div className="flex w-full flex-col gap-2">
<span className="font-medium">
Deploy failed — the config was not applied, this
server keeps shipping with its previous one.
</span>
<pre className="max-h-48 overflow-auto whitespace-pre-wrap break-all rounded-md bg-background/50 p-2 font-mono text-xs">
{errors[key]}
</pre>
<Button
variant="ghost"
size="sm"
className="w-fit"
onClick={() => {
navigator.clipboard.writeText(
errors[key] ?? "",
);
toast.success("Error copied");
}}
>
<Copy className="size-3.5" />
Copy
</Button>
</div>
</AlertBlock>
)}
<div className="flex items-center justify-end gap-3">
{ids.length > 0 && !hasEnabledSelected && (
<span className="text-xs text-muted-foreground">
Every selected provider is disabled — nothing would
ship.
</span>
)}
<Button
size="sm"
isLoading={isBusy}
disabled={!hasEnabledSelected}
onClick={() => handleDeploy(target.serverId, ids)}
>
{isDeployed ? "Redeploy" : "Deploy"}
</Button>
</div>
</div>
);
})}
{targets?.length === 0 && (
<div className="flex flex-col items-center gap-3 min-h-[15vh] justify-center">
<Server className="size-8 self-center text-muted-foreground" />
<span className="text-base text-muted-foreground text-center">
You don't have any server yet
</span>
<Link
href="/dashboard/settings/servers"
className="text-sm text-primary underline"
>
Go to Servers
</Link>
</div>
)}
</div>
apps/dokploy/components/dashboard/settings/log-management/show-log-providers.tsx
+3 -68
View File
@@ -1,10 +1,4 @@
import {
AlertTriangle,
CheckCircle2,
Loader2,
ScrollText,
Trash2,
} from "lucide-react";
import { Loader2, ScrollText, Trash2 } from "lucide-react";
import { toast } from "sonner";
import { logProviderIcons } from "@/components/icons/log-provider-icons";
import { DialogAction } from "@/components/shared/dialog-action";
@@ -44,9 +38,6 @@ const LogProviderRow = ({
}: LogProviderRowProps) => {
const { mutateAsync, isPending: isRemoving } =
api.logProvider.remove.useMutation();
const { mutateAsync: testConnectionById, isPending: isTesting } =
api.logProvider.testConnectionById.useMutation();
const ProviderIcon =
logProviderIcons[provider.providerType as keyof typeof logProviderIcons];
@@ -67,35 +58,6 @@ const LogProviderRow = ({
</div>
<div className="flex flex-row gap-1">
{canEdit && (
<Button
variant="ghost"
size="sm"
isLoading={isTesting}
onClick={async () => {
await testConnectionById({
logProviderId: provider.logProviderId,
})
.then((result) => {
if (result.warning) {
toast.message(result.warning);
} else {
toast.success("Connection tested successfully");
}
})
.catch((error) => {
toast.error(
error instanceof Error
? error.message
: "Connection test failed",
);
});
}}
>
<CheckCircle2 className="size-4" />
Test
</Button>
)}
{canEdit && (
<HandleLogProvider logProviderId={provider.logProviderId} />
)}
@@ -109,13 +71,8 @@ const LogProviderRow = ({
await mutateAsync({
logProviderId: provider.logProviderId,
})
.then((result) => {
.then(() => {
toast.success("Log provider deleted successfully");
if (result.syncErrors && result.syncErrors.length > 0) {
toast.error(
`Failed to sync ${result.syncErrors.length} server(s) — they may still be shipping with the old config`,
);
}
onDeleted();
})
.catch(() => {
@@ -144,21 +101,10 @@ export const ShowLogProviders = () => {
const { data, isPending, refetch } = api.logProvider.all.useQuery();
const { data: availableTypes } = api.logProvider.availableTypes.useQuery();
const { data: permissions } = api.user.getPermissions.useQuery();
const { data: webServerSettings } =
api.settings.getWebServerSettings.useQuery();
const { data: servers } = api.server.all.useQuery();
const providerListLabel = availableTypes?.length
? ` (${availableTypes.map((t) => t.label).join(", ")})`
: "";
const hasAnyProvider = (data?.length ?? 0) > 0;
const knowsAboutServers = servers !== undefined;
const hasActiveTarget =
!!webServerSettings?.enableLogManagement ||
!!servers?.some((s) => s.enableLogManagement);
const showNoActiveTargetBanner =
hasAnyProvider && knowsAboutServers && !hasActiveTarget;
return (
<div className="w-full">
<Card className="h-full bg-sidebar p-2.5 rounded-xl max-w-5xl mx-auto">
@@ -169,21 +115,10 @@ export const ShowLogProviders = () => {
Log Management
</CardTitle>
<CardDescription>
{`Ship container logs to an external provider${providerListLabel}. Needs at least one provider here and Log Management turned on for the servers below.`}
{`Ship container logs to an external provider${providerListLabel}. Add one here, then deploy the Vector agent on the servers below.`}
</CardDescription>
</CardHeader>
<CardContent className="space-y-2 py-8 border-t">
{showNoActiveTargetBanner && (
<div className="flex items-start gap-3 rounded-lg border border-yellow-500/30 bg-yellow-500/10 p-3 mb-4">
<AlertTriangle className="size-4 text-yellow-600 dark:text-yellow-400 mt-0.5 shrink-0" />
<div className="flex flex-col gap-2 text-sm">
<span className="text-yellow-700 dark:text-yellow-400">
Nothing will ship yet — turn on Log Management for at least
one server below.
</span>
</div>
</div>
)}
{isPending ? (
<div className="flex flex-row gap-2 items-center justify-center text-sm text-muted-foreground min-h-[25vh]">
<span>Loading...</span>
apps/dokploy/components/dashboard/settings/servers/actions/toggle-log-management.tsx
-125
View File
@@ -1,125 +0,0 @@
import { HelpCircle, Loader2 } from "lucide-react";
import { toast } from "sonner";
import { Label } from "@/components/ui/label";
import { Switch } from "@/components/ui/switch";
import {
Tooltip,
TooltipContent,
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { api } from "@/utils/api";
interface Props {
serverId?: string;
}
export const ToggleLogManagement = ({ serverId }: Props) => {
const utils = api.useUtils();
const { data: server } = api.server.one.useQuery(
{ serverId: serverId || "" },
{ enabled: !!serverId },
);
const { data: webServerSettings } =
api.settings.getWebServerSettings.useQuery(undefined, {
enabled: !serverId,
});
const { mutateAsync: updateServer, isPending: isServerPending } =
api.server.updateLogManagement.useMutation();
const { mutateAsync: updateLocal, isPending: isLocalPending } =
api.settings.updateLogManagement.useMutation();
const isPending = serverId ? isServerPending : isLocalPending;
const enabled = serverId
? !!server?.enableLogManagement
: !!webServerSettings?.enableLogManagement;
const handleToggle = async (checked: boolean) => {
if (serverId) {
await updateServer({ serverId, enableLogManagement: checked })
.then((result) => {
utils.server.one.setData({ serverId }, (old) =>
old
? { ...old, enableLogManagement: result.enableLogManagement }
: old,
);
if (checked && !result.installed) {
toast.message(
"Log Management enabled, but no log provider is configured yet — add one under Settings > Log Management.",
);
} else {
toast.success("Log Management updated");
}
})
.catch((e) => {
toast.error(
e instanceof Error ? e.message : "Error updating Log Management",
);
});
return;
}
await updateLocal({ enableLogManagement: checked })
.then((result) => {
utils.settings.getWebServerSettings.setData(undefined, (old) =>
old
? { ...old, enableLogManagement: result.enableLogManagement }
: old,
);
if (checked && !result.installed) {
toast.message(
"Log Management enabled, but no log provider is configured yet — add one under Settings > Log Management.",
);
} else {
toast.success("Log Management updated");
}
})
.catch((e) => {
toast.error(
e instanceof Error ? e.message : "Error updating Log Management",
);
});
};
return (
<div className="flex items-center gap-4">
<Switch
checked={enabled}
onCheckedChange={handleToggle}
disabled={isPending}
/>
{isPending && (
<Loader2 className="size-4 animate-spin text-muted-foreground" />
)}
<TooltipProvider delayDuration={0}>
<Tooltip>
<TooltipTrigger asChild>
<Label className="text-primary flex items-center gap-1.5 cursor-pointer">
Log Management
<HelpCircle className="size-4 text-muted-foreground" />
</Label>
</TooltipTrigger>
<TooltipContent side="top" className="max-w-sm flex-col items-start">
<p>
Ships {serverId ? "this server's" : "this machine's"} container
logs to the log providers configured under Settings &gt; Log
Management, via a Vector agent. May take a few seconds to install
or remove.
</p>
<p className="mt-1">
Has no effect (and installs nothing) unless at least one log
provider is enabled for this organization.
</p>
{!serverId && (
<p className="mt-1">
Ships every container on this machine, not just this
organization's — if another organization on this instance also
deploys here, its logs ship too (unscoped, no project/app tags).
</p>
)}
</TooltipContent>
</Tooltip>
</TooltipProvider>
</div>
);
};
apps/dokploy/components/dashboard/settings/servers/handle-servers.tsx
+1 -4
View File
@@ -126,13 +126,10 @@ export const HandleServers = ({ serverId, asButton = false }: Props) => {
enableDockerCleanup: data.enableDockerCleanup,
serverId: serverId || "",
})
.then(async (data) => {
.then(async () => {
await utils.server.all.invalidate();
refetchServer();
toast.success(serverId ? "Server Updated" : "Server Created");
if ("vectorAgentWarning" in data && data.vectorAgentWarning) {
toast.error(data.vectorAgentWarning);
}
setIsOpen(false);
})
.catch(() => {
apps/dokploy/drizzle/0198_kind_hemingway.sql
+4
View File
@@ -0,0 +1,4 @@
ALTER TABLE "server" ADD COLUMN "logProviderIds" text[] DEFAULT ARRAY[]::text[] NOT NULL;--> statement-breakpoint
ALTER TABLE "webServerSettings" ADD COLUMN "logProviderIds" text[] DEFAULT ARRAY[]::text[] NOT NULL;--> statement-breakpoint
ALTER TABLE "server" DROP COLUMN "enableLogManagement";--> statement-breakpoint
ALTER TABLE "webServerSettings" DROP COLUMN "enableLogManagement";
apps/dokploy/drizzle/meta/0198_snapshot.json
+9305
View File
File diff suppressed because it is too large Load Diff
apps/dokploy/drizzle/meta/_journal.json
+7
View File
@@ -1387,6 +1387,13 @@
"when": 1789042957674,
"tag": "0197_past_omega_red",
"breakpoints": true
},
{
"idx": 198,
"version": "7",
"when": 1790203407320,
"tag": "0198_kind_hemingway",
"breakpoints": true
}
]
}
apps/dokploy/server/api/routers/log-provider.ts
+110 -13
View File
@@ -1,14 +1,21 @@
import {
claimWebServerLogManagement,
createLogProvider,
findLogProviderById,
findLogProvidersByOrganization,
findServerById,
getLogManagementServerStatus,
logProviderAdapters,
removeLogProvider,
removeVectorAgent,
sanitizeLogProvider,
setupVectorAgent,
testLogProviderConnection,
updateLogProvider,
updateServerLogProviders,
} from "@dokploy/server";
import { TRPCError } from "@trpc/server";
import { z } from "zod";
import { audit } from "@/server/api/utils/audit";
import {
apiCreateLogProvider,
@@ -17,9 +24,46 @@ import {
apiTestLogProvider,
apiUpdateLogProvider,
} from "@/server/db/schema";
import { safeSyncVectorAgentsForOrganization } from "@/server/utils/vector-resync";
import { createTRPCRouter, protectedProcedure, withPermission } from "../trpc";
const assertServerBelongsToOrg = async (
serverId: string | undefined,
organizationId: string,
) => {
if (!serverId) return;
const server = await findServerById(serverId);
if (server.organizationId !== organizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You are not authorized to access this server",
});
}
};
const assertLogProvidersBelongToOrg = async (
logProviderIds: string[],
organizationId: string,
) => {
const providers = await findLogProvidersByOrganization(organizationId);
const byId = new Map(providers.map((p) => [p.logProviderId, p]));
for (const id of logProviderIds) {
if (!byId.has(id)) {
throw new TRPCError({
code: "UNAUTHORIZED",
message:
"One of the selected log providers is not in this organization",
});
}
}
if (!logProviderIds.some((id) => byId.get(id)?.enabled)) {
throw new TRPCError({
code: "BAD_REQUEST",
message:
"Select at least one enabled log provider — a disabled one ships nothing.",
});
}
};
export const logProviderRouter = createTRPCRouter({
create: withPermission("logProvider", "create")
.input(apiCreateLogProvider)
@@ -34,10 +78,7 @@ export const logProviderRouter = createTRPCRouter({
resourceId: provider.logProviderId,
resourceName: provider.name,
});
const syncErrors = await safeSyncVectorAgentsForOrganization(
ctx.session.activeOrganizationId,
);
return { ...sanitizeLogProvider(provider), syncErrors };
return sanitizeLogProvider(provider);
}),
update: withPermission("logProvider", "create")
.input(apiUpdateLogProvider)
@@ -57,10 +98,7 @@ export const logProviderRouter = createTRPCRouter({
resourceId: logProviderId,
resourceName: provider.name,
});
const syncErrors = await safeSyncVectorAgentsForOrganization(
ctx.session.activeOrganizationId,
);
return { ...sanitizeLogProvider(updated), syncErrors };
return sanitizeLogProvider(updated);
}),
remove: withPermission("logProvider", "delete")
.input(apiRemoveLogProvider)
@@ -79,10 +117,7 @@ export const logProviderRouter = createTRPCRouter({
resourceId: provider.logProviderId,
resourceName: provider.name,
});
const syncErrors = await safeSyncVectorAgentsForOrganization(
ctx.session.activeOrganizationId,
);
return { ...sanitizeLogProvider(removed), syncErrors };
return sanitizeLogProvider(removed);
}),
all: withPermission("logProvider", "read").query(async ({ ctx }) => {
return await findLogProvidersByOrganization(
@@ -130,6 +165,68 @@ export const logProviderRouter = createTRPCRouter({
logProviderId: input.logProviderId,
});
}),
serverStatus: withPermission("logProvider", "read").query(({ ctx }) =>
getLogManagementServerStatus(ctx.session.activeOrganizationId),
),
deployOnServer: withPermission("logProvider", "create")
.input(
z.object({
serverId: z.string().nullable().optional(),
logProviderIds: z.array(z.string()).min(1),
}),
)
.mutation(async ({ ctx, input }) => {
const organizationId = ctx.session.activeOrganizationId;
const serverId = input.serverId ?? undefined;
await assertServerBelongsToOrg(serverId, organizationId);
await assertLogProvidersBelongToOrg(input.logProviderIds, organizationId);
if (serverId) {
await updateServerLogProviders(serverId, input.logProviderIds);
} else {
const claimed = await claimWebServerLogManagement(
organizationId,
input.logProviderIds,
);
if (!claimed) {
throw new TRPCError({
code: "CONFLICT",
message:
"The local Vector agent is already claimed by another organization",
});
}
}
await setupVectorAgent(organizationId, serverId, input.logProviderIds);
await audit(ctx, {
action: "create",
resourceType: "server",
resourceId: input.serverId ?? "local",
resourceName: "log-management",
});
return { installed: true };
}),
removeOnServer: withPermission("logProvider", "create")
.input(z.object({ serverId: z.string().nullable().optional() }))
.mutation(async ({ ctx, input }) => {
const organizationId = ctx.session.activeOrganizationId;
const serverId = input.serverId ?? undefined;
await assertServerBelongsToOrg(serverId, organizationId);
await removeVectorAgent(serverId);
if (serverId) {
await updateServerLogProviders(serverId, []);
} else {
await claimWebServerLogManagement(organizationId, []);
}
await audit(ctx, {
action: "delete",
resourceType: "server",
resourceId: input.serverId ?? "local",
resourceName: "log-management",
});
return { installed: false };
}),
availableTypes: protectedProcedure.query(() => {
return Object.values(logProviderAdapters).map((adapter) => ({
type: adapter.type,
apps/dokploy/server/api/routers/organization.ts
+2 -6
View File
@@ -1,5 +1,6 @@
import { db } from "@dokploy/server/db";
import {
deleteOrganization,
hasValidLicense,
IS_CLOUD,
sendInvitationEmail,
@@ -20,7 +21,6 @@ import {
organizationRole,
user,
} from "@/server/db/schema";
import { teardownVectorForOrganizationDeletion } from "@/server/utils/vector-resync";
import { createTRPCRouter, protectedProcedure, withPermission } from "../trpc";
export const organizationRouter = createTRPCRouter({
create: protectedProcedure
@@ -286,11 +286,7 @@ export const organizationRouter = createTRPCRouter({
});
}
await teardownVectorForOrganizationDeletion(input.organizationId);
const result = await db
.delete(organization)
.where(eq(organization.id, input.organizationId));
const result = await deleteOrganization(input.organizationId);
await audit(ctx, {
action: "delete",
apps/dokploy/server/api/routers/server.ts
+11 -135
View File
@@ -51,10 +51,6 @@ import {
server,
} from "@/server/db/schema";
import { applyDockerCleanupSchedule } from "@/server/utils/docker-cleanup";
import {
applyVectorResyncSchedule,
syncVectorAgentAndSchedule,
} from "@/server/utils/vector-resync";
export const serverRouter = createTRPCRouter({
create: withPermission("server", "create")
@@ -458,73 +454,6 @@ export const serverRouter = createTRPCRouter({
throw error;
}
}),
setupLogManagement: withPermission("server", "create")
.input(apiFindOneServer)
.mutation(async ({ input, ctx }) => {
const server = await findServerById(input.serverId);
if (server.organizationId !== ctx.session.activeOrganizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You are not authorized to setup this server",
});
}
const result = await syncVectorAgentAndSchedule(input.serverId);
await audit(ctx, {
action: "update",
resourceType: "server",
resourceId: input.serverId,
resourceName: server.name,
});
return result;
}),
updateLogManagement: withPermission("server", "create")
.input(
z.object({
serverId: z.string().min(1),
enableLogManagement: z.boolean(),
}),
)
.mutation(async ({ input, ctx }) => {
const server = await findServerById(input.serverId);
if (server.organizationId !== ctx.session.activeOrganizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You are not authorized to update this server",
});
}
await updateServerById(input.serverId, {
enableLogManagement: input.enableLogManagement,
});
try {
const result = await syncVectorAgentAndSchedule(input.serverId);
await audit(ctx, {
action: "update",
resourceType: "server",
resourceId: input.serverId,
resourceName: server.name,
});
return { ...result, enableLogManagement: input.enableLogManagement };
} catch (error) {
await updateServerById(input.serverId, {
enableLogManagement: server.enableLogManagement,
});
await syncVectorAgentAndSchedule(input.serverId).catch(
(teardownError) => {
console.error(
`[Vector] Failed to tear down agent for server ${input.serverId} after a failed sync — it may still be running:`,
teardownError,
);
},
);
throw new TRPCError({
code: "BAD_REQUEST",
message:
error instanceof Error
? `Failed to sync Vector agent: ${error.message}`
: "Failed to sync Vector agent",
});
}
}),
remove: withPermission("server", "delete")
.input(apiRemoveServer)
.mutation(async ({ input, ctx }) => {
@@ -546,18 +475,15 @@ export const serverRouter = createTRPCRouter({
});
}
let vectorAgentRemovalWarning: string | undefined;
if (currentServer.enableLogManagement) {
await removeVectorAgent({ serverId: input.serverId }).catch(
(error) => {
vectorAgentRemovalWarning =
error instanceof Error ? error.message : String(error);
console.error(
`[Vector] Failed to remove agent for server ${input.serverId} before deletion:`,
error,
);
},
);
await applyVectorResyncSchedule(input.serverId, false);
if ((currentServer.logProviderIds?.length ?? 0) > 0) {
await removeVectorAgent(input.serverId).catch((error) => {
vectorAgentRemovalWarning =
error instanceof Error ? error.message : String(error);
console.error(
`[Vector] Failed to remove agent for server ${input.serverId} before deletion:`,
error,
);
});
}
await audit(ctx, {
action: "delete",
@@ -604,33 +530,7 @@ export const serverRouter = createTRPCRouter({
});
}
const connectionChanged =
server.ipAddress !== input.ipAddress ||
server.port !== input.port ||
server.username !== input.username ||
server.sshKeyId !== input.sshKeyId;
let vectorAgentWarning: string | undefined;
const recordVectorWarning = (prefix: string, error: unknown) => {
const message = `${prefix}: ${error instanceof Error ? error.message : String(error)}`;
vectorAgentWarning = vectorAgentWarning
? `${vectorAgentWarning} ${message}`
: message;
console.error(`[Vector] ${message}`);
};
if (connectionChanged && server.enableLogManagement) {
await removeVectorAgent({ serverId: input.serverId }).catch(
(error) => {
recordVectorWarning(
"Failed to remove the agent from the old host",
error,
);
},
);
}
let currentServer = await updateServerById(input.serverId, {
const currentServer = await updateServerById(input.serverId, {
...input,
});
@@ -640,37 +540,13 @@ export const serverRouter = createTRPCRouter({
input.enableDockerCleanup,
);
if (connectionChanged && server.enableLogManagement) {
try {
await syncVectorAgentAndSchedule(input.serverId);
} catch (error) {
currentServer =
(await updateServerById(input.serverId, {
enableLogManagement: false,
})) ?? currentServer;
await syncVectorAgentAndSchedule(input.serverId).catch(
(teardownError) => {
console.error(
`[Vector] Failed to tear down agent for server ${input.serverId} after a failed sync — it may still be running:`,
teardownError,
);
},
);
recordVectorWarning(
"Failed to install the agent on the new host, log management has been turned off for this server",
error,
);
}
}
await audit(ctx, {
action: "update",
resourceType: "server",
resourceId: input.serverId,
resourceName: server.name,
...(vectorAgentWarning ? { metadata: { vectorAgentWarning } } : {}),
});
return { ...currentServer, vectorAgentWarning };
return currentServer;
} catch (error) {
throw error;
}
apps/dokploy/server/api/routers/settings.ts
-71
View File
@@ -4,7 +4,6 @@ import {
checkPortInUse,
checkPostgresHealth,
checkTraefikHealth,
claimWebServerLogManagement,
cleanupAll,
cleanupAllBackground,
cleanupBuilders,
@@ -72,7 +71,6 @@ import {
} from "@/server/db/schema";
import { cleanAllDeploymentQueue } from "@/server/queues/queueSetup";
import { removeJob, schedule } from "@/server/utils/backup";
import { syncWebVectorAgentAndSchedule } from "@/server/utils/vector-resync";
import packageInfo from "../../../package.json";
import { appRouter } from "../root";
import {
@@ -412,75 +410,6 @@ export const settingsRouter = createTRPCRouter({
return true;
}),
updateLogManagement: adminProcedure
.input(z.object({ enableLogManagement: z.boolean() }))
.mutation(async ({ input, ctx }) => {
if (IS_CLOUD) {
throw new TRPCError({
code: "BAD_REQUEST",
message: "This feature is only available for self-hosted instances",
});
}
const activeOrganizationId = ctx.session.activeOrganizationId;
const current = await getWebServerSettings();
const currentOwner = current?.logManagementOrganizationId ?? null;
if (currentOwner && currentOwner !== activeOrganizationId) {
throw new TRPCError({
code: "FORBIDDEN",
message: input.enableLogManagement
? "The local Vector agent is already used by another organization on this instance — disable it there first."
: "You are not authorized to disable the local Vector agent — it belongs to another organization.",
});
}
const previous = {
enableLogManagement: !!current?.enableLogManagement,
logManagementOrganizationId: currentOwner,
};
const claimed = await claimWebServerLogManagement(
activeOrganizationId,
input.enableLogManagement,
);
if (!claimed) {
throw new TRPCError({
code: "FORBIDDEN",
message:
"The local Vector agent was just claimed by another organization — try again.",
});
}
try {
const result =
await syncWebVectorAgentAndSchedule(activeOrganizationId);
await audit(ctx, {
action: "update",
resourceType: "settings",
resourceName: "log-management",
});
return { ...result, enableLogManagement: input.enableLogManagement };
} catch (error) {
await updateWebServerSettings(previous);
await syncWebVectorAgentAndSchedule(activeOrganizationId).catch(
(teardownError) => {
console.error(
"[Vector] Failed to reconcile local agent after a failed sync:",
teardownError,
);
},
);
throw new TRPCError({
code: "BAD_REQUEST",
message:
error instanceof Error
? `Failed to sync local Vector agent: ${error.message}`
: "Failed to sync local Vector agent",
});
}
}),
updateRemoteServersOnly: enterpriseProcedure
.input(z.object({ remoteServersOnly: z.boolean() }))
.mutation(async ({ input, ctx }) => {
apps/dokploy/server/server.ts
-3
View File
@@ -16,7 +16,6 @@ import {
import { config } from "dotenv";
import next from "next";
import packageInfo from "../package.json";
import { initVectorResyncSchedules } from "./utils/vector-resync";
import { setupDockerContainerLogsWebSocketServer } from "./wss/docker-container-logs";
import { setupDockerContainerTerminalWebSocketServer } from "./wss/docker-container-terminal";
import { setupDockerStatsMonitoringSocketServer } from "./wss/docker-stats";
@@ -66,10 +65,8 @@ void app.prepare().then(async () => {
await initSchedules();
await initCancelDeployments();
await initVolumeBackupsCronJobs();
await initVectorResyncSchedules();
await sendDokployRestartNotifications();
} else if (IS_CLOUD) {
await initVectorResyncSchedules();
}
await initEnterpriseBackupCronJobs();
apps/dokploy/server/utils/vector-resync.ts
-249
View File
@@ -1,249 +0,0 @@
import {
findAllServersWithLogManagementEnabled,
findServersWithLogManagementEnabled,
getWebServerSettings,
hasEnabledLogProvider,
IS_CLOUD,
loadVectorOrgData,
removeVectorAgent,
removeWebVectorAgent,
syncVectorAgent,
syncVectorConfig,
syncWebVectorAgent,
syncWebVectorConfig,
updateWebServerSettings,
VECTOR_RESYNC_CRON_JOB,
type VectorOrgData,
} from "@dokploy/server";
import { TRPCError } from "@trpc/server";
import { scheduledJobs, scheduleJob } from "node-schedule";
export const applyVectorResyncSchedule = async (
serverId: string,
enable: boolean,
) => {
const jobName = `vector-resync:${serverId}`;
if (IS_CLOUD) {
console.warn(
`[Vector] Periodic resync for server ${serverId} is running in-process, not via the Cloud Jobs service (no handler for "vector-resync" yet).`,
);
}
if (enable) {
scheduleJob(jobName, VECTOR_RESYNC_CRON_JOB, async () => {
try {
await syncVectorConfig({ serverId });
} catch (error) {
console.error(`[Vector] Resync failed for server ${serverId}:`, error);
if (error instanceof TRPCError && error.code === "NOT_FOUND") {
scheduledJobs[jobName]?.cancel();
}
}
});
} else {
scheduledJobs[jobName]?.cancel();
}
};
const WEB_VECTOR_RESYNC_JOB_NAME = "vector-resync-web";
export const applyWebVectorResyncSchedule = async (
organizationId: string,
enable: boolean,
) => {
if (enable) {
scheduleJob(
WEB_VECTOR_RESYNC_JOB_NAME,
VECTOR_RESYNC_CRON_JOB,
async () => {
try {
await syncWebVectorConfig(organizationId);
} catch (error) {
console.error("[Vector] Local resync failed:", error);
}
},
);
} else {
scheduledJobs[WEB_VECTOR_RESYNC_JOB_NAME]?.cancel();
}
};
export const initVectorResyncSchedules = async () => {
let servers: Awaited<
ReturnType<typeof findAllServersWithLogManagementEnabled>
>;
try {
servers = await findAllServersWithLogManagementEnabled();
} catch (error) {
console.error(
"[Vector] Failed to load servers for resync schedules:",
error,
);
return;
}
const hasProviderByOrg = new Map<
string,
ReturnType<typeof hasEnabledLogProvider>
>();
const hasProviderCached = (organizationId: string) => {
let result = hasProviderByOrg.get(organizationId);
if (!result) {
result = hasEnabledLogProvider(organizationId);
hasProviderByOrg.set(organizationId, result);
}
return result;
};
await Promise.all(
servers.map(async (s) => {
try {
if (await hasProviderCached(s.organizationId)) {
await applyVectorResyncSchedule(s.serverId, true);
}
} catch (error) {
console.error(
`[Vector] Failed to re-register resync schedule for server ${s.serverId}:`,
error,
);
}
}),
);
if (!IS_CLOUD) {
try {
const settings = await getWebServerSettings();
const organizationId = settings?.logManagementOrganizationId;
if (
settings?.enableLogManagement &&
organizationId &&
(await hasEnabledLogProvider(organizationId))
) {
await applyWebVectorResyncSchedule(organizationId, true);
}
} catch (error) {
console.error(
"[Vector] Failed to re-register local resync schedule:",
error,
);
}
}
};
export const syncVectorAgentAndSchedule = async (
serverId: string,
preloaded?: VectorOrgData,
) => {
const { installed } = await syncVectorAgent({ serverId, preloaded });
await applyVectorResyncSchedule(serverId, installed);
return { installed };
};
export const syncWebVectorAgentAndSchedule = async (
organizationId: string,
preloaded?: VectorOrgData,
) => {
const { installed } = await syncWebVectorAgent(preloaded);
await applyWebVectorResyncSchedule(organizationId, installed);
return { installed };
};
export const syncVectorAgentsForOrganization = async (
organizationId: string,
): Promise<Array<{ serverId: string; error: string }>> => {
const servers = await findServersWithLogManagementEnabled(organizationId);
const webOwnsThisOrg =
!IS_CLOUD &&
(await getWebServerSettings())?.logManagementOrganizationId ===
organizationId;
if (servers.length === 0 && !webOwnsThisOrg) {
return [];
}
const preloaded = await loadVectorOrgData(organizationId);
const targets: Array<{
id: string;
run: () => Promise<{ installed: boolean }>;
}> = [
...servers.map((s) => ({
id: s.serverId,
run: () => syncVectorAgentAndSchedule(s.serverId, preloaded),
})),
...(webOwnsThisOrg
? [
{
id: "web",
run: () => syncWebVectorAgentAndSchedule(organizationId, preloaded),
},
]
: []),
];
const results = await Promise.allSettled(targets.map((t) => t.run()));
const errors: Array<{ serverId: string; error: string }> = [];
results.forEach((result, index) => {
if (result.status === "rejected") {
const target = targets[index];
if (target) {
errors.push({
serverId: target.id,
error:
result.reason instanceof Error
? result.reason.message
: String(result.reason),
});
}
}
});
return errors;
};
export const safeSyncVectorAgentsForOrganization = async (
organizationId: string,
) => {
try {
return await syncVectorAgentsForOrganization(organizationId);
} catch (error) {
return [
{
serverId: "unknown",
error: error instanceof Error ? error.message : String(error),
},
];
}
};
export const teardownVectorForOrganizationDeletion = async (
organizationId: string,
): Promise<void> => {
const servers = await findServersWithLogManagementEnabled(organizationId);
await Promise.all(
servers.map(async (s) => {
await removeVectorAgent({ serverId: s.serverId }).catch((error) => {
console.error(
`[Vector] Failed to remove agent for server ${s.serverId} before deleting organization ${organizationId}:`,
error,
);
});
await applyVectorResyncSchedule(s.serverId, false);
}),
);
if (!IS_CLOUD) {
const settings = await getWebServerSettings();
if (settings?.logManagementOrganizationId === organizationId) {
await removeWebVectorAgent().catch((error) => {
console.error(
`[Vector] Failed to remove local agent before deleting organization ${organizationId}:`,
error,
);
});
await applyWebVectorResyncSchedule(organizationId, false);
await updateWebServerSettings({
enableLogManagement: false,
logManagementOrganizationId: null,
});
}
}
};
apps/dokploy/utils/api.ts
+2 -1
View File
@@ -52,7 +52,8 @@ const links =
transformer: superjson,
}),
false: splitLink({
condition: (op) => op.input instanceof FormData,
condition: (op) =>
op.input instanceof FormData || op.context.skipBatch === true,
true: httpLink({
url: `${getBaseUrl()}/api/trpc`,
transformer: superjson,
packages/server/src/constants/index.ts
-2
View File
@@ -13,8 +13,6 @@ export const DOKPLOY_DOCKER_PORT = process.env.DOKPLOY_DOCKER_PORT
export const CLEANUP_CRON_JOB = "50 23 * * *";
export const VECTOR_RESYNC_CRON_JOB = "*/15 * * * *";
// Body size limits for the OpenAPI catch-all route (pages/api/[...trpc].ts).
const parseByteSize = (envVar: string, fallback: number): number => {
const raw = process.env[envVar];
packages/server/src/db/schema/server.ts
+5 -2
View File
@@ -1,4 +1,4 @@
import { relations } from "drizzle-orm";
import { relations, sql } from "drizzle-orm";
import {
boolean,
integer,
@@ -42,7 +42,10 @@ export const server = pgTable("server", {
.notNull()
.$defaultFn(() => generateAppName("server")),
enableDockerCleanup: boolean("enableDockerCleanup").notNull().default(false),
enableLogManagement: boolean("enableLogManagement").notNull().default(false),
logProviderIds: text("logProviderIds")
.array()
.notNull()
.default(sql`ARRAY[]::text[]`),
buildsConcurrency: integer("buildsConcurrency").notNull().default(1),
createdAt: text("createdAt").notNull(),
organizationId: text("organizationId")
packages/server/src/db/schema/web-server-settings.ts
+5 -2
View File
@@ -1,4 +1,4 @@
import { relations } from "drizzle-orm";
import { relations, sql } from "drizzle-orm";
import {
boolean,
integer,
@@ -27,7 +27,10 @@ export const webServerSettings = pgTable("webServerSettings", {
sshPrivateKey: text("sshPrivateKey"),
enableDockerCleanup: boolean("enableDockerCleanup").notNull().default(true),
logCleanupCron: text("logCleanupCron").default("0 0 * * *"),
enableLogManagement: boolean("enableLogManagement").notNull().default(false),
logProviderIds: text("logProviderIds")
.array()
.notNull()
.default(sql`ARRAY[]::text[]`),
logManagementOrganizationId: text("logManagementOrganizationId").references(
() => organization.id,
{ onDelete: "set null" },
packages/server/src/index.ts
+1
View File
@@ -36,6 +36,7 @@ export * from "./services/mount";
export * from "./services/mysql";
export * from "./services/network";
export * from "./services/notification";
export * from "./services/organization";
export * from "./services/overview";
export * from "./services/patch";
export * from "./services/patch-repo";
packages/server/src/services/log-management/providers/elasticsearch.ts
+1
View File
@@ -117,6 +117,7 @@ export const elasticsearchAdapter: LogProviderAdapter = {
type: "elasticsearch",
inputs: [inputId],
endpoints: [normalizeEndpointUrl(config.endpoint ?? "")],
api_version: "v8",
...(typeof index === "string" && index.length > 0
? { bulk: { index } }
: {}),
packages/server/src/services/organization.ts
+26
View File
@@ -0,0 +1,26 @@
import { db } from "@dokploy/server/db";
import { organization } from "@dokploy/server/db/schema";
import { findServersWithLogManagementEnabled } from "@dokploy/server/services/server";
import { removeVectorAgent } from "@dokploy/server/setup/vector-setup";
import { eq } from "drizzle-orm";
const removeVectorAgents = async (organizationId: string) => {
const servers = await findServersWithLogManagementEnabled(organizationId);
await Promise.all(
servers.map((server) =>
removeVectorAgent(server.serverId).catch((error) => {
console.error(
`[Vector] Failed to remove the agent on server ${server.serverId} before deleting organization ${organizationId}:`,
error,
);
}),
),
);
};
export const deleteOrganization = async (organizationId: string) => {
await removeVectorAgents(organizationId);
return await db
.delete(organization)
.where(eq(organization.id, organizationId));
};
packages/server/src/services/server.ts
+29 -5
View File
@@ -7,7 +7,7 @@ import {
} from "@dokploy/server/db/schema";
import { hasValidLicense } from "@dokploy/server/services/proprietary/license-key";
import { TRPCError } from "@trpc/server";
import { and, eq } from "drizzle-orm";
import { and, desc, eq, isNotNull, sql } from "drizzle-orm";
import type { z } from "zod";
export type Server = typeof server.$inferSelect;
@@ -93,19 +93,43 @@ export const findServersWithLogManagementEnabled = async (
return await db.query.server.findMany({
where: and(
eq(server.organizationId, organizationId),
eq(server.enableLogManagement, true),
sql`array_length(${server.logProviderIds}, 1) > 0`,
),
columns: { serverId: true },
});
};
export const findAllServersWithLogManagementEnabled = async () => {
export const findServersByOrganizationForLogManagement = async (
organizationId: string,
) => {
return await db.query.server.findMany({
where: eq(server.enableLogManagement, true),
columns: { serverId: true, organizationId: true },
where: and(
eq(server.organizationId, organizationId),
isNotNull(server.sshKeyId),
eq(server.serverType, "deploy"),
),
columns: {
serverId: true,
name: true,
ipAddress: true,
logProviderIds: true,
},
orderBy: [desc(server.createdAt)],
});
};
export const updateServerLogProviders = async (
serverId: string,
logProviderIds: string[],
) => {
const [updated] = await db
.update(server)
.set({ logProviderIds })
.where(eq(server.serverId, serverId))
.returning();
return updated ?? null;
};
export const deleteServer = async (serverId: string) => {
const currentServer = await db
.delete(server)
packages/server/src/services/web-server-settings.ts
+4 -3
View File
@@ -45,7 +45,7 @@ export const updateWebServerSettings = async (
export const claimWebServerLogManagement = async (
organizationId: string,
enableLogManagement: boolean,
logProviderIds: string[],
) => {
const current = await getWebServerSettings();
if (!current) {
@@ -55,8 +55,9 @@ export const claimWebServerLogManagement = async (
const [updated] = await db
.update(webServerSettings)
.set({
enableLogManagement,
logManagementOrganizationId: enableLogManagement ? organizationId : null,
logProviderIds,
logManagementOrganizationId:
logProviderIds.length > 0 ? organizationId : null,
updatedAt: new Date(),
})
.where(
packages/server/src/setup/vector-setup.ts
+205 -451
View File
@@ -1,33 +1,24 @@
import {
chmodSync as fsChmodSync,
mkdirSync,
renameSync,
rmSync,
writeFileSync,
} from "node:fs";
import { paths } from "@dokploy/server/constants";
import { db } from "@dokploy/server/db";
import { dbUrl } from "@dokploy/server/db/constants";
import { projects } from "@dokploy/server/db/schema";
import { IS_CLOUD, paths } from "@dokploy/server/constants";
import {
findEnabledLogProvidersByOrganization,
hasEnabledLogProvider,
type LogProvider,
toRuntimeConfig,
} from "@dokploy/server/services/log-management/service";
import { findServerById } from "@dokploy/server/services/server";
import {
findServerById,
findServersByOrganizationForLogManagement,
} from "@dokploy/server/services/server";
import { getWebServerSettings } from "@dokploy/server/services/web-server-settings";
import { pullImage, pullRemoteImage } from "@dokploy/server/utils/docker/utils";
import { encodeBase64 } from "@dokploy/server/utils/docker/utils";
import { ExecError } from "@dokploy/server/utils/process/ExecError";
import {
execAsync,
execAsyncRemote,
writeFileRemote,
} from "@dokploy/server/utils/process/execAsync";
import { getRemoteDocker } from "@dokploy/server/utils/servers/remote-docker";
import type { CreateServiceOptions } from "dockerode";
import { eq } from "drizzle-orm";
import postgres from "postgres";
import { nanoid } from "nanoid";
import { quote } from "shell-quote";
import { stringify } from "yaml";
import { getLogProviderAdapter } from "../services/log-management/providers/registry";
import type {
@@ -41,124 +32,17 @@ const VECTOR_CONFIG_CONTAINER_DIR = "/etc/vector";
const VECTOR_CONFIG_CONTAINER_PATH = `${VECTOR_CONFIG_CONTAINER_DIR}/vector.yaml`;
const VECTOR_DATA_DIR_CONTAINER = "/var/lib/vector";
interface AppNameLookupEntry {
projectId: string;
projectName: string;
environmentId: string;
environmentName: string;
applicationId: string;
applicationName: string;
}
const escapeVrlString = (value: string): string =>
value
.replace(/\\/g, "\\\\")
.replace(/"/g, '\\"')
// biome-ignore lint/suspicious/noControlCharactersInRegex: replacing them is the point
.replace(/[\u0000-\u001f\u007f]/g, " ");
const buildAppNameLookup = async (
organizationId: string,
): Promise<Record<string, AppNameLookupEntry>> => {
const projectRows = await db.query.projects.findMany({
where: eq(projects.organizationId, organizationId),
columns: { projectId: true, name: true },
with: {
environments: {
columns: { environmentId: true, name: true },
with: {
applications: {
columns: { applicationId: true, appName: true, name: true },
},
compose: {
columns: { composeId: true, appName: true, name: true },
},
},
},
},
});
const lookup: Record<string, AppNameLookupEntry> = {};
for (const project of projectRows) {
for (const environment of project.environments) {
for (const application of environment.applications) {
lookup[application.appName] = {
projectId: project.projectId,
projectName: project.name,
environmentId: environment.environmentId,
environmentName: environment.name,
applicationId: application.applicationId,
applicationName: application.name,
};
}
for (const composeService of environment.compose) {
lookup[composeService.appName] = {
projectId: project.projectId,
projectName: project.name,
environmentId: environment.environmentId,
environmentName: environment.name,
applicationId: composeService.composeId,
applicationName: composeService.name,
};
}
}
}
return lookup;
};
const buildScopeTransformSource = (
organizationId: string,
lookup: Record<string, AppNameLookupEntry>,
): string => {
const lines: string[] = [
`.dokploy_organization = "${escapeVrlString(organizationId)}"`,
'.dokploy_project = ""',
'.dokploy_project_id = ""',
'.dokploy_environment = ""',
'.dokploy_environment_id = ""',
'.dokploy_application = ""',
'.dokploy_application_id = ""',
];
const entries = Object.entries(lookup);
if (entries.length === 0) {
return lines.join("\n");
}
lines.push('app_name = .label."com.docker.compose.project"');
lines.push("if app_name == null {");
lines.push(' app_name = .label."com.docker.stack.namespace"');
lines.push("}");
lines.push("if app_name == null {");
lines.push(' app_name = .label."com.docker.swarm.service.name"');
lines.push("}");
lines.push("if app_name != null {");
entries.forEach(([appName, entry], index) => {
const branch = index === 0 ? " if" : " } else if";
lines.push(`${branch} app_name == "${escapeVrlString(appName)}" {`);
lines.push(
` .dokploy_project = "${escapeVrlString(entry.projectName)}"`,
);
lines.push(
` .dokploy_project_id = "${escapeVrlString(entry.projectId)}"`,
);
lines.push(
` .dokploy_environment = "${escapeVrlString(entry.environmentName)}"`,
);
lines.push(
` .dokploy_environment_id = "${escapeVrlString(entry.environmentId)}"`,
);
lines.push(
` .dokploy_application = "${escapeVrlString(entry.applicationName)}"`,
);
lines.push(
` .dokploy_application_id = "${escapeVrlString(entry.applicationId)}"`,
);
});
lines.push(" }");
lines.push("}");
return lines.join("\n");
};
const buildScopeTransformSource = (): string =>
[
'.dokploy_organization = .label."dokploy.organization.id" || ""',
'.dokploy_project = .label."dokploy.project" || ""',
'.dokploy_project_id = .label."dokploy.project.id" || ""',
'.dokploy_environment = .label."dokploy.environment" || ""',
'.dokploy_environment_id = .label."dokploy.environment.id" || ""',
'.dokploy_application = .label."dokploy.application" || ""',
'.dokploy_application_id = .label."dokploy.application.id" || ""',
'.dokploy_service = .label."dokploy.service" || ""',
].join("\n");
const buildSinksAndTransforms = (
providers: Array<LogProvider>,
@@ -206,28 +90,22 @@ const buildSinksAndTransforms = (
return { sinks, transforms };
};
export interface VectorOrgData {
providers: Array<LogProvider>;
lookup: Record<string, AppNameLookupEntry>;
}
export const loadVectorOrgData = async (
const loadEnabledLogProviders = async (
organizationId: string,
): Promise<VectorOrgData> => {
const [providers, lookup] = await Promise.all([
findEnabledLogProvidersByOrganization(organizationId),
buildAppNameLookup(organizationId),
]);
return { providers, lookup };
logProviderIds?: string[],
): Promise<Array<LogProvider>> => {
const providers = await findEnabledLogProvidersByOrganization(organizationId);
return logProviderIds
? providers.filter((provider) =>
logProviderIds.includes(provider.logProviderId),
)
: providers;
};
export const buildVectorConfigYaml = async (
organizationId: string,
preloaded?: VectorOrgData,
providers: Array<LogProvider>,
options?: { dropUnmatched?: boolean },
) => {
const { providers, lookup } =
preloaded ?? (await loadVectorOrgData(organizationId));
const baseTransformId = options?.dropUnmatched
? "dokploy_scope_local_only"
: "dokploy_scope";
@@ -248,7 +126,7 @@ export const buildVectorConfigYaml = async (
dokploy_scope: {
type: "remap",
inputs: ["docker_logs_source"],
source: buildScopeTransformSource(organizationId, lookup),
source: buildScopeTransformSource(),
},
...(options?.dropUnmatched
? {
@@ -290,138 +168,89 @@ export const redactSecrets = (text: string, secrets: string[]): string => {
return redacted;
};
const VECTOR_CONFIG_LOCK_NAMESPACE = 87_234_501;
const hashServerIdToInt32 = (serverId: string): number => {
let hash = 0;
for (let i = 0; i < serverId.length; i++) {
hash = (Math.imul(31, hash) + serverId.charCodeAt(i)) | 0;
}
return hash;
};
let lockPool: ReturnType<typeof postgres> | undefined;
const getLockPool = () => {
if (!lockPool) {
lockPool = postgres(dbUrl, { max: 10 });
}
return lockPool;
};
const LOCK_RETRY_ATTEMPTS = 3;
const LOCK_RETRY_DELAY_MS = 1000;
const configWriteLocks = new Map<string, Promise<void>>();
export const withConfigWriteLock = async <T>(
serverId: string,
fn: () => Promise<T>,
): Promise<T> => {
const previous = configWriteLocks.get(serverId) ?? Promise.resolve();
let release: () => void = () => {};
const current = new Promise<void>((resolve) => {
release = resolve;
});
configWriteLocks.set(
serverId,
previous.then(() => current),
);
await previous;
try {
return await runWithCrossProcessLock(serverId, fn);
} finally {
release();
}
};
const runWithCrossProcessLock = async <T>(
serverId: string,
fn: () => Promise<T>,
): Promise<T> => {
const key1 = VECTOR_CONFIG_LOCK_NAMESPACE;
const key2 = hashServerIdToInt32(serverId);
for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) {
const reserved = await getLockPool().reserve();
try {
const rows =
await reserved`select pg_try_advisory_lock(${key1}, ${key2}) as locked`;
if (rows[0]?.locked) {
try {
return await fn();
} finally {
await reserved`select pg_advisory_unlock(${key1}, ${key2})`;
}
}
} finally {
reserved.release();
}
if (attempt < LOCK_RETRY_ATTEMPTS) {
await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_DELAY_MS));
}
}
throw new Error(
`Vector config sync for server ${serverId} is already in progress on another Dokploy instance, skipped`,
);
};
export const syncVectorConfig = async ({
serverId,
preloaded,
}: {
serverId: string;
preloaded?: VectorOrgData;
}) =>
withConfigWriteLock(serverId, async () => {
const getServerLogProviderIds = async (
serverId?: string,
): Promise<string[]> => {
if (serverId) {
const server = await findServerById(serverId);
const { VECTOR_PATH } = paths(true);
const configPath = `${VECTOR_PATH}/vector.yaml`;
const candidatePath = `${VECTOR_PATH}/vector.yaml.candidate`;
const orgData =
preloaded ?? (await loadVectorOrgData(server.organizationId));
const yamlStr = await buildVectorConfigYaml(server.organizationId, orgData);
return server.logProviderIds ?? [];
}
const settings = await getWebServerSettings();
return settings?.logProviderIds ?? [];
};
await execAsyncRemote(serverId, `mkdir -p ${VECTOR_PATH}/data`);
await writeFileRemote(serverId, candidatePath, yamlStr);
await execAsyncRemote(
serverId,
`chmod 600 ${candidatePath} && chmod 700 ${VECTOR_PATH} && chmod 700 ${VECTOR_PATH}/data`,
);
try {
await execAsyncRemote(
serverId,
`docker run --rm -v /var/run/docker.sock:/var/run/docker.sock:ro -v ${candidatePath}:/etc/vector/vector.yaml:ro ${VECTOR_IMAGE} validate --skip-healthchecks /etc/vector/vector.yaml`,
);
} catch (error) {
const detail =
error instanceof ExecError
? error.stderr || error.stdout || error.message
: error instanceof Error
? error.message
: String(error);
const sanitizedDetail = redactSecrets(
detail,
collectSecretValues(orgData.providers),
);
await execAsyncRemote(serverId, `rm -f ${candidatePath}`).catch(
(cleanupError) => {
console.error(
`[Vector] Failed to remove invalid config candidate ${candidatePath} on server ${serverId}:`,
cleanupError,
);
},
);
throw new Error(
`Generated Vector config failed validation, not applying it: ${sanitizedDetail}`,
);
}
await execAsyncRemote(serverId, `mv ${candidatePath} ${configPath}`);
const syncVectorConfig = async (
organizationId: string,
serverId?: string,
logProviderIds?: string[],
) => {
const { VECTOR_PATH } = paths(!!serverId);
const configPath = `${VECTOR_PATH}/vector.yaml`;
const candidatePath = `${VECTOR_PATH}/vector.yaml.candidate.${nanoid(10)}`;
const ids = logProviderIds ?? (await getServerLogProviderIds(serverId));
const providers = await loadEnabledLogProviders(organizationId, ids);
const yamlStr = await buildVectorConfigYaml(providers, {
dropUnmatched: !serverId,
});
export const buildServiceSettings = (
vectorPath: string,
): CreateServiceOptions => {
const VECTOR_PATH = vectorPath;
return {
const writeCommand = `umask 077 && mkdir -p ${quote([`${VECTOR_PATH}/data`])} && chmod 700 ${quote([VECTOR_PATH])} ${quote([`${VECTOR_PATH}/data`])} && echo "${encodeBase64(yamlStr)}" | base64 -d > ${quote([candidatePath])}`;
if (serverId) {
await execAsyncRemote(serverId, writeCommand);
} else {
await execAsync(writeCommand);
}
const validateCommand = `docker run --rm -v /var/run/docker.sock:/var/run/docker.sock:ro -v ${quote([`${candidatePath}:/etc/vector/vector.yaml:ro`])} ${VECTOR_IMAGE} validate --skip-healthchecks /etc/vector/vector.yaml`;
try {
if (serverId) {
await execAsyncRemote(serverId, validateCommand);
} else {
await execAsync(validateCommand);
}
} catch (error) {
const detail =
error instanceof ExecError
? error.stderr || error.stdout || error.message
: error instanceof Error
? error.message
: String(error);
const sanitizedDetail = redactSecrets(
detail,
collectSecretValues(providers),
);
const cleanupCommand = `rm -f ${quote([candidatePath])}`;
try {
if (serverId) {
await execAsyncRemote(serverId, cleanupCommand);
} else {
await execAsync(cleanupCommand);
}
} catch (cleanupError) {
console.error(
`[Vector] Failed to remove invalid config candidate ${candidatePath}${
serverId ? ` on server ${serverId}` : ""
}:`,
cleanupError,
);
}
throw new Error(
`Generated Vector config failed validation, not applying it: ${sanitizedDetail}`,
);
}
const applyCommand = `mv ${quote([candidatePath])} ${quote([configPath])}`;
if (serverId) {
await execAsyncRemote(serverId, applyCommand);
} else {
await execAsync(applyCommand);
}
};
const deployVectorService = async (serverId?: string) => {
const { VECTOR_PATH } = paths(!!serverId);
const docker = await getRemoteDocker(serverId);
const settings: CreateServiceOptions = {
Name: VECTOR_SERVICE_NAME,
TaskTemplate: {
ContainerSpec: {
@@ -456,40 +285,10 @@ export const buildServiceSettings = (
},
},
};
};
export const stripImageDigest = (image: string): string =>
image.split("@")[0] ?? image;
export const vectorServiceSpecUnchanged = (
inspect: { Spec: { TaskTemplate: any; Mode?: unknown } },
settings: CreateServiceOptions,
): boolean => {
const desiredTaskTemplate = settings.TaskTemplate as any;
const existing = inspect.Spec.TaskTemplate?.ContainerSpec ?? {};
const desired = desiredTaskTemplate?.ContainerSpec ?? {};
return (
stripImageDigest(existing.Image ?? "") ===
stripImageDigest(desired.Image ?? "") &&
JSON.stringify(existing.Args ?? []) ===
JSON.stringify(desired.Args ?? []) &&
JSON.stringify(existing.Mounts ?? []) ===
JSON.stringify(desired.Mounts ?? []) &&
JSON.stringify(inspect.Spec.Mode ?? {}) ===
JSON.stringify(settings.Mode ?? {})
);
};
const deployVectorService = async (
docker: Awaited<ReturnType<typeof getRemoteDocker>>,
settings: CreateServiceOptions,
) => {
try {
const service = docker.getService(VECTOR_SERVICE_NAME);
const inspect = await service.inspect();
if (vectorServiceSpecUnchanged(inspect, settings)) {
return;
}
await service.update({
version: Number.parseInt(inspect.Version.Index, 10),
...settings,
@@ -498,130 +297,31 @@ const deployVectorService = async (
ForceUpdate: (inspect.Spec.TaskTemplate.ForceUpdate ?? 0) + 1,
},
});
} catch (error: any) {
if (error?.statusCode !== 404) {
throw error;
}
await docker.createService(settings);
}
};
export const setupVectorAgent = async ({
serverId,
preloaded,
}: {
serverId: string;
preloaded?: VectorOrgData;
}) => {
await syncVectorConfig({ serverId, preloaded });
const { VECTOR_PATH } = paths(true);
await pullRemoteImage(VECTOR_IMAGE, serverId);
const docker = await getRemoteDocker(serverId);
await deployVectorService(docker, buildServiceSettings(VECTOR_PATH));
};
export const removeVectorAgent = async ({ serverId }: { serverId: string }) => {
const server = await findServerById(serverId);
if (!server.sshKeyId) {
throw new Error(`No SSH key configured for server ${serverId}`);
}
const docker = await getRemoteDocker(serverId);
try {
await docker.getService(VECTOR_SERVICE_NAME).remove();
} catch (error: any) {
if (error?.statusCode !== 404) {
throw error;
}
}
const { VECTOR_PATH } = paths(true);
await execAsyncRemote(serverId, `rm -rf ${VECTOR_PATH}`);
};
export const syncVectorAgent = async ({
serverId,
preloaded,
}: {
serverId: string;
preloaded?: VectorOrgData;
}): Promise<{ installed: boolean }> => {
const server = await findServerById(serverId);
const hasProvider = preloaded
? preloaded.providers.length > 0
: await hasEnabledLogProvider(server.organizationId);
const shouldRun = server.enableLogManagement && hasProvider;
if (shouldRun) {
await setupVectorAgent({ serverId, preloaded });
} else {
await removeVectorAgent({ serverId });
}
return { installed: shouldRun };
};
export const syncWebVectorConfig = async (
organizationId: string,
preloaded?: VectorOrgData,
) =>
withConfigWriteLock("web", async () => {
const { VECTOR_PATH } = paths();
const configPath = `${VECTOR_PATH}/vector.yaml`;
const candidatePath = `${VECTOR_PATH}/vector.yaml.candidate`;
const orgData = preloaded ?? (await loadVectorOrgData(organizationId));
const yamlStr = await buildVectorConfigYaml(organizationId, orgData, {
dropUnmatched: true,
});
mkdirSync(`${VECTOR_PATH}/data`, { recursive: true });
writeFileSync(candidatePath, yamlStr, "utf8");
fsChmodSync(candidatePath, 0o600);
fsChmodSync(VECTOR_PATH, 0o700);
fsChmodSync(`${VECTOR_PATH}/data`, 0o700);
console.log("Vector Updated ✅");
} catch (_) {
try {
await execAsync(
`docker run --rm -v /var/run/docker.sock:/var/run/docker.sock:ro -v ${candidatePath}:/etc/vector/vector.yaml:ro ${VECTOR_IMAGE} validate --skip-healthchecks /etc/vector/vector.yaml`,
);
} catch (error) {
const detail =
error instanceof ExecError
? error.stderr || error.stdout || error.message
: error instanceof Error
? error.message
: String(error);
const sanitizedDetail = redactSecrets(
detail,
collectSecretValues(orgData.providers),
);
try {
rmSync(candidatePath, { force: true });
} catch (cleanupError) {
console.error(
`[Vector] Failed to remove invalid local config candidate ${candidatePath}:`,
cleanupError,
);
await docker.createService(settings);
console.log("Vector Started ✅");
} catch (error: any) {
if (error?.statusCode !== 409) {
throw error;
}
throw new Error(
`Generated Vector config failed validation, not applying it: ${sanitizedDetail}`,
);
console.log("Vector service already exists, continuing...");
}
renameSync(candidatePath, configPath);
});
export const setupWebVectorAgent = async (
organizationId: string,
preloaded?: VectorOrgData,
) => {
await syncWebVectorConfig(organizationId, preloaded);
const { VECTOR_PATH } = paths();
await pullImage(VECTOR_IMAGE);
const docker = await getRemoteDocker();
await deployVectorService(docker, buildServiceSettings(VECTOR_PATH));
}
};
export const removeWebVectorAgent = async () => {
const docker = await getRemoteDocker();
export const setupVectorAgent = async (
organizationId: string,
serverId?: string,
logProviderIds?: string[],
) => {
await syncVectorConfig(organizationId, serverId, logProviderIds);
await deployVectorService(serverId);
};
export const removeVectorAgent = async (serverId?: string) => {
const docker = await getRemoteDocker(serverId);
try {
await docker.getService(VECTOR_SERVICE_NAME).remove();
} catch (error: any) {
@@ -629,28 +329,82 @@ export const removeWebVectorAgent = async () => {
throw error;
}
}
const { VECTOR_PATH } = paths();
rmSync(VECTOR_PATH, { recursive: true, force: true });
};
export const syncWebVectorAgent = async (
preloaded?: VectorOrgData,
): Promise<{ installed: boolean }> => {
const settings = await getWebServerSettings();
const organizationId = settings?.logManagementOrganizationId ?? null;
const hasProvider = organizationId
? preloaded
? preloaded.providers.length > 0
: await hasEnabledLogProvider(organizationId)
: false;
const shouldRun =
!!settings?.enableLogManagement && !!organizationId && hasProvider;
if (shouldRun && organizationId) {
await setupWebVectorAgent(organizationId, preloaded);
const { VECTOR_PATH } = paths(!!serverId);
const cleanupCommand = `rm -rf ${quote([VECTOR_PATH])}`;
if (serverId) {
await execAsyncRemote(serverId, cleanupCommand);
} else {
await removeWebVectorAgent();
await execAsync(cleanupCommand);
}
return { installed: shouldRun };
};
const isVectorRunning = async (serverId?: string): Promise<boolean> => {
const docker = await getRemoteDocker(serverId);
try {
await docker.getService(VECTOR_SERVICE_NAME).inspect();
return true;
} catch {
return false;
}
};
const VECTOR_CHECK_TIMEOUT_MS = 5000;
const vectorStatus = async (
serverId: string | null,
): Promise<"running" | "stopped" | "unknown"> => {
try {
const running = await Promise.race([
isVectorRunning(serverId ?? undefined),
new Promise<never>((_, reject) =>
setTimeout(() => reject(new Error("timeout")), VECTOR_CHECK_TIMEOUT_MS),
),
]);
return running ? "running" : "stopped";
} catch {
return "unknown";
}
};
export const getLogManagementServerStatus = async (organizationId: string) => {
const [servers, settings] = await Promise.all([
findServersByOrganizationForLogManagement(organizationId),
IS_CLOUD ? Promise.resolve(null) : getWebServerSettings(),
]);
const webOwnsThisOrg =
!!settings &&
(settings.logManagementOrganizationId === null ||
settings.logManagementOrganizationId === organizationId);
const targets: Array<{
serverId: string | null;
name: string;
ipAddress: string | null;
logProviderIds: string[];
}> = [
...(settings && webOwnsThisOrg
? [
{
serverId: null,
name: "Dokploy Server (local)",
ipAddress: null,
logProviderIds: settings.logProviderIds ?? [],
},
]
: []),
...servers.map((s) => ({
serverId: s.serverId,
name: s.name,
ipAddress: s.ipAddress,
logProviderIds: s.logProviderIds ?? [],
})),
];
return Promise.all(
targets.map(async (target) => ({
...target,
status: await vectorStatus(target.serverId),
})),
);
};
packages/server/src/utils/builders/index.ts
+13 -1
View File
@@ -3,6 +3,7 @@ import { findRegistryByIdWithCredentials } from "@dokploy/server/services/regist
import type { InferResultType } from "@dokploy/server/types/with";
import type { CreateServiceOptions } from "dockerode";
import { getRegistryTag, uploadImageRemoteCommand } from "../cluster/upload";
import { buildDokployLabels } from "../docker/dokploy-labels";
import {
calculateResources,
generateBindMounts,
@@ -150,7 +151,18 @@ export const mechanizeDockerContainer = async (
Args: args,
}),
...(Ulimits && { Ulimits }),
Labels,
Labels: {
...Labels,
...buildDokployLabels({
organizationId: application.environment.project.organizationId,
projectId: application.environment.projectId,
projectName: application.environment.project.name,
environmentId: application.environmentId,
environmentName: application.environment.name,
applicationId: application.applicationId,
applicationName: application.name,
}),
},
},
Networks: resolvedNetworks,
RestartPolicy,
packages/server/src/utils/docker/dokploy-labels.ts
+34
View File
@@ -0,0 +1,34 @@
export interface DokployResourceMetadata {
organizationId: string;
projectId: string;
projectName: string;
environmentId: string;
environmentName: string;
applicationId: string;
applicationName: string;
serviceName?: string;
}
export const DOKPLOY_LABEL_PREFIX = "dokploy";
export const buildDokployLabels = (
metadata: DokployResourceMetadata,
): Record<string, string> => ({
[`${DOKPLOY_LABEL_PREFIX}.organization.id`]: metadata.organizationId,
[`${DOKPLOY_LABEL_PREFIX}.project.id`]: metadata.projectId,
[`${DOKPLOY_LABEL_PREFIX}.project`]: metadata.projectName,
[`${DOKPLOY_LABEL_PREFIX}.environment.id`]: metadata.environmentId,
[`${DOKPLOY_LABEL_PREFIX}.environment`]: metadata.environmentName,
[`${DOKPLOY_LABEL_PREFIX}.application.id`]: metadata.applicationId,
[`${DOKPLOY_LABEL_PREFIX}.application`]: metadata.applicationName,
...(metadata.serviceName
? { [`${DOKPLOY_LABEL_PREFIX}.service`]: metadata.serviceName }
: {}),
});
export const dokployLabelsAsList = (
metadata: DokployResourceMetadata,
): string[] =>
Object.entries(buildDokployLabels(metadata)).map(
([key, value]) => `${key}=${value}`,
);
packages/server/src/utils/docker/domain.ts
+63 -2
View File
@@ -17,6 +17,7 @@ import { cloneGitlabRepository } from "../providers/gitlab";
import { getCreateComposeFileCommand } from "../providers/raw";
import { randomizeDeployableSpecificationFile } from "./collision";
import { randomizeSpecificationFile } from "./compose";
import { buildDokployLabels } from "./dokploy-labels";
import type {
ComposeSpecification,
DefinitionsService,
@@ -111,7 +112,7 @@ export const readComposeFile = async (compose: Compose) => {
};
export const writeDomainsToCompose = async (
compose: Compose,
compose: ComposeWithMetadata,
domains: Domain[],
) => {
try {
@@ -196,8 +197,66 @@ const removeDomainLabels = (
return labels;
};
type ComposeWithMetadata = Compose & {
environment?: {
environmentId: string;
name: string;
projectId: string;
project?: { name: string; organizationId: string };
};
};
const applyDokployLabelsToCompose = (
result: ComposeSpecification,
compose: ComposeWithMetadata,
) => {
const { environment } = compose;
if (!environment?.project) return;
for (const [serviceName, service] of Object.entries(result.services ?? {})) {
if (!service) continue;
const dokployLabels = buildDokployLabels({
organizationId: environment.project.organizationId,
projectId: environment.projectId,
projectName: environment.project.name,
environmentId: environment.environmentId,
environmentName: environment.name,
applicationId: compose.composeId,
applicationName: compose.name,
serviceName,
});
if (compose.composeType === "docker-compose") {
service.labels = mergeDokployLabels(service.labels, dokployLabels);
continue;
}
if (!service.deploy) {
service.deploy = {};
}
service.deploy.labels = mergeDokployLabels(
service.deploy.labels,
dokployLabels,
);
}
};
const mergeDokployLabels = (
current: DefinitionsService["labels"],
dokployLabels: Record<string, string>,
): DefinitionsService["labels"] => {
if (current && !Array.isArray(current)) {
return { ...current, ...dokployLabels };
}
const entries = Object.entries(dokployLabels);
const keys = new Set(entries.map(([key]) => key));
const kept = (current ?? []).filter(
(label) => !keys.has(label.slice(0, label.indexOf("="))),
);
return [...kept, ...entries.map(([key, value]) => `${key}=${value}`)];
};
export const addDomainToCompose = async (
compose: Compose,
compose: ComposeWithMetadata,
domains: Domain[],
) => {
const { appName } = compose;
@@ -326,6 +385,8 @@ export const addDomainToCompose = async (
}
}
applyDokployLabelsToCompose(result, compose);
const injectedNetworkNames = await applyServiceNetworks(result, compose);
if (!compose.isolatedDeployment) {