mirror of
https://github.com/Dokploy/dokploy.git
synced 2026-10-20 07:31:37 +00:00
fix(log-management): scope local agent per organization and harden provider operations
This commit is contained in:
@@ -33,8 +33,16 @@ services:
|
||||
volumes:
|
||||
test-data:
|
||||
`,
|
||||
composeId: "compose-1",
|
||||
name: "raw-stack",
|
||||
composePath: "./docker-compose.yml",
|
||||
composeType: "stack",
|
||||
environment: {
|
||||
environmentId: "env-1",
|
||||
name: "production",
|
||||
projectId: "project-1",
|
||||
project: { name: "shop", organizationId: "org-1" },
|
||||
},
|
||||
isolatedDeployment: false,
|
||||
isolatedDeploymentsVolume: false,
|
||||
randomize: false,
|
||||
@@ -54,6 +62,16 @@ volumes:
|
||||
},
|
||||
]);
|
||||
expect(converted?.services?.test?.tmpfs).toEqual(["/cache"]);
|
||||
expect(converted?.services?.test?.labels).toEqual(
|
||||
expect.arrayContaining([
|
||||
"dokploy.organization.id=org-1",
|
||||
"dokploy.project=shop",
|
||||
"dokploy.service=test",
|
||||
]),
|
||||
);
|
||||
expect(converted?.services?.test?.deploy?.labels).toEqual(
|
||||
expect.arrayContaining(["dokploy.organization.id=org-1"]),
|
||||
);
|
||||
expect(execAsyncRemote).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,11 +8,16 @@ vi.mock("node:dns/promises", () => ({
|
||||
lookup: mocks.lookup,
|
||||
}));
|
||||
|
||||
const { logProviderFetch } = await import(
|
||||
"@dokploy/server/services/log-management/types"
|
||||
);
|
||||
const loadFetch = async (isCloud: boolean) => {
|
||||
vi.resetModules();
|
||||
vi.doMock("@dokploy/server/constants", () => ({ IS_CLOUD: isCloud }));
|
||||
const { logProviderFetch } = await import(
|
||||
"@dokploy/server/services/log-management/types"
|
||||
);
|
||||
return logProviderFetch;
|
||||
};
|
||||
|
||||
describe("logProviderFetch — metadata endpoint guard", () => {
|
||||
describe("logProviderFetch — blocked endpoint guard", () => {
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -25,6 +30,7 @@ describe("logProviderFetch — metadata endpoint guard", () => {
|
||||
});
|
||||
|
||||
it("rejects a literal AWS/GCP/Azure metadata IP without ever calling fetch", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
await expect(
|
||||
logProviderFetch("http://169.254.169.254/latest/meta-data/"),
|
||||
).rejects.toThrow(/metadata/i);
|
||||
@@ -32,7 +38,19 @@ describe("logProviderFetch — metadata endpoint guard", () => {
|
||||
expect(mocks.lookup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects the metadata IP written as an IPv4-mapped IPv6 address", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
for (const address of ["[::ffff:169.254.169.254]", "[::ffff:a9fe:a9fe]"]) {
|
||||
await expect(
|
||||
logProviderFetch(`http://${address}/latest/meta-data/`),
|
||||
`expected ${address} to be rejected`,
|
||||
).rejects.toThrow(/metadata/i);
|
||||
}
|
||||
expect(global.fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects the AWS ECS metadata IPv6 address", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
await expect(
|
||||
logProviderFetch("http://[fd00:ec2::254]/v2/credentials"),
|
||||
).rejects.toThrow(/metadata/i);
|
||||
@@ -40,6 +58,7 @@ describe("logProviderFetch — metadata endpoint guard", () => {
|
||||
});
|
||||
|
||||
it("rejects every fe80::/10 link-local address, not just literal fe80: ones", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
for (const address of [
|
||||
"[fe80::1]",
|
||||
"[fe90::1]",
|
||||
@@ -54,7 +73,16 @@ describe("logProviderFetch — metadata endpoint guard", () => {
|
||||
expect(global.fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not block a hostname just because it starts like a link-local prefix", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
mocks.lookup.mockResolvedValue([{ address: "203.0.113.10", family: 4 }]);
|
||||
|
||||
await logProviderFetch("https://feature-logs.example.com/ready");
|
||||
expect(global.fetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects the GCP metadata hostname without needing DNS resolution", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
await expect(
|
||||
logProviderFetch("http://metadata.google.internal/computeMetadata/v1/"),
|
||||
).rejects.toThrow(/metadata/i);
|
||||
@@ -63,6 +91,7 @@ describe("logProviderFetch — metadata endpoint guard", () => {
|
||||
});
|
||||
|
||||
it("rejects a hostname whose DNS record resolves to a metadata address", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
mocks.lookup.mockResolvedValue([{ address: "169.254.169.254", family: 4 }]);
|
||||
|
||||
await expect(
|
||||
@@ -71,14 +100,39 @@ describe("logProviderFetch — metadata endpoint guard", () => {
|
||||
expect(global.fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows a private-network endpoint (self-hosted Loki behind a VPN, say)", async () => {
|
||||
it("allows a private-network endpoint when self-hosted (Loki behind a VPN, say)", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
mocks.lookup.mockResolvedValue([{ address: "10.0.5.20", family: 4 }]);
|
||||
|
||||
await logProviderFetch("http://loki.internal.example.com:3100/ready");
|
||||
expect(global.fetch).toHaveBeenCalledTimes(1);
|
||||
await logProviderFetch("http://10.0.0.5:3100/ready");
|
||||
expect(global.fetch).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("rejects loopback and private-network endpoints in cloud, literal or resolved", async () => {
|
||||
const logProviderFetch = await loadFetch(true);
|
||||
mocks.lookup.mockResolvedValue([{ address: "10.0.5.20", family: 4 }]);
|
||||
|
||||
for (const url of [
|
||||
"http://10.0.0.5:3100/ready",
|
||||
"http://127.0.0.1:3100/ready",
|
||||
"http://localhost:3100/ready",
|
||||
"http://[::1]:3100/ready",
|
||||
"http://0.0.0.0:3100/ready",
|
||||
"http://100.100.100.200/latest/meta-data/",
|
||||
"http://[::]:3100/ready",
|
||||
"http://loki.internal.example.com:3100/ready",
|
||||
]) {
|
||||
await expect(
|
||||
logProviderFetch(url),
|
||||
`expected ${url} to be rejected`,
|
||||
).rejects.toThrow(/blocked/i);
|
||||
}
|
||||
expect(global.fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows a normal public endpoint", async () => {
|
||||
const logProviderFetch = await loadFetch(true);
|
||||
mocks.lookup.mockResolvedValue([{ address: "203.0.113.10", family: 4 }]);
|
||||
|
||||
await logProviderFetch(
|
||||
@@ -87,7 +141,19 @@ describe("logProviderFetch — metadata endpoint guard", () => {
|
||||
expect(global.fetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not follow redirects", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
mocks.lookup.mockResolvedValue([{ address: "203.0.113.10", family: 4 }]);
|
||||
|
||||
await logProviderFetch("https://logs.example.com/ready");
|
||||
expect(global.fetch).toHaveBeenCalledWith(
|
||||
"https://logs.example.com/ready",
|
||||
expect.objectContaining({ redirect: "error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("still calls fetch (letting it surface its own error) when DNS resolution fails", async () => {
|
||||
const logProviderFetch = await loadFetch(false);
|
||||
mocks.lookup.mockRejectedValue(new Error("ENOTFOUND"));
|
||||
|
||||
await logProviderFetch("http://does-not-exist.invalid/");
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import type { SQL } from "drizzle-orm";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
deleted: [] as unknown[],
|
||||
updates: [] as Array<{ table: unknown; values: Record<string, unknown> }>,
|
||||
}));
|
||||
|
||||
vi.mock("@dokploy/server/db", () => {
|
||||
const tx = {
|
||||
delete: () => ({
|
||||
where: () => ({
|
||||
returning: () => Promise.resolve(mocks.deleted),
|
||||
}),
|
||||
}),
|
||||
update: (table: unknown) => ({
|
||||
set: (values: Record<string, unknown>) => ({
|
||||
where: () => {
|
||||
mocks.updates.push({ table, values });
|
||||
return Promise.resolve();
|
||||
},
|
||||
}),
|
||||
}),
|
||||
};
|
||||
return {
|
||||
db: { transaction: (fn: (tx: unknown) => unknown) => fn(tx) },
|
||||
};
|
||||
});
|
||||
|
||||
const { server, webServerSettings } = await import("@dokploy/server/db/schema");
|
||||
const { removeLogProvider } = await import(
|
||||
"@dokploy/server/services/log-management/service"
|
||||
);
|
||||
|
||||
const render = (value: unknown) => new PgDialect().sqlToQuery(value as SQL).sql;
|
||||
|
||||
describe("removeLogProvider", () => {
|
||||
beforeEach(() => {
|
||||
mocks.updates = [];
|
||||
});
|
||||
|
||||
it("drops the provider id from every server and from the local host", async () => {
|
||||
mocks.deleted = [{ logProviderId: "lp-1" }];
|
||||
|
||||
await removeLogProvider("lp-1");
|
||||
|
||||
expect(mocks.updates.map((u) => u.table)).toEqual([
|
||||
server,
|
||||
webServerSettings,
|
||||
]);
|
||||
expect(render(mocks.updates[0]?.values.logProviderIds)).toBe(
|
||||
'array_remove("server"."logProviderIds", $1)',
|
||||
);
|
||||
expect(render(mocks.updates[1]?.values.logProviderIds)).toBe(
|
||||
'array_remove("webServerSettings"."logProviderIds", $1)',
|
||||
);
|
||||
expect(render(mocks.updates[1]?.values.logManagementOrganizationId)).toBe(
|
||||
'CASE WHEN cardinality(array_remove("webServerSettings"."logProviderIds", $1)) = 0 THEN NULL ELSE "webServerSettings"."logManagementOrganizationId" END',
|
||||
);
|
||||
});
|
||||
|
||||
it("does not touch assignments when the provider does not exist", async () => {
|
||||
mocks.deleted = [];
|
||||
|
||||
await expect(removeLogProvider("missing")).rejects.toMatchObject({
|
||||
code: "NOT_FOUND",
|
||||
});
|
||||
expect(mocks.updates).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
claimWebServerLogManagement: vi.fn(),
|
||||
updateServerLogProviders: vi.fn(),
|
||||
removeVectorAgent: vi.fn(),
|
||||
setupVectorAgent: vi.fn(),
|
||||
assertServerBelongsToOrg: vi.fn(),
|
||||
assertLogProvidersBelongToOrg: vi.fn(),
|
||||
}));
|
||||
|
||||
const loadAgent = async (isCloud: boolean) => {
|
||||
vi.resetModules();
|
||||
vi.doMock("@dokploy/server/constants", () => ({ IS_CLOUD: isCloud }));
|
||||
vi.doMock("@dokploy/server/services/server", () => ({
|
||||
updateServerLogProviders: mocks.updateServerLogProviders,
|
||||
}));
|
||||
vi.doMock("@dokploy/server/services/web-server-settings", () => ({
|
||||
claimWebServerLogManagement: mocks.claimWebServerLogManagement,
|
||||
}));
|
||||
vi.doMock("@dokploy/server/setup/vector-setup", () => ({
|
||||
removeVectorAgent: mocks.removeVectorAgent,
|
||||
setupVectorAgent: mocks.setupVectorAgent,
|
||||
}));
|
||||
vi.doMock("@dokploy/server/services/log-management/service", () => ({
|
||||
assertServerBelongsToOrg: mocks.assertServerBelongsToOrg,
|
||||
assertLogProvidersBelongToOrg: mocks.assertLogProvidersBelongToOrg,
|
||||
}));
|
||||
return await import("@dokploy/server/services/log-management/vector-agent");
|
||||
};
|
||||
|
||||
describe("local Vector agent guards", () => {
|
||||
beforeEach(() => {
|
||||
for (const mock of Object.values(mocks)) mock.mockReset();
|
||||
});
|
||||
|
||||
it("does not remove the local agent when another organization owns it", async () => {
|
||||
mocks.claimWebServerLogManagement.mockResolvedValue(null);
|
||||
const { removeLogManagement } = await loadAgent(false);
|
||||
|
||||
await expect(removeLogManagement("org-b", undefined)).rejects.toMatchObject(
|
||||
{ code: "CONFLICT" },
|
||||
);
|
||||
expect(mocks.claimWebServerLogManagement).toHaveBeenCalledWith("org-b", []);
|
||||
expect(mocks.removeVectorAgent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("removes the local agent once the claim is released", async () => {
|
||||
mocks.claimWebServerLogManagement.mockResolvedValue({ id: "settings" });
|
||||
const { removeLogManagement } = await loadAgent(false);
|
||||
|
||||
await removeLogManagement("org-a", undefined);
|
||||
expect(mocks.removeVectorAgent).toHaveBeenCalledWith();
|
||||
});
|
||||
|
||||
it("rejects deploying the local agent in cloud", async () => {
|
||||
const { deployLogManagement } = await loadAgent(true);
|
||||
|
||||
await expect(
|
||||
deployLogManagement("org-a", undefined, ["lp-1"]),
|
||||
).rejects.toMatchObject({ code: "BAD_REQUEST" });
|
||||
expect(mocks.claimWebServerLogManagement).not.toHaveBeenCalled();
|
||||
expect(mocks.setupVectorAgent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects removing the local agent in cloud", async () => {
|
||||
const { removeLogManagement } = await loadAgent(true);
|
||||
|
||||
await expect(removeLogManagement("org-a", undefined)).rejects.toMatchObject(
|
||||
{ code: "BAD_REQUEST" },
|
||||
);
|
||||
expect(mocks.claimWebServerLogManagement).not.toHaveBeenCalled();
|
||||
expect(mocks.removeVectorAgent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still manages remote servers in cloud", async () => {
|
||||
const { removeLogManagement } = await loadAgent(true);
|
||||
|
||||
await removeLogManagement("org-a", "server-1");
|
||||
expect(mocks.assertServerBelongsToOrg).toHaveBeenCalledWith(
|
||||
"server-1",
|
||||
"org-a",
|
||||
);
|
||||
expect(mocks.removeVectorAgent).toHaveBeenCalledWith("server-1");
|
||||
expect(mocks.updateServerLogProviders).toHaveBeenCalledWith("server-1", []);
|
||||
});
|
||||
});
|
||||
@@ -112,7 +112,7 @@ describe("buildVectorConfigYaml", () => {
|
||||
expect(Object.keys(config.sinks)).toEqual(["sink_good-loki"]);
|
||||
});
|
||||
|
||||
it("drops containers Dokploy did not deploy on the local host, which also runs unrelated containers", async () => {
|
||||
it("only keeps containers of the owning organization on the local host, which is shared by every organization", async () => {
|
||||
const yamlStr = await buildVectorConfigYaml(
|
||||
[
|
||||
provider({
|
||||
@@ -122,21 +122,21 @@ describe("buildVectorConfigYaml", () => {
|
||||
endpoint: "https://loki.example.com",
|
||||
}),
|
||||
],
|
||||
{ dropUnmatched: true },
|
||||
{ organizationId: "org-a" },
|
||||
);
|
||||
|
||||
const config = parse(yamlStr) as any;
|
||||
expect(config.transforms.dokploy_scope_local_only).toEqual({
|
||||
type: "filter",
|
||||
inputs: ["dokploy_scope"],
|
||||
condition: '.dokploy_project != ""',
|
||||
condition: '.dokploy_organization == "org-a"',
|
||||
});
|
||||
expect(config.sinks["sink_loki-1"].inputs).toEqual([
|
||||
"dokploy_scope_local_only",
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not add the drop-unmatched filter for the per-server (non-local) path", async () => {
|
||||
it("does not add the organization filter for the per-server (non-local) path", async () => {
|
||||
const yamlStr = await buildVectorConfigYaml([]);
|
||||
const config = parse(yamlStr) as any;
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
findServerById: vi.fn(),
|
||||
getRemoteDocker: vi.fn(),
|
||||
execAsyncRemote: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@dokploy/server/db", () => ({ db: {} }));
|
||||
vi.mock("@dokploy/server/services/server", () => ({
|
||||
findServerById: mocks.findServerById,
|
||||
findServersByOrganizationForLogManagement: vi.fn(),
|
||||
}));
|
||||
vi.mock("@dokploy/server/utils/servers/remote-docker", () => ({
|
||||
getRemoteDocker: mocks.getRemoteDocker,
|
||||
}));
|
||||
vi.mock("@dokploy/server/utils/process/execAsync", () => ({
|
||||
execAsync: vi.fn(),
|
||||
execAsyncRemote: mocks.execAsyncRemote,
|
||||
}));
|
||||
|
||||
const { removeVectorAgent } = await import(
|
||||
"@dokploy/server/setup/vector-setup"
|
||||
);
|
||||
|
||||
describe("removeVectorAgent on a server without SSH key", () => {
|
||||
it("refuses instead of falling back to the local docker socket", async () => {
|
||||
mocks.findServerById.mockResolvedValue({
|
||||
serverId: "server-1",
|
||||
sshKeyId: null,
|
||||
});
|
||||
|
||||
await expect(removeVectorAgent("server-1")).rejects.toThrow(/SSH key/);
|
||||
expect(mocks.getRemoteDocker).not.toHaveBeenCalled();
|
||||
expect(mocks.execAsyncRemote).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,15 @@ interface Props {
|
||||
logProviderId?: string;
|
||||
}
|
||||
|
||||
export const agentsUsingProvider = (
|
||||
targets: Array<{ name: string; logProviderIds: string[] }> | undefined,
|
||||
logProviderId: string,
|
||||
) =>
|
||||
(targets ?? [])
|
||||
.filter((target) => target.logProviderIds.includes(logProviderId))
|
||||
.map((target) => target.name)
|
||||
.join(", ");
|
||||
|
||||
export const HandleLogProvider = ({ logProviderId }: Props) => {
|
||||
const utils = api.useUtils();
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
@@ -181,8 +190,19 @@ export const HandleLogProvider = ({ logProviderId }: Props) => {
|
||||
} as any)
|
||||
.then(() => {
|
||||
utils.logProvider.all.invalidate();
|
||||
const agents =
|
||||
logProviderId &&
|
||||
agentsUsingProvider(
|
||||
utils.logProvider.serverStatus.getData(),
|
||||
logProviderId,
|
||||
);
|
||||
toast.success(
|
||||
logProviderId ? "Log provider updated" : "Log provider added",
|
||||
agents
|
||||
? {
|
||||
description: `Redeploy the Vector agent on ${agents} to apply the changes.`,
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
setIsOpen(false);
|
||||
})
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { api } from "@/utils/api";
|
||||
import { HandleLogProvider } from "./handle-log-provider";
|
||||
import { agentsUsingProvider, HandleLogProvider } from "./handle-log-provider";
|
||||
|
||||
interface LogProviderRowProps {
|
||||
provider: {
|
||||
@@ -36,6 +36,7 @@ const LogProviderRow = ({
|
||||
canDelete,
|
||||
onDeleted,
|
||||
}: LogProviderRowProps) => {
|
||||
const utils = api.useUtils();
|
||||
const { mutateAsync, isPending: isRemoving } =
|
||||
api.logProvider.remove.useMutation();
|
||||
const ProviderIcon =
|
||||
@@ -65,14 +66,25 @@ const LogProviderRow = ({
|
||||
{canDelete && (
|
||||
<DialogAction
|
||||
title="Delete Log Provider"
|
||||
description="Are you sure you want to delete this log provider? Vector will stop shipping logs to it."
|
||||
description="Are you sure you want to delete this log provider? Agents already deployed keep shipping logs to it until you redeploy or remove them."
|
||||
type="destructive"
|
||||
onClick={async () => {
|
||||
const agents = agentsUsingProvider(
|
||||
utils.logProvider.serverStatus.getData(),
|
||||
provider.logProviderId,
|
||||
);
|
||||
await mutateAsync({
|
||||
logProviderId: provider.logProviderId,
|
||||
})
|
||||
.then(() => {
|
||||
toast.success("Log provider deleted successfully");
|
||||
toast.success(
|
||||
"Log provider deleted successfully",
|
||||
agents
|
||||
? {
|
||||
description: `Redeploy or remove the Vector agent on ${agents} to stop shipping logs to it.`,
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
onDeleted();
|
||||
})
|
||||
.catch(() => {
|
||||
@@ -151,7 +163,7 @@ export const ShowLogProviders = () => {
|
||||
canDelete={!!permissions?.logProvider.delete}
|
||||
onDeleted={() => {
|
||||
refetch();
|
||||
utils.server.all?.invalidate?.();
|
||||
utils.logProvider.serverStatus.invalidate();
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
|
||||
+2
-2
@@ -12,8 +12,8 @@ CREATE TABLE "logProvider" (
|
||||
"organizationId" text NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "server" ADD COLUMN "enableLogManagement" boolean DEFAULT false NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "webServerSettings" ADD COLUMN "enableLogManagement" boolean DEFAULT false NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "server" ADD COLUMN "logProviderIds" text[] DEFAULT ARRAY[]::text[] NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "webServerSettings" ADD COLUMN "logProviderIds" text[] DEFAULT ARRAY[]::text[] NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "webServerSettings" ADD COLUMN "logManagementOrganizationId" text;--> statement-breakpoint
|
||||
ALTER TABLE "logProvider" ADD CONSTRAINT "logProvider_organizationId_organization_id_fk" FOREIGN KEY ("organizationId") REFERENCES "public"."organization"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "webServerSettings" ADD CONSTRAINT "webServerSettings_logManagementOrganizationId_organization_id_fk" FOREIGN KEY ("logManagementOrganizationId") REFERENCES "public"."organization"("id") ON DELETE set null ON UPDATE no action;
|
||||
@@ -1,4 +0,0 @@
|
||||
ALTER TABLE "server" ADD COLUMN "logProviderIds" text[] DEFAULT ARRAY[]::text[] NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "webServerSettings" ADD COLUMN "logProviderIds" text[] DEFAULT ARRAY[]::text[] NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "server" DROP COLUMN "enableLogManagement";--> statement-breakpoint
|
||||
ALTER TABLE "webServerSettings" DROP COLUMN "enableLogManagement";
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"id": "04d812fc-56cf-48bf-92a4-8afdf426b380",
|
||||
"id": "bcb4d155-4bad-4373-bbfa-73705d3a9714",
|
||||
"prevId": "c9ad226a-7a8f-4c3b-8f04-fef6bccf9bfb",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
@@ -7765,12 +7765,12 @@
|
||||
"notNull": true,
|
||||
"default": false
|
||||
},
|
||||
"enableLogManagement": {
|
||||
"name": "enableLogManagement",
|
||||
"type": "boolean",
|
||||
"logProviderIds": {
|
||||
"name": "logProviderIds",
|
||||
"type": "text[]",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": false
|
||||
"default": "ARRAY[]::text[]"
|
||||
},
|
||||
"buildsConcurrency": {
|
||||
"name": "buildsConcurrency",
|
||||
@@ -8901,12 +8901,12 @@
|
||||
"notNull": false,
|
||||
"default": "'0 0 * * *'"
|
||||
},
|
||||
"enableLogManagement": {
|
||||
"name": "enableLogManagement",
|
||||
"type": "boolean",
|
||||
"logProviderIds": {
|
||||
"name": "logProviderIds",
|
||||
"type": "text[]",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": false
|
||||
"default": "ARRAY[]::text[]"
|
||||
},
|
||||
"logManagementOrganizationId": {
|
||||
"name": "logManagementOrganizationId",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1384,15 +1384,8 @@
|
||||
{
|
||||
"idx": 197,
|
||||
"version": "7",
|
||||
"when": 1789042957674,
|
||||
"tag": "0197_past_omega_red",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 198,
|
||||
"version": "7",
|
||||
"when": 1790203407320,
|
||||
"tag": "0198_kind_hemingway",
|
||||
"when": 1791377838843,
|
||||
"tag": "0197_smiling_madrox",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "dokploy",
|
||||
"version": "v0.30.7",
|
||||
"version": "v0.30.6",
|
||||
"private": true,
|
||||
"license": "Apache-2.0",
|
||||
"type": "module",
|
||||
|
||||
@@ -1,21 +1,16 @@
|
||||
import {
|
||||
assertLogProvidersBelongToOrg,
|
||||
assertServerBelongsToOrg,
|
||||
claimWebServerLogManagement,
|
||||
createLogProvider,
|
||||
findLogProviderById,
|
||||
deployLogManagement,
|
||||
findLogProviderForOrganization,
|
||||
findLogProvidersByOrganization,
|
||||
getLogManagementServerStatus,
|
||||
logProviderAdapters,
|
||||
removeLogManagement,
|
||||
removeLogProvider,
|
||||
removeVectorAgent,
|
||||
sanitizeLogProvider,
|
||||
setupVectorAgent,
|
||||
testLogProviderConnection,
|
||||
updateLogProvider,
|
||||
updateServerLogProviders,
|
||||
} from "@dokploy/server";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { z } from "zod";
|
||||
import { audit } from "@/server/api/utils/audit";
|
||||
import {
|
||||
@@ -47,13 +42,10 @@ export const logProviderRouter = createTRPCRouter({
|
||||
.input(apiUpdateLogProvider)
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const { logProviderId, ...rest } = input;
|
||||
const provider = await findLogProviderById(logProviderId);
|
||||
if (provider.organizationId !== ctx.session.activeOrganizationId) {
|
||||
throw new TRPCError({
|
||||
code: "UNAUTHORIZED",
|
||||
message: "You are not allowed to update this log provider",
|
||||
});
|
||||
}
|
||||
const provider = await findLogProviderForOrganization(
|
||||
logProviderId,
|
||||
ctx.session.activeOrganizationId,
|
||||
);
|
||||
const updated = await updateLogProvider(logProviderId, rest);
|
||||
await audit(ctx, {
|
||||
action: "update",
|
||||
@@ -66,13 +58,10 @@ export const logProviderRouter = createTRPCRouter({
|
||||
remove: withPermission("logProvider", "delete")
|
||||
.input(apiRemoveLogProvider)
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const provider = await findLogProviderById(input.logProviderId);
|
||||
if (provider.organizationId !== ctx.session.activeOrganizationId) {
|
||||
throw new TRPCError({
|
||||
code: "UNAUTHORIZED",
|
||||
message: "You are not allowed to delete this log provider",
|
||||
});
|
||||
}
|
||||
const provider = await findLogProviderForOrganization(
|
||||
input.logProviderId,
|
||||
ctx.session.activeOrganizationId,
|
||||
);
|
||||
const removed = await removeLogProvider(input.logProviderId);
|
||||
await audit(ctx, {
|
||||
action: "delete",
|
||||
@@ -90,14 +79,10 @@ export const logProviderRouter = createTRPCRouter({
|
||||
one: withPermission("logProvider", "read")
|
||||
.input(apiFindOneLogProvider)
|
||||
.query(async ({ ctx, input }) => {
|
||||
const provider = await findLogProviderById(input.logProviderId);
|
||||
if (provider.organizationId !== ctx.session.activeOrganizationId) {
|
||||
throw new TRPCError({
|
||||
code: "UNAUTHORIZED",
|
||||
message: "You are not allowed to access this log provider",
|
||||
});
|
||||
}
|
||||
return provider;
|
||||
return await findLogProviderForOrganization(
|
||||
input.logProviderId,
|
||||
ctx.session.activeOrganizationId,
|
||||
);
|
||||
}),
|
||||
testConnection: withPermission("logProvider", "create")
|
||||
.input(apiTestLogProvider)
|
||||
@@ -117,13 +102,10 @@ export const logProviderRouter = createTRPCRouter({
|
||||
testConnectionById: withPermission("logProvider", "create")
|
||||
.input(apiFindOneLogProvider)
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const provider = await findLogProviderById(input.logProviderId);
|
||||
if (provider.organizationId !== ctx.session.activeOrganizationId) {
|
||||
throw new TRPCError({
|
||||
code: "UNAUTHORIZED",
|
||||
message: "You are not allowed to access this log provider",
|
||||
});
|
||||
}
|
||||
await findLogProviderForOrganization(
|
||||
input.logProviderId,
|
||||
ctx.session.activeOrganizationId,
|
||||
);
|
||||
return await testLogProviderConnection({
|
||||
logProviderId: input.logProviderId,
|
||||
});
|
||||
@@ -139,28 +121,11 @@ export const logProviderRouter = createTRPCRouter({
|
||||
}),
|
||||
)
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const organizationId = ctx.session.activeOrganizationId;
|
||||
const serverId = input.serverId ?? undefined;
|
||||
await assertServerBelongsToOrg(serverId, organizationId);
|
||||
await assertLogProvidersBelongToOrg(input.logProviderIds, organizationId);
|
||||
|
||||
if (serverId) {
|
||||
await updateServerLogProviders(serverId, input.logProviderIds);
|
||||
} else {
|
||||
const claimed = await claimWebServerLogManagement(
|
||||
organizationId,
|
||||
input.logProviderIds,
|
||||
);
|
||||
if (!claimed) {
|
||||
throw new TRPCError({
|
||||
code: "CONFLICT",
|
||||
message:
|
||||
"The local Vector agent is already claimed by another organization",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await setupVectorAgent(organizationId, serverId, input.logProviderIds);
|
||||
await deployLogManagement(
|
||||
ctx.session.activeOrganizationId,
|
||||
input.serverId ?? undefined,
|
||||
input.logProviderIds,
|
||||
);
|
||||
await audit(ctx, {
|
||||
action: "create",
|
||||
resourceType: "server",
|
||||
@@ -172,16 +137,10 @@ export const logProviderRouter = createTRPCRouter({
|
||||
removeOnServer: withPermission("logProvider", "create")
|
||||
.input(z.object({ serverId: z.string().nullable().optional() }))
|
||||
.mutation(async ({ ctx, input }) => {
|
||||
const organizationId = ctx.session.activeOrganizationId;
|
||||
const serverId = input.serverId ?? undefined;
|
||||
await assertServerBelongsToOrg(serverId, organizationId);
|
||||
|
||||
await removeVectorAgent(serverId);
|
||||
if (serverId) {
|
||||
await updateServerLogProviders(serverId, []);
|
||||
} else {
|
||||
await claimWebServerLogManagement(organizationId, []);
|
||||
}
|
||||
await removeLogManagement(
|
||||
ctx.session.activeOrganizationId,
|
||||
input.serverId ?? undefined,
|
||||
);
|
||||
await audit(ctx, {
|
||||
action: "delete",
|
||||
resourceType: "server",
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
IS_CLOUD,
|
||||
redactServerSshKey,
|
||||
removeDeploymentsByServerId,
|
||||
removeVectorAgent,
|
||||
removeServerLogManagement,
|
||||
serverAudit,
|
||||
serverSetup,
|
||||
serverValidate,
|
||||
@@ -474,17 +474,8 @@ export const serverRouter = createTRPCRouter({
|
||||
message: "Server has active services, please delete them first",
|
||||
});
|
||||
}
|
||||
let vectorAgentRemovalWarning: string | undefined;
|
||||
if ((currentServer.logProviderIds?.length ?? 0) > 0) {
|
||||
await removeVectorAgent(input.serverId).catch((error) => {
|
||||
vectorAgentRemovalWarning =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
console.error(
|
||||
`[Vector] Failed to remove agent for server ${input.serverId} before deletion:`,
|
||||
error,
|
||||
);
|
||||
});
|
||||
}
|
||||
const vectorAgentRemovalWarning =
|
||||
await removeServerLogManagement(currentServer);
|
||||
await audit(ctx, {
|
||||
action: "delete",
|
||||
resourceType: "server",
|
||||
|
||||
@@ -66,7 +66,6 @@ void app.prepare().then(async () => {
|
||||
await initCancelDeployments();
|
||||
await initVolumeBackupsCronJobs();
|
||||
await sendDokployRestartNotifications();
|
||||
} else if (IS_CLOUD) {
|
||||
}
|
||||
await initEnterpriseBackupCronJobs();
|
||||
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
# Log Management
|
||||
|
||||
> **Nota para quien mergee esto:** este contenido vive acá porque no había un checkout local del
|
||||
> repo de docs públicas (`docs.dokploy.com`) disponible al escribir esta feature. Migrar este
|
||||
> archivo a ese repo (sección "Core" o similar, junto a Registry/Monitoring) y borrarlo de acá.
|
||||
|
||||
## Qué es
|
||||
|
||||
Dokploy puede shippear los logs de los containers de un server a un proveedor externo de log
|
||||
management — hoy: **Grafana Loki**, **Datadog Logs**, **Better Stack** (ex-Logtail),
|
||||
**Elasticsearch / OpenSearch**, **Splunk (HTTP Event Collector)** y **AWS CloudWatch Logs**. Antes de
|
||||
esta feature, la única forma de ver logs era el stream en vivo por WebSocket en el dashboard, sin
|
||||
historial ni persistencia. Esta feature no reemplaza ese viewer en vivo — sigue funcionando igual
|
||||
— agrega la posibilidad de mandar una copia de los logs a un sistema externo para
|
||||
retención/búsqueda/alerting.
|
||||
|
||||
Por debajo corre [Vector](https://vector.dev) (Apache-2.0), un único binario, como un agente por
|
||||
server registrado. Dokploy no implementa clientes HTTP para cada proveedor — solo traduce
|
||||
credenciales guardadas en el dashboard a un bloque de configuración de Vector (un "sink").
|
||||
|
||||
## Cómo activarla
|
||||
|
||||
Hacen falta **dos cosas**, ambas apagadas por default:
|
||||
|
||||
1. **Al menos un log provider configurado**, en Settings → Log Management. Elegís el tipo
|
||||
(Loki/Datadog/Better Stack/Elasticsearch/OpenSearch/Splunk/AWS CloudWatch), completás las
|
||||
credenciales que pida ese tipo, y podés probar la
|
||||
conexión antes de guardar.
|
||||
2. **El toggle "Log Management" prendido** en cada lugar donde quieras shippear logs:
|
||||
- Por server registrado: en el diálogo de acciones del server ("Server Actions"), al lado del
|
||||
toggle de Docker Cleanup.
|
||||
- **En la máquina donde corre Dokploy** (self-hosted únicamente — no existe en Dokploy Cloud,
|
||||
donde esa máquina es infraestructura de Dokploy, no del cliente): en Settings → Web Server,
|
||||
al lado del toggle de Docker Cleanup local. La primera organización que lo prende "reclama"
|
||||
ese host — otra organización de la misma instancia no puede prenderlo hasta que la primera lo
|
||||
apague (rechazo explícito, no se pisan silenciosamente).
|
||||
|
||||
Si cualquiera de las dos condiciones no se cumple, el agente Vector **no corre en absoluto** en
|
||||
ese host — no consume CPU/memoria/disco, y nada más del dashboard se ve afectado (el streaming en
|
||||
vivo de logs sigue funcionando igual).
|
||||
|
||||
**Host local en una instancia self-hosted multi-organización:** Vector lee el socket de Docker
|
||||
completo, sin distinguir de qué organización es cada container — pero solo el agente local (el
|
||||
registrado por `server` siempre pertenece a una sola organización, sin este problema). Para
|
||||
evitar que le lleguen logs de otra organización a los providers del dueño del host, el pipeline
|
||||
del agente local descarta todo evento que no matchee una app de la organización dueña (filtro
|
||||
`dokploy_scope_local_only` en `vector-setup.ts`) antes de que llegue a cualquier sink — esto
|
||||
también implica que, a diferencia del agente por-server, containers de terceros (no gestionados
|
||||
por Dokploy) corriendo en la misma máquina ya no shippean via el agente local.
|
||||
|
||||
Un log provider es a nivel de organización: si tenés varios servers con el toggle prendido, todos
|
||||
ellos shippean a todos los providers habilitados. No hay (todavía) una asociación
|
||||
provider-por-server — es una mejora futura posible si hace falta más granularidad.
|
||||
|
||||
### Scoping por proyecto/entorno/aplicación
|
||||
|
||||
Aunque Vector ve **todos** los containers del host (no solo los de Dokploy), cada evento que sí
|
||||
corresponde a una app gestionada por Dokploy llega taggeado con:
|
||||
|
||||
- `dokploy_organization` — siempre presente.
|
||||
- `dokploy_project` / `dokploy_project_id`
|
||||
- `dokploy_environment` / `dokploy_environment_id`
|
||||
- `dokploy_application` / `dokploy_application_id`
|
||||
|
||||
Un container de terceros (no gestionado por Dokploy) corriendo en el mismo host sigue
|
||||
shippeando igual, simplemente sin esos 6 campos.
|
||||
|
||||
Esta tabla de scoping se recalcula cada vez que se toca algo de log management (activar/crear/
|
||||
editar un provider, prender el toggle de un server) y además cada 15 minutos como red de
|
||||
seguridad, para que una app creada después de la configuración inicial no quede sin estos tags
|
||||
indefinidamente.
|
||||
|
||||
## Cómo agregar un provider nuevo (guía para contribuidores)
|
||||
|
||||
Toda la lógica de un provider vive en un solo archivo:
|
||||
`packages/server/src/services/log-management/providers/<nombre>.ts`, implementando la interfaz
|
||||
`LogProviderAdapter` (`packages/server/src/services/log-management/types.ts`). **No hace falta
|
||||
tocar el router tRPC ni la UI** — la UI (`logProvider.availableTypes`) lista los providers
|
||||
disponibles dinámicamente a partir del registry de adapters, y el orquestador de Vector
|
||||
(`packages/server/src/setup/vector-setup.ts`) resuelve el resto de forma genérica.
|
||||
|
||||
Hay dos caminos, según si Vector ya tiene un sink nativo para tu backend:
|
||||
|
||||
### Camino 1: tu backend ya tiene un sink en Vector
|
||||
|
||||
Es el caso de Loki, Datadog, Elasticsearch/OpenSearch, Splunk HEC y AWS CloudWatch Logs. Mirá la
|
||||
[lista de sinks de Vector](https://vector.dev/docs/reference/configuration/sinks/)
|
||||
— si tu backend está ahí, este es tu camino.
|
||||
|
||||
1. Agregar el tipo al enum de Drizzle: `logProviderType` en
|
||||
`packages/server/src/db/schema/log-provider.ts`, + correr
|
||||
`pnpm --filter dokploy migration:generate` y commitear el SQL generado.
|
||||
2. Ampliar el union `LogProviderType` en `services/log-management/types.ts`.
|
||||
3. Crear `providers/<nombre>.ts`:
|
||||
- `credentialFields`: qué campos pedirle al usuario. `key` tiene que ser exactamente
|
||||
`"endpoint"` | `"apiKey"` | `"apiSecret"` (las 3 únicas columnas con espacio propio en la
|
||||
tabla) o, si tu backend necesita algo que no encaja en esas 3, cualquier otro nombre — ese
|
||||
valor termina en `extraConfig` (jsonb). **Este `key` no es solo para mostrar en la UI: la UI
|
||||
genérica lo usa para decidir dónde guardar el valor** (columna directa vs. `extraConfig`) —
|
||||
ver `providers/loki.ts` para el caso simple (`endpoint`) y `providers/betterstack.ts` para
|
||||
el comentario completo sobre este punto (ahí hubo un bug real por esto durante el desarrollo:
|
||||
usar un `key` "bonito" que no coincidía con la columna real).
|
||||
- `toVectorSink(config, sinkId, inputId)`: devolvé el bloque de sink de Vector con tus
|
||||
credenciales mapeadas. **Siempre incluir `buffer: { type: "disk", max_size, when_full }`**
|
||||
(ver `DEFAULT_DISK_BUFFER` en `types.ts` para el default) — el default de Vector es buffer
|
||||
en memoria, que no protege contra un backend caído.
|
||||
- `testConnection(config)` (opcional pero recomendado): un ping barato para validar
|
||||
credenciales antes de guardar.
|
||||
4. Registrar en `providers/registry.ts` (`logProviderAdapters`) — una línea.
|
||||
5. Tests unitarios junto a los de `loki`/`datadog`/`betterstack`
|
||||
(`apps/dokploy/__test__/log-management/log-provider-adapters.test.ts`) verificando la forma
|
||||
del sink generado.
|
||||
|
||||
### Camino 2: tu backend NO tiene sink nativo en Vector
|
||||
|
||||
Es el caso de Better Stack — usar `providers/betterstack.ts` como plantilla. Mismos pasos que
|
||||
arriba, con dos diferencias:
|
||||
|
||||
- El sink usa el genérico `http` (o `socket`/`file` si tu backend expone algo distinto a HTTP).
|
||||
- Si tu backend espera el evento en una forma distinta a la que produce el scoping (ej. un
|
||||
nombre de campo distinto para el timestamp), implementar `toVectorTransform(config,
|
||||
transformId, scopeTransformId)` — un paso VRL intermedio entre el scoping y tu sink. El
|
||||
orquestador (`vector-setup.ts`) ya sabe encadenar `scope → tu transform → tu sink` cuando este
|
||||
método existe, y `scope → sink` directo cuando no — no hay que tocar el orquestador.
|
||||
|
||||
## Decisiones y límites conocidos
|
||||
|
||||
- **Un server = un nodo Swarm de un solo nodo, típicamente.** Dokploy no modela clusters Swarm
|
||||
multi-nodo (cada `server` registrado es una máquina SSH independiente). Si tu server participa
|
||||
de un swarm con workers que no están registrados como su propio `server` en Dokploy, el agente
|
||||
Vector puede terminar programado en un nodo sin la config — fuera de alcance del MVP actual.
|
||||
- **No hay viewer de logs históricos embebido en el dashboard** — esta feature es solo shipping.
|
||||
Para ver los logs, usar la UI del provider externo (Grafana, Datadog, Better Stack, Kibana/
|
||||
OpenSearch Dashboards, Splunk, CloudWatch Logs Insights).
|
||||
- **`logProvider` es free tier**, no enterprise-only — a diferencia de `registry`/`server`/
|
||||
`domain`/`monitoring`, disponible para cualquier organización sin importar el plan.
|
||||
- **Deployment en la misma máquina de Dokploy**: soportado self-hosted (Settings → Web Server),
|
||||
nunca en Dokploy Cloud — ver "Cómo activarla" arriba para el caveat multi-organización.
|
||||
@@ -31,6 +31,7 @@ export * from "./services/libsql";
|
||||
export * from "./services/log-management/providers/registry";
|
||||
export * from "./services/log-management/service";
|
||||
export * from "./services/log-management/types";
|
||||
export * from "./services/log-management/vector-agent";
|
||||
export * from "./services/mariadb";
|
||||
export * from "./services/mongo";
|
||||
export * from "./services/mount";
|
||||
|
||||
@@ -2,10 +2,12 @@ import { db } from "@dokploy/server/db";
|
||||
import {
|
||||
type apiCreateLogProvider,
|
||||
logProvider,
|
||||
server,
|
||||
webServerSettings,
|
||||
} from "@dokploy/server/db/schema";
|
||||
import { findServerById } from "@dokploy/server/services/server";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { and, arrayContains, eq, sql } from "drizzle-orm";
|
||||
import type { z } from "zod";
|
||||
import { getLogProviderAdapter } from "./providers/registry";
|
||||
import type { LogProviderRuntimeConfig, LogProviderType } from "./types";
|
||||
@@ -135,19 +137,52 @@ export const updateLogProvider = async (
|
||||
};
|
||||
|
||||
export const removeLogProvider = async (logProviderId: string) => {
|
||||
const removed = await db
|
||||
.delete(logProvider)
|
||||
.where(eq(logProvider.logProviderId, logProviderId))
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
return await db.transaction(async (tx) => {
|
||||
const removed = await tx
|
||||
.delete(logProvider)
|
||||
.where(eq(logProvider.logProviderId, logProviderId))
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
if (!removed) {
|
||||
if (!removed) {
|
||||
throw new TRPCError({
|
||||
code: "NOT_FOUND",
|
||||
message: "Log provider not found",
|
||||
});
|
||||
}
|
||||
|
||||
await tx
|
||||
.update(server)
|
||||
.set({
|
||||
logProviderIds: sql`array_remove(${server.logProviderIds}, ${logProviderId})`,
|
||||
})
|
||||
.where(arrayContains(server.logProviderIds, [logProviderId]));
|
||||
|
||||
const remainingLocalIds = sql`array_remove(${webServerSettings.logProviderIds}, ${logProviderId})`;
|
||||
await tx
|
||||
.update(webServerSettings)
|
||||
.set({
|
||||
logProviderIds: remainingLocalIds,
|
||||
logManagementOrganizationId: sql`CASE WHEN cardinality(${remainingLocalIds}) = 0 THEN NULL ELSE ${webServerSettings.logManagementOrganizationId} END`,
|
||||
})
|
||||
.where(arrayContains(webServerSettings.logProviderIds, [logProviderId]));
|
||||
|
||||
return removed;
|
||||
});
|
||||
};
|
||||
|
||||
export const findLogProviderForOrganization = async (
|
||||
logProviderId: string,
|
||||
organizationId: string,
|
||||
) => {
|
||||
const provider = await findLogProviderById(logProviderId);
|
||||
if (provider.organizationId !== organizationId) {
|
||||
throw new TRPCError({
|
||||
code: "NOT_FOUND",
|
||||
message: "Log provider not found",
|
||||
code: "UNAUTHORIZED",
|
||||
message: "You are not allowed to access this log provider",
|
||||
});
|
||||
}
|
||||
return removed;
|
||||
return provider;
|
||||
};
|
||||
|
||||
export const findLogProviderById = async (logProviderId: string) => {
|
||||
|
||||
@@ -1,11 +1,35 @@
|
||||
import { BlockList, isIP } from "node:net";
|
||||
import { IS_CLOUD } from "@dokploy/server/constants";
|
||||
|
||||
export const LOG_PROVIDER_REQUEST_TIMEOUT_MS = 15_000;
|
||||
|
||||
const isMetadataAddress = (address: string): boolean => {
|
||||
const lower = address.toLowerCase().replace(/^\[|\]$/g, "");
|
||||
if (lower === "fd00:ec2::254") return true;
|
||||
if (lower.startsWith("169.254.")) return true;
|
||||
if (/^fe[89ab]/.test(lower)) return true;
|
||||
return false;
|
||||
const metadataAddresses = new BlockList();
|
||||
metadataAddresses.addSubnet("169.254.0.0", 16, "ipv4");
|
||||
metadataAddresses.addSubnet("fe80::", 10, "ipv6");
|
||||
metadataAddresses.addAddress("fd00:ec2::254", "ipv6");
|
||||
|
||||
const privateAddresses = new BlockList();
|
||||
privateAddresses.addSubnet("0.0.0.0", 8, "ipv4");
|
||||
privateAddresses.addSubnet("127.0.0.0", 8, "ipv4");
|
||||
privateAddresses.addSubnet("10.0.0.0", 8, "ipv4");
|
||||
privateAddresses.addSubnet("100.64.0.0", 10, "ipv4");
|
||||
privateAddresses.addSubnet("172.16.0.0", 12, "ipv4");
|
||||
privateAddresses.addSubnet("192.168.0.0", 16, "ipv4");
|
||||
privateAddresses.addAddress("::", "ipv6");
|
||||
privateAddresses.addAddress("::1", "ipv6");
|
||||
privateAddresses.addSubnet("fc00::", 7, "ipv6");
|
||||
|
||||
const isBlockedAddress = (address: string): boolean => {
|
||||
const ip = address.replace(/^\[|\]$/g, "");
|
||||
const family = isIP(ip);
|
||||
if (family === 0) {
|
||||
return IS_CLOUD && ip.toLowerCase() === "localhost";
|
||||
}
|
||||
const type = family === 4 ? "ipv4" : "ipv6";
|
||||
return (
|
||||
metadataAddresses.check(ip, type) ||
|
||||
(IS_CLOUD && privateAddresses.check(ip, type))
|
||||
);
|
||||
};
|
||||
|
||||
const METADATA_HOSTNAMES = new Set([
|
||||
@@ -13,7 +37,7 @@ const METADATA_HOSTNAMES = new Set([
|
||||
"metadata.goog",
|
||||
]);
|
||||
|
||||
const assertNotMetadataEndpoint = async (rawUrl: string): Promise<void> => {
|
||||
const assertNotBlockedEndpoint = async (rawUrl: string): Promise<void> => {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(rawUrl);
|
||||
@@ -22,31 +46,31 @@ const assertNotMetadataEndpoint = async (rawUrl: string): Promise<void> => {
|
||||
}
|
||||
const blocked = () =>
|
||||
new Error(
|
||||
"This endpoint resolves to a cloud metadata address and can't be used here.",
|
||||
"This endpoint resolves to a blocked address (cloud metadata or private network) and can't be used here.",
|
||||
);
|
||||
if (METADATA_HOSTNAMES.has(url.hostname.toLowerCase())) {
|
||||
throw blocked();
|
||||
}
|
||||
if (isMetadataAddress(url.hostname)) {
|
||||
if (isBlockedAddress(url.hostname)) {
|
||||
throw blocked();
|
||||
}
|
||||
let results: Array<{ address: string }>;
|
||||
try {
|
||||
const { lookup } = await import("node:dns/promises");
|
||||
const results = await lookup(url.hostname, { all: true });
|
||||
if (results.some((r) => isMetadataAddress(r.address))) {
|
||||
throw blocked();
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.includes("metadata")) {
|
||||
throw error;
|
||||
}
|
||||
results = await lookup(url.hostname, { all: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (results.some((r) => isBlockedAddress(r.address))) {
|
||||
throw blocked();
|
||||
}
|
||||
};
|
||||
|
||||
export const logProviderFetch = async (url: string, init: RequestInit = {}) => {
|
||||
await assertNotMetadataEndpoint(url);
|
||||
await assertNotBlockedEndpoint(url);
|
||||
return await fetch(url, {
|
||||
...init,
|
||||
redirect: "error",
|
||||
signal: AbortSignal.timeout(LOG_PROVIDER_REQUEST_TIMEOUT_MS),
|
||||
});
|
||||
};
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { IS_CLOUD } from "@dokploy/server/constants";
|
||||
import {
|
||||
type Server,
|
||||
updateServerLogProviders,
|
||||
} from "@dokploy/server/services/server";
|
||||
import { claimWebServerLogManagement } from "@dokploy/server/services/web-server-settings";
|
||||
import {
|
||||
removeVectorAgent,
|
||||
setupVectorAgent,
|
||||
} from "@dokploy/server/setup/vector-setup";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import {
|
||||
assertLogProvidersBelongToOrg,
|
||||
assertServerBelongsToOrg,
|
||||
} from "./service";
|
||||
|
||||
const assertLocalAgentAllowed = (serverId: string | undefined) => {
|
||||
if (!serverId && IS_CLOUD) {
|
||||
throw new TRPCError({
|
||||
code: "BAD_REQUEST",
|
||||
message:
|
||||
"Log management on the Dokploy host is not available in Dokploy Cloud",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const deployLogManagement = async (
|
||||
organizationId: string,
|
||||
serverId: string | undefined,
|
||||
logProviderIds: string[],
|
||||
) => {
|
||||
assertLocalAgentAllowed(serverId);
|
||||
await assertServerBelongsToOrg(serverId, organizationId);
|
||||
await assertLogProvidersBelongToOrg(logProviderIds, organizationId);
|
||||
|
||||
if (serverId) {
|
||||
await updateServerLogProviders(serverId, logProviderIds);
|
||||
} else {
|
||||
const claimed = await claimWebServerLogManagement(
|
||||
organizationId,
|
||||
logProviderIds,
|
||||
);
|
||||
if (!claimed) {
|
||||
throw new TRPCError({
|
||||
code: "CONFLICT",
|
||||
message:
|
||||
"The local Vector agent is already claimed by another organization",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await setupVectorAgent(organizationId, serverId, logProviderIds);
|
||||
};
|
||||
|
||||
export const removeLogManagement = async (
|
||||
organizationId: string,
|
||||
serverId: string | undefined,
|
||||
) => {
|
||||
assertLocalAgentAllowed(serverId);
|
||||
|
||||
if (serverId) {
|
||||
await assertServerBelongsToOrg(serverId, organizationId);
|
||||
await removeVectorAgent(serverId);
|
||||
await updateServerLogProviders(serverId, []);
|
||||
return;
|
||||
}
|
||||
|
||||
const released = await claimWebServerLogManagement(organizationId, []);
|
||||
if (!released) {
|
||||
throw new TRPCError({
|
||||
code: "CONFLICT",
|
||||
message: "The local Vector agent is claimed by another organization",
|
||||
});
|
||||
}
|
||||
await removeVectorAgent();
|
||||
};
|
||||
|
||||
export const removeServerLogManagement = async (server: Server) => {
|
||||
if (!server.logProviderIds?.length) return;
|
||||
try {
|
||||
await removeVectorAgent(server.serverId);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[Vector] Failed to remove agent for server ${server.serverId} before deletion:`,
|
||||
error,
|
||||
);
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
};
|
||||
@@ -1,6 +1,11 @@
|
||||
import { IS_CLOUD } from "@dokploy/server/constants";
|
||||
import { db } from "@dokploy/server/db";
|
||||
import { organization } from "@dokploy/server/db/schema";
|
||||
import { findServersWithLogManagementEnabled } from "@dokploy/server/services/server";
|
||||
import {
|
||||
claimWebServerLogManagement,
|
||||
getWebServerSettings,
|
||||
} from "@dokploy/server/services/web-server-settings";
|
||||
import { removeVectorAgent } from "@dokploy/server/setup/vector-setup";
|
||||
import { eq } from "drizzle-orm";
|
||||
|
||||
@@ -16,6 +21,17 @@ const removeVectorAgents = async (organizationId: string) => {
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
if (IS_CLOUD) return;
|
||||
const settings = await getWebServerSettings();
|
||||
if (settings?.logManagementOrganizationId !== organizationId) return;
|
||||
await removeVectorAgent().catch((error) => {
|
||||
console.error(
|
||||
`[Vector] Failed to remove the local agent before deleting organization ${organizationId}:`,
|
||||
error,
|
||||
);
|
||||
});
|
||||
await claimWebServerLogManagement(organizationId, []);
|
||||
};
|
||||
|
||||
export const deleteOrganization = async (organizationId: string) => {
|
||||
|
||||
@@ -104,9 +104,10 @@ const loadEnabledLogProviders = async (
|
||||
|
||||
export const buildVectorConfigYaml = async (
|
||||
providers: Array<LogProvider>,
|
||||
options?: { dropUnmatched?: boolean },
|
||||
options?: { organizationId?: string },
|
||||
) => {
|
||||
const baseTransformId = options?.dropUnmatched
|
||||
const organizationId = options?.organizationId;
|
||||
const baseTransformId = organizationId
|
||||
? "dokploy_scope_local_only"
|
||||
: "dokploy_scope";
|
||||
const { sinks, transforms } = buildSinksAndTransforms(
|
||||
@@ -128,12 +129,12 @@ export const buildVectorConfigYaml = async (
|
||||
inputs: ["docker_logs_source"],
|
||||
source: buildScopeTransformSource(),
|
||||
},
|
||||
...(options?.dropUnmatched
|
||||
...(organizationId
|
||||
? {
|
||||
dokploy_scope_local_only: {
|
||||
type: "filter",
|
||||
inputs: ["dokploy_scope"],
|
||||
condition: '.dokploy_project != ""',
|
||||
condition: `.dokploy_organization == ${JSON.stringify(organizationId)}`,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
@@ -190,7 +191,7 @@ const syncVectorConfig = async (
|
||||
const ids = logProviderIds ?? (await getServerLogProviderIds(serverId));
|
||||
const providers = await loadEnabledLogProviders(organizationId, ids);
|
||||
const yamlStr = await buildVectorConfigYaml(providers, {
|
||||
dropUnmatched: !serverId,
|
||||
organizationId: serverId ? undefined : organizationId,
|
||||
});
|
||||
|
||||
const writeCommand = `umask 077 && mkdir -p ${quote([`${VECTOR_PATH}/data`])} && chmod 700 ${quote([VECTOR_PATH])} ${quote([`${VECTOR_PATH}/data`])} && echo "${encodeBase64(yamlStr)}" | base64 -d > ${quote([candidatePath])}`;
|
||||
@@ -278,6 +279,9 @@ const deployVectorService = async (serverId?: string) => {
|
||||
],
|
||||
},
|
||||
Networks: [{ Target: "host" }],
|
||||
Placement: {
|
||||
Constraints: ["node.role==manager"],
|
||||
},
|
||||
},
|
||||
Mode: {
|
||||
Replicated: {
|
||||
@@ -321,6 +325,12 @@ export const setupVectorAgent = async (
|
||||
};
|
||||
|
||||
export const removeVectorAgent = async (serverId?: string) => {
|
||||
if (serverId) {
|
||||
const server = await findServerById(serverId);
|
||||
if (!server.sshKeyId) {
|
||||
throw new Error(`Server ${serverId} has no SSH key`);
|
||||
}
|
||||
}
|
||||
const docker = await getRemoteDocker(serverId);
|
||||
try {
|
||||
await docker.getService(VECTOR_SERVICE_NAME).remove();
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { InferResultType } from "@dokploy/server/types/with";
|
||||
import type { CreateServiceOptions, PortConfig } from "dockerode";
|
||||
import { resolveServiceNetworks } from "../../services/network";
|
||||
import { buildDokployLabels } from "../docker/dokploy-labels";
|
||||
import {
|
||||
calculateResources,
|
||||
generateBindMounts,
|
||||
@@ -98,7 +99,18 @@ export const buildLibsql = async (rawLibsql: LibsqlNested) => {
|
||||
Args: ["-c", finalCommand],
|
||||
}
|
||||
: {}),
|
||||
Labels,
|
||||
Labels: {
|
||||
...Labels,
|
||||
...buildDokployLabels({
|
||||
organizationId: libsql.environment.project.organizationId,
|
||||
projectId: libsql.environment.projectId,
|
||||
projectName: libsql.environment.project.name,
|
||||
environmentId: libsql.environmentId,
|
||||
environmentName: libsql.environment.name,
|
||||
applicationId: libsql.libsqlId,
|
||||
applicationName: libsql.name,
|
||||
}),
|
||||
},
|
||||
},
|
||||
Networks: resolvedNetworks,
|
||||
RestartPolicy,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { InferResultType } from "@dokploy/server/types/with";
|
||||
import type { CreateServiceOptions } from "dockerode";
|
||||
import { resolveServiceNetworks } from "../../services/network";
|
||||
import { buildDokployLabels } from "../docker/dokploy-labels";
|
||||
import {
|
||||
calculateResources,
|
||||
generateBindMounts,
|
||||
@@ -89,7 +90,18 @@ export const buildMariadb = async (rawMariadb: MariadbNested) => {
|
||||
Args: args,
|
||||
}),
|
||||
...(Ulimits && { Ulimits }),
|
||||
Labels,
|
||||
Labels: {
|
||||
...Labels,
|
||||
...buildDokployLabels({
|
||||
organizationId: mariadb.environment.project.organizationId,
|
||||
projectId: mariadb.environment.projectId,
|
||||
projectName: mariadb.environment.project.name,
|
||||
environmentId: mariadb.environmentId,
|
||||
environmentName: mariadb.environment.name,
|
||||
applicationId: mariadb.mariadbId,
|
||||
applicationName: mariadb.name,
|
||||
}),
|
||||
},
|
||||
},
|
||||
Networks: resolvedNetworks,
|
||||
RestartPolicy,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { InferResultType } from "@dokploy/server/types/with";
|
||||
import type { CreateServiceOptions } from "dockerode";
|
||||
import { resolveServiceNetworks } from "../../services/network";
|
||||
import { buildDokployLabels } from "../docker/dokploy-labels";
|
||||
import {
|
||||
calculateResources,
|
||||
generateBindMounts,
|
||||
@@ -145,7 +146,18 @@ ${command ?? "wait $MONGOD_PID"}`;
|
||||
Args: args,
|
||||
}),
|
||||
...(Ulimits && { Ulimits }),
|
||||
Labels,
|
||||
Labels: {
|
||||
...Labels,
|
||||
...buildDokployLabels({
|
||||
organizationId: mongo.environment.project.organizationId,
|
||||
projectId: mongo.environment.projectId,
|
||||
projectName: mongo.environment.project.name,
|
||||
environmentId: mongo.environmentId,
|
||||
environmentName: mongo.environment.name,
|
||||
applicationId: mongo.mongoId,
|
||||
applicationName: mongo.name,
|
||||
}),
|
||||
},
|
||||
},
|
||||
Networks: resolvedNetworks,
|
||||
RestartPolicy,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { InferResultType } from "@dokploy/server/types/with";
|
||||
import type { CreateServiceOptions } from "dockerode";
|
||||
import { resolveServiceNetworks } from "../../services/network";
|
||||
import { buildDokployLabels } from "../docker/dokploy-labels";
|
||||
import {
|
||||
calculateResources,
|
||||
generateBindMounts,
|
||||
@@ -95,7 +96,18 @@ export const buildMysql = async (rawMysql: MysqlNested) => {
|
||||
Args: args,
|
||||
}),
|
||||
...(Ulimits && { Ulimits }),
|
||||
Labels,
|
||||
Labels: {
|
||||
...Labels,
|
||||
...buildDokployLabels({
|
||||
organizationId: mysql.environment.project.organizationId,
|
||||
projectId: mysql.environment.projectId,
|
||||
projectName: mysql.environment.project.name,
|
||||
environmentId: mysql.environmentId,
|
||||
environmentName: mysql.environment.name,
|
||||
applicationId: mysql.mysqlId,
|
||||
applicationName: mysql.name,
|
||||
}),
|
||||
},
|
||||
},
|
||||
Networks: resolvedNetworks,
|
||||
RestartPolicy,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { InferResultType } from "@dokploy/server/types/with";
|
||||
import type { CreateServiceOptions } from "dockerode";
|
||||
import { resolveServiceNetworks } from "../../services/network";
|
||||
import { buildDokployLabels } from "../docker/dokploy-labels";
|
||||
import {
|
||||
calculateResources,
|
||||
generateBindMounts,
|
||||
@@ -87,7 +88,18 @@ export const buildPostgres = async (rawPostgres: PostgresNested) => {
|
||||
Args: args,
|
||||
}),
|
||||
...(Ulimits && { Ulimits }),
|
||||
Labels,
|
||||
Labels: {
|
||||
...Labels,
|
||||
...buildDokployLabels({
|
||||
organizationId: postgres.environment.project.organizationId,
|
||||
projectId: postgres.environment.projectId,
|
||||
projectName: postgres.environment.project.name,
|
||||
environmentId: postgres.environmentId,
|
||||
environmentName: postgres.environment.name,
|
||||
applicationId: postgres.postgresId,
|
||||
applicationName: postgres.name,
|
||||
}),
|
||||
},
|
||||
},
|
||||
Networks: resolvedNetworks,
|
||||
RestartPolicy,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { InferResultType } from "@dokploy/server/types/with";
|
||||
import type { CreateServiceOptions } from "dockerode";
|
||||
import { resolveServiceNetworks } from "../../services/network";
|
||||
import { buildDokployLabels } from "../docker/dokploy-labels";
|
||||
import {
|
||||
calculateResources,
|
||||
generateBindMounts,
|
||||
@@ -93,7 +94,18 @@ export const buildRedis = async (rawRedis: RedisNested) => {
|
||||
Args: ["-c", `redis-server --requirepass ${databasePassword}`],
|
||||
}),
|
||||
...(Ulimits && { Ulimits }),
|
||||
Labels,
|
||||
Labels: {
|
||||
...Labels,
|
||||
...buildDokployLabels({
|
||||
organizationId: redis.environment.project.organizationId,
|
||||
projectId: redis.environment.projectId,
|
||||
projectName: redis.environment.project.name,
|
||||
environmentId: redis.environmentId,
|
||||
environmentName: redis.environment.name,
|
||||
applicationId: redis.redisId,
|
||||
applicationName: redis.name,
|
||||
}),
|
||||
},
|
||||
},
|
||||
Networks: resolvedNetworks,
|
||||
RestartPolicy,
|
||||
|
||||
@@ -226,8 +226,8 @@ const applyDokployLabelsToCompose = (
|
||||
serviceName,
|
||||
});
|
||||
|
||||
service.labels = mergeDokployLabels(service.labels, dokployLabels);
|
||||
if (compose.composeType === "docker-compose") {
|
||||
service.labels = mergeDokployLabels(service.labels, dokployLabels);
|
||||
continue;
|
||||
}
|
||||
if (!service.deploy) {
|
||||
|
||||
Reference in New Issue
Block a user