fix(project-export-import): address review findings

- Allow absolute bind mount host paths on import
- Create imported networks in Docker and only reuse a network on the target server
- Register imported enabled backups, volume backups and schedules
- Write file mounts before restoring dumps; remove import files when the job ends
- Claim the import before validating so it cannot start twice
- Rewrite app names in project, environment and build variables; keep source project variables when importing into an existing project
- Only reuse the raw Traefik config when the imported domains are unchanged
- Validate conflict renames and detect database external port conflicts
- Skip rotation when a root password is unsafe
- Detect uploaded certificates by the hosts they cover
- Export shared resources by value in organization exports
- Fix Back after unlock and the missing "Create from bundle" option in the import wizard
This commit is contained in:
Narciso
2026-10-15 17:56:51 -04:00
parent 63a1a0c89f
commit 3f90bb9159
18 changed files with 798 additions and 153 deletions
@@ -185,6 +185,30 @@ const readDb = (): TransferDb =>
webServerSettings: { findFirst: async () => undefined },
sshKeys: { findMany: async () => [] },
domains: { findMany: async () => [] },
postgres: {
findMany: async () => [],
findFirst: async () => undefined,
},
mysql: {
findMany: async () => [],
findFirst: async () => undefined,
},
mariadb: {
findMany: async () => [],
findFirst: async () => undefined,
},
mongo: {
findMany: async () => [],
findFirst: async () => undefined,
},
redis: {
findMany: async () => [],
findFirst: async () => undefined,
},
libsql: {
findMany: async () => [],
findFirst: async () => undefined,
},
ports: { findMany: async () => [] },
},
}) as any as TransferDb;
@@ -138,6 +138,30 @@ const fakeDb = (): TransferDb =>
{ domainId: "dom-existing-1", host: "existing.example.com" },
],
},
postgres: {
findMany: async () => [],
findFirst: async () => undefined,
},
mysql: {
findMany: async () => [],
findFirst: async () => undefined,
},
mariadb: {
findMany: async () => [],
findFirst: async () => undefined,
},
mongo: {
findMany: async () => [],
findFirst: async () => undefined,
},
redis: {
findMany: async () => [],
findFirst: async () => undefined,
},
libsql: {
findMany: async () => [],
findFirst: async () => undefined,
},
ports: {
findMany: async () => [
{
@@ -244,26 +244,53 @@ describe("detectRequirements", () => {
});
});
it("detects a certificate requirement via customCertResolver, not a direct FK", async () => {
it("detects an uploaded certificate by the https domains it covers on the same server", async () => {
const application = {
...baseApplication,
serverId: "server-1",
domains: [
{
domainId: "dom-1",
certificateType: "custom",
customCertResolver: "cert-path-abc",
host: "app.example.com",
https: true,
certificateType: "none",
},
{
domainId: "dom-2",
host: "other.example.com",
https: true,
certificateType: "none",
},
],
};
const certificate = {
certificateId: "cert-1",
certificateData:
"-----BEGIN CERTIFICATE-----\n" +
"MIICPjCCAeOgAwIBAgIJAOEJ+5C9HjNZMAoGCCqGSM49BAMCMBoxGDAWBgNVBAMM\n" +
"D2FwcC5leGFtcGxlLmNvbTAeFw0yNjEwMTUyMTQ0NDlaFw0zNjEwMTIyMTQ0NDla\n" +
"MBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTCCAUswggEDBgcqhkjOPQIBMIH3\n" +
"AgEBMCwGByqGSM49AQECIQD/////AAAAAQAAAAAAAAAAAAAAAP//////////////\n" +
"/zBbBCD/////AAAAAQAAAAAAAAAAAAAAAP///////////////AQgWsY12Ko6k+ez\n" +
"671VdpiGvGUdBrDMU7D2O848PifSYEsDFQDEnTYIhucEk2pmeOETnSa3gZ9+kARB\n" +
"BGsX0fLhLEJH+Lzm5WOkQPJ3A32BLeszoPShOUXYmMKWT+NC4v4af5uO5+tKfA+e\n" +
"FivOM1drMV7Oy7ZAaDe/UfUCIQD/////AAAAAP//////////vOb6racXnoTzucrC\n" +
"/GMlUQIBAQNCAASL1XU5x61CV4NiEe+/UmBo6LjNyEpG597XYQ1EPv1QhtboQ1Uo\n" +
"no7Yf/H6T/8elaaCFtgl/sZqxHP4HDLkIFYtox4wHDAaBgNVHREEEzARgg9hcHAu\n" +
"ZXhhbXBsZS5jb20wCgYIKoZIzj0EAwIDSQAwRgIhANb3ATUO9PvG5Ta/5M4ggr11\n" +
"Rb3EoP/+A1AjbYmYEszLAiEA4nxjPPJXLb1zXMeiqz6xyI+8ETyjucTZTkfj4hyV\n" +
"6DM=\n" +
"-----END CERTIFICATE-----\n",
serverId: "server-1",
};
const db = fakeDb({
certificates: {
findMany: async () => [
certificate,
{
certificateId: "cert-1",
certificatePath: "cert-path-abc",
certificateData:
"-----BEGIN CERTIFICATE-----\nabc\n-----END CERTIFICATE-----",
serverId: "server-1",
...certificate,
certificateId: "cert-other-server",
serverId: null,
},
],
},
@@ -278,6 +305,7 @@ describe("detectRequirements", () => {
const certReqs = requirements.filter((r) => r.kind === "certificate");
expect(certReqs).toHaveLength(1);
expect(certReqs[0]?.originalId).toBe("cert-1");
expect(certReqs[0]?.usedBy).toHaveLength(1);
expect((certReqs[0] as any).fingerprint).toMatch(/^[a-f0-9]{64}$/);
expect((certReqs[0] as any).serverId).toBe("server-1");
});
@@ -181,6 +181,41 @@ describe("buildRotationPlan", () => {
]);
});
it("keeps a mysql service whose root password is too short", async () => {
const tx = {
query: {
mysql: {
findFirst: async () => ({
mysqlId: "my-new-1",
databaseUser: "app",
databaseName: "app",
databasePassword: "long-enough-password",
databaseRootPassword: "1",
}),
},
},
} as any;
const snapshot = {
resources: {
services: [
{
kind: "mysql",
mysqlId: "my-old-1",
name: "db",
environmentId: "env-1",
},
],
},
} as any;
const { specs, unsafe } = await buildRotationPlan(tx, snapshot, {
mysql: { "my-old-1": "my-new-1" },
});
expect(specs).toHaveLength(0);
expect(unsafe).toHaveLength(1);
});
it("still generates independent new passwords when app and root passwords differ", async () => {
const tx = {
query: {
@@ -209,6 +209,30 @@ const readDb = (): TransferDb =>
webServerSettings: { findFirst: async () => undefined },
sshKeys: { findMany: async () => [] },
domains: { findMany: async () => [] },
postgres: {
findMany: async () => [],
findFirst: async () => undefined,
},
mysql: {
findMany: async () => [],
findFirst: async () => undefined,
},
mariadb: {
findMany: async () => [],
findFirst: async () => undefined,
},
mongo: {
findMany: async () => [],
findFirst: async () => undefined,
},
redis: {
findMany: async () => [],
findFirst: async () => undefined,
},
libsql: {
findMany: async () => [],
findFirst: async () => undefined,
},
ports: { findMany: async () => [] },
},
}) as any as TransferDb;
@@ -187,6 +187,11 @@ export const ImportProject = ({
const handleUnlock = async () => {
if (!importId) return;
// The server deletes the encrypted payload after the first unlock, so Back + Unlock reuses that result.
if (unlockResult) {
setStep(2);
return;
}
try {
const result = await unlock({ importId, passphrase });
setUnlockResult(result);
@@ -130,11 +130,18 @@ const RequirementRow = ({
{candidates.length > 0 && (
<Select
value={
choice.action === "skip" ? "__skip__" : (choice.targetId ?? "")
choice.action === "skip"
? "__skip__"
: choice.action === "create"
? "__create__"
: (choice.targetId ?? "")
}
onValueChange={(value) => {
if (value === "__skip__") {
onChange({ requirementKey: key, action: "skip" });
if (value === "__skip__" || value === "__create__") {
onChange({
requirementKey: key,
action: value === "__skip__" ? "skip" : "create",
});
return;
}
onChange({
@@ -148,6 +155,10 @@ const RequirementRow = ({
<SelectValue placeholder="Map to..." />
</SelectTrigger>
<SelectContent>
{"configIncluded" in requirement &&
requirement.configIncluded === true && (
<SelectItem value="__create__">Create from bundle</SelectItem>
)}
{candidates.map((candidate) => (
<SelectItem key={candidate.id} value={candidate.id}>
{candidate.name}
@@ -1,9 +1,18 @@
import { randomUUID } from "node:crypto";
import { getDokployImageTag, IS_CLOUD, TRAEFIK_VERSION } from "@dokploy/server";
import {
findBackupById,
getDokployImageTag,
IS_CLOUD,
scheduleBackup,
scheduleJob,
scheduleVolumeBackup,
TRAEFIK_VERSION,
} from "@dokploy/server";
import { db } from "@dokploy/server/db";
import type {
DeployCandidate,
InstanceVersions,
TransferResult,
} from "@dokploy/server/services/project-export-import";
import {
assertNoActiveTransfer,
@@ -60,6 +69,7 @@ import { z } from "zod";
import { PLAN_LIMITS } from "@/server/api/utils/plan-limits";
import type { DeploymentJob } from "@/server/queues/queue-types";
import { myQueue } from "@/server/queues/queueSetup";
import { schedule } from "@/server/utils/backup";
import { getCurrentPlan } from "@/server/utils/billing";
import { deploy } from "@/server/utils/deploy";
import packageInfo from "../../../package.json";
@@ -79,6 +89,94 @@ const localInstanceContext = () => {
return { dokployVersion, instance };
};
const scheduleImportedJobs = async (result: TransferResult): Promise<void> => {
const serviceIds = Object.values(result.idMap).flatMap((ids) =>
Object.values(ids),
);
if (serviceIds.length === 0) return;
const [importedBackups, importedVolumeBackups, importedSchedules] =
await Promise.all([
db.query.backups.findMany({
where: (fields, { and, eq, inArray, or }) =>
and(
eq(fields.enabled, true),
or(
inArray(fields.postgresId, serviceIds),
inArray(fields.mysqlId, serviceIds),
inArray(fields.mariadbId, serviceIds),
inArray(fields.mongoId, serviceIds),
inArray(fields.libsqlId, serviceIds),
inArray(fields.composeId, serviceIds),
),
),
columns: { backupId: true },
}),
db.query.volumeBackups.findMany({
where: (fields, { and, eq, inArray, or }) =>
and(
eq(fields.enabled, true),
or(
inArray(fields.applicationId, serviceIds),
inArray(fields.postgresId, serviceIds),
inArray(fields.mysqlId, serviceIds),
inArray(fields.mariadbId, serviceIds),
inArray(fields.mongoId, serviceIds),
inArray(fields.redisId, serviceIds),
inArray(fields.libsqlId, serviceIds),
inArray(fields.composeId, serviceIds),
),
),
}),
db.query.schedules.findMany({
where: (fields, { and, eq, inArray, or }) =>
and(
eq(fields.enabled, true),
or(
inArray(fields.applicationId, serviceIds),
inArray(fields.composeId, serviceIds),
),
),
}),
]);
for (const { backupId } of importedBackups) {
const backup = await findBackupById(backupId);
if (IS_CLOUD) {
await schedule({
cronSchedule: backup.schedule,
backupId,
type: "backup",
});
} else {
scheduleBackup(backup);
}
}
for (const volumeBackup of importedVolumeBackups) {
if (IS_CLOUD) {
await schedule({
cronSchedule: volumeBackup.cronExpression,
volumeBackupId: volumeBackup.volumeBackupId,
type: "volume-backup",
});
} else {
await scheduleVolumeBackup(volumeBackup.volumeBackupId);
}
}
for (const importedSchedule of importedSchedules) {
if (IS_CLOUD) {
await schedule({
scheduleId: importedSchedule.scheduleId,
type: "schedule",
cronSchedule: importedSchedule.cronExpression,
timezone: importedSchedule.timezone,
});
} else {
scheduleJob(importedSchedule);
}
}
};
const enqueueImportDeploys = async (
candidates: DeployCandidate[],
): Promise<void> => {
@@ -801,20 +899,26 @@ export const projectExportImportRouter = createTRPCRouter({
});
}
assertPlanChecksPass(
await validatePlan(
db,
job.snapshot,
job.plan,
ctx.session.activeOrganizationId,
),
);
updateImportJob(input.importId, { status: "applying" });
try {
assertPlanChecksPass(
await validatePlan(
db,
job.snapshot,
job.plan,
ctx.session.activeOrganizationId,
),
);
} catch (error) {
updateImportJob(input.importId, { status: "planned" });
throw error;
}
runImport(input.importId, {
db,
organizationId: ctx.session.activeOrganizationId,
enqueueDeploy: enqueueImportDeploys,
scheduleJobs: scheduleImportedJobs,
});
await audit(ctx, {
@@ -4,6 +4,7 @@ import {
projectTags,
tags,
} from "@dokploy/server/db/schema";
import { parse } from "dotenv";
import { eq } from "drizzle-orm";
import type { ApplyServicesContext, DbServiceKind } from "./apply/services";
import {
@@ -11,8 +12,10 @@ import {
insertApplication,
insertCompose,
insertDbService,
rewriteVaultNames,
} from "./apply/services";
import { applyVaultAssignments, insertSharedByValue } from "./apply/shared";
import { replaceAppNames } from "./appname";
import { conflictResolutionsOf, resolvedProjectName } from "./conflicts";
import type { TransferTx, TransferWriteDb } from "./queries";
import {
@@ -31,20 +34,48 @@ import type {
} from "./types";
import { transferResultSchema } from "./types";
const rewriteSharedEnv = (
row: Record<string, unknown>,
plan: Plan,
vaultNameMap: Record<string, string>,
): Record<string, unknown> =>
typeof row.env === "string"
? {
...row,
env: rewriteVaultNames(
replaceAppNames(row.env, plan.appNameMap),
vaultNameMap,
),
}
: row;
const ENV_KEY_REGEX = /^\s*(?:export\s+)?([\w.-]+)\s*=/;
const appendMissingEnv = (target: string, source: string): string => {
const targetKeys = new Set(Object.keys(parse(target)));
const missing = source.split("\n").filter((line) => {
const key = ENV_KEY_REGEX.exec(line)?.[1];
return key && !targetKeys.has(key);
});
if (missing.length === 0) return target;
return [target.trimEnd(), ...missing].filter(Boolean).join("\n");
};
const insertProjectAndEnvironments = async (
tx: TransferTx,
snapshot: Snapshot,
plan: Plan,
organizationId: string,
secretsPlain: Map<string, string>,
vaultNameMap: Record<string, string>,
): Promise<{ projectId: string; environmentIdMap: Record<string, string> }> => {
const { tags: projectTagRows, ...restoredProject } = rewriteSharedEnv(
restoreSecretFields("project", snapshot.resources.project, secretsPlain),
plan,
vaultNameMap,
);
let projectId: string;
if (plan.target.kind === "newProject") {
const { tags: projectTagRows, ...restoredProject } = restoreSecretFields(
"project",
snapshot.resources.project,
secretsPlain,
);
const [inserted] = await tx
.insert(projects)
.values({
@@ -84,7 +115,11 @@ const insertProjectAndEnvironments = async (
let firstInsertedId: string | undefined;
let sawDefault = false;
for (const env of snapshot.resources.environments) {
const restored = restoreSecretFields("environment", env, secretsPlain);
const restored = rewriteSharedEnv(
restoreSecretFields("environment", env, secretsPlain),
plan,
vaultNameMap,
);
const [inserted] = await tx
.insert(environments)
.values({
@@ -110,7 +145,16 @@ const insertProjectAndEnvironments = async (
const baseName = plan.target.environmentName;
let index = 0;
for (const env of snapshot.resources.environments) {
const restored = restoreSecretFields("environment", env, secretsPlain);
const restored = rewriteSharedEnv(
restoreSecretFields("environment", env, secretsPlain),
plan,
vaultNameMap,
);
// Imported services read ${{project.*}} as ${{environment.*}} here, so the existing project's variables stay untouched.
restored.env = appendMissingEnv(
typeof restored.env === "string" ? restored.env : "",
typeof restoredProject.env === "string" ? restoredProject.env : "",
);
const name =
index === 0
? baseName
@@ -188,6 +232,7 @@ export const apply = async (
plan,
organizationId,
secretsPlain,
vaultNameMap,
);
const conflictResolutions = conflictResolutionsOf(plan);
@@ -97,11 +97,12 @@ export const insertMounts = async (
secretsPlain: Map<string, string>,
): Promise<void> => {
for (const row of rows) {
for (const field of ["filePath", "hostPath"]) {
const value = row[field];
if (typeof value === "string" && value && isUnsafeBundlePath(value)) {
throw new Error(`Unsafe mount ${field} in bundle: "${value}"`);
}
if (
typeof row.filePath === "string" &&
row.filePath &&
isUnsafeBundlePath(row.filePath)
) {
throw new Error(`Unsafe mount filePath in bundle: "${row.filePath}"`);
}
const restored = restoreSecretFields("mount", row, secretsPlain);
const volumeName =
@@ -125,6 +126,27 @@ export const insertMounts = async (
}
};
export const resolvePublishedPort = (
port: unknown,
conflictResolutions: Record<string, ConflictResolution>,
): number | null => {
const resolution =
conflictResolutions[conflictKey({ kind: "port", value: String(port) })];
if (resolution?.action === "skip") return null;
const publishedPort =
resolution?.action === "rename" && resolution.value
? Number.parseInt(resolution.value, 10)
: port;
if (
typeof publishedPort !== "number" ||
publishedPort < 1 ||
publishedPort > 65535
) {
throw new Error(`Invalid published port: ${String(publishedPort)}`);
}
return publishedPort;
};
export const insertPorts = async (
tx: TransferTx,
rows: Record<string, unknown>[],
@@ -134,22 +156,11 @@ export const insertPorts = async (
): Promise<void> => {
if (portsStrategy === "disable") return;
for (const row of rows) {
const resolution =
conflictResolutions[
conflictKey({ kind: "port", value: String(row.publishedPort) })
];
if (resolution?.action === "skip") continue;
const publishedPort =
resolution?.action === "rename" && resolution.value
? Number.parseInt(resolution.value, 10)
: row.publishedPort;
if (
typeof publishedPort !== "number" ||
publishedPort < 1 ||
publishedPort > 65535
) {
throw new Error(`Invalid published port: ${String(publishedPort)}`);
}
const publishedPort = resolvePublishedPort(
row.publishedPort,
conflictResolutions,
);
if (publishedPort === null) continue;
await tx.insert(ports).values({
...row,
portId: undefined,
@@ -31,10 +31,11 @@ import {
insertSchedules,
insertSecurity,
insertVolumeBackups,
resolvePublishedPort,
} from "./children";
import type { SharedIdMaps } from "./shared";
const rewriteVaultNames = (
export const rewriteVaultNames = (
env: string,
vaultNameMap: Record<string, string>,
): string => {
@@ -59,6 +60,14 @@ const CHILD_ARRAY_KEYS = [
"backups",
];
const VARIABLE_FIELDS = [
"env",
"buildArgs",
"buildSecrets",
"previewEnv",
"previewBuildArgs",
];
const remapCommonFields = (
ctx: ApplyServicesContext,
row: Record<string, unknown>,
@@ -74,11 +83,17 @@ const remapCommonFields = (
for (const key of CHILD_ARRAY_KEYS) {
delete restored[key];
}
if (typeof restored.env === "string") {
restored.env = rewriteVaultNames(
replaceAppNames(restored.env, plan.appNameMap),
for (const field of VARIABLE_FIELDS) {
const value = restored[field];
if (typeof value !== "string") continue;
const rewritten = rewriteVaultNames(
replaceAppNames(value, plan.appNameMap),
vaultNameMap,
);
restored[field] =
plan.target.kind === "existingProject"
? rewritten.replaceAll("${{project.", "${{environment.")
: rewritten;
}
const oldNetworkIds = (row.networkIds as string[] | null) ?? [];
const networkIds = oldNetworkIds
@@ -470,9 +485,14 @@ export const insertDbService = async (
);
const { table, hasBackups } = DB_TABLES[kind];
const externalPort =
typeof common.externalPort === "number" && plan.portsStrategy !== "disable"
? resolvePublishedPort(common.externalPort, ctx.conflictResolutions)
: null;
const [inserted] = await tx
.insert(table)
.values({ ...common, [idField]: undefined } as never)
.values({ ...common, externalPort, [idField]: undefined } as never)
.returning();
if (!inserted) {
throw new Error(`Failed to insert ${kind} ${oldId}`);
@@ -223,6 +223,7 @@ export const insertSharedByValue = async (
.values({
...row,
networkId: undefined,
dockerId: null,
name:
resolution?.action === "rename" && resolution.value
? resolution.value.trim()
@@ -63,7 +63,18 @@ export type RunExportContext = {
};
export const fullExportScope = (): ExportScope =>
exportScopeSchema.parse({ dumps: { mode: "embed" } });
exportScopeSchema.parse({
shared: {
destination: "value",
registry: "value",
sshKey: "value",
network: "value",
certificate: "value",
dnsProvider: "value",
vaultProvider: "reference",
},
dumps: { mode: "embed" },
});
export const bundleBuildServer = async (
db: TransferDb,
@@ -13,11 +13,13 @@ import { postApply } from "./post-apply";
import type { TransferWriteDb } from "./queries";
import { remove, safeErrorMessage } from "./remote-fs";
import { restoreDumps } from "./restore";
import type { TransferResult } from "./types";
export type RunImportContext = {
db: TransferWriteDb;
organizationId: string;
enqueueDeploy?: (candidates: DeployCandidate[]) => Promise<void>;
scheduleJobs?: (result: TransferResult) => Promise<void>;
};
const discardImportFiles = async (job: ImportJob | undefined) => {
@@ -26,17 +28,16 @@ const discardImportFiles = async (job: ImportJob | undefined) => {
};
export const runImport = (importId: string, ctx: RunImportContext): void => {
executeImport(importId, ctx).catch(async (error) => {
const job = getImportJob(importId);
updateImportJob(importId, {
status: job?.result ? "done" : "failed",
partial: !!job?.result,
error: safeErrorMessage(error),
});
if (!job?.result) {
await discardImportFiles(job);
}
});
executeImport(importId, ctx)
.catch((error) => {
const job = getImportJob(importId);
updateImportJob(importId, {
status: job?.result ? "done" : "failed",
partial: !!job?.result,
error: safeErrorMessage(error),
});
})
.finally(() => discardImportFiles(getImportJob(importId)));
};
const executeImport = async (
@@ -73,16 +74,7 @@ const executeImport = async (
updateImportJob(importId, {
status: "restoring",
message: "Restoring database dumps...",
});
const restoreErrors = await restoreDumps(ctx.db, snapshot, result, workDir, {
emit: (message) => updateImportJob(importId, { message }),
});
updateImportJob(importId, { restoreErrors });
updateImportJob(importId, {
status: "deploying",
message: "Copying files and preparing deployments...",
message: "Copying files...",
});
const { deployCandidates, postApplyErrors } = await postApply(
ctx.db,
@@ -94,11 +86,23 @@ const executeImport = async (
emit: (message) => updateImportJob(importId, { message }),
},
);
if (postApplyErrors.length > 0) {
restoreErrors.push(...postApplyErrors);
updateImportJob(importId, { restoreErrors });
updateImportJob(importId, { message: "Restoring database dumps..." });
const restoreErrors = [
...postApplyErrors,
...(await restoreDumps(ctx.db, snapshot, result, workDir, {
emit: (message) => updateImportJob(importId, { message }),
})),
];
updateImportJob(importId, { restoreErrors, deployCandidates });
updateImportJob(importId, {
status: "deploying",
message: "Preparing deployments...",
});
if (ctx.scheduleJobs) {
await ctx.scheduleJobs(result);
}
updateImportJob(importId, { deployCandidates });
if (ctx.enqueueDeploy && plan.deployAfterImport !== false) {
await ctx.enqueueDeploy(deployCandidates);
@@ -9,7 +9,11 @@ import {
} from "@dokploy/server/utils/process/execAsync";
import { TRPCError } from "@trpc/server";
import { finalDomainHost, regenerateAppName } from "./appname";
import { CONFLICT_OPTIONS, resolveConflicts } from "./conflicts";
import {
CONFLICT_OPTIONS,
conflictResolutionsOf,
resolveConflicts,
} from "./conflicts";
import { DUMPABLE_DB_TYPES } from "./dumps";
import type { TransferDb } from "./queries";
import { escapeShellArg, safeErrorMessage } from "./remote-fs";
@@ -177,19 +181,23 @@ const collectPortUsage = (
>();
const serviceServerMap = buildServiceServerMap(snapshot);
for (const service of snapshot.resources.services) {
if (service.kind !== "application") continue;
const ports =
(service.ports as Record<string, unknown>[] | undefined) ?? [];
if (ports.length === 0) continue;
const originalServerId =
serviceServerMap.get(service.applicationId as string) ?? null;
const kind = service.kind as ServiceType;
const publishedPorts =
kind === "application"
? ((service.ports as Record<string, unknown>[] | undefined) ?? []).map(
(port) => port.publishedPort as number,
)
: typeof service.externalPort === "number"
? [service.externalPort]
: [];
if (publishedPorts.length === 0) continue;
const serviceId = service[SERVICE_ID_FIELD[kind]] as string;
const originalServerId = serviceServerMap.get(serviceId) ?? null;
const destServerId = serverMap[originalServerId ?? "null"] ?? null;
const serviceId = service.applicationId as string;
const environmentId = service.environmentId as string;
const serviceName =
(service.name as string | undefined) ?? (service.appName as string) ?? "";
for (const port of ports) {
const publishedPort = port.publishedPort as number;
for (const publishedPort of publishedPorts) {
const key = `${destServerId ?? "local"}:${publishedPort}`;
const entry = byKey.get(key) ?? {
publishedPort,
@@ -198,7 +206,7 @@ const collectPortUsage = (
};
entry.usedBy.push({
serviceId,
serviceType: "application",
serviceType: kind,
serviceName,
environmentId,
});
@@ -208,6 +216,81 @@ const collectPortUsage = (
return byKey;
};
const networkDestServerId = (
requirement: Requirement,
serviceServerMap: Map<string, string | null>,
serverMap: Record<string, string | null>,
): string | null => {
const serviceId = requirement.usedBy[0]?.serviceId;
const originalServerId = serviceId
? (serviceServerMap.get(serviceId) ?? null)
: null;
return serverMap[originalServerId ?? "null"] ?? null;
};
const findUsedPorts = async (
db: TransferDb,
publishedPorts: number[],
): Promise<{ existingId: string; port: number; serverId: string | null }[]> => {
const existingPorts = await db.query.ports.findMany({
where: (fields, { inArray }) =>
inArray(fields.publishedPort, publishedPorts),
with: { application: { columns: { serverId: true } } },
});
const usedPorts = existingPorts.map((row) => ({
existingId: row.portId,
port: row.publishedPort,
serverId: row.application?.serverId ?? null,
}));
const dbRows = await Promise.all([
db.query.postgres
.findMany({
where: (fields, { inArray }) =>
inArray(fields.externalPort, publishedPorts),
})
.then((rows) => rows.map((row) => [row.postgresId, row] as const)),
db.query.mysql
.findMany({
where: (fields, { inArray }) =>
inArray(fields.externalPort, publishedPorts),
})
.then((rows) => rows.map((row) => [row.mysqlId, row] as const)),
db.query.mariadb
.findMany({
where: (fields, { inArray }) =>
inArray(fields.externalPort, publishedPorts),
})
.then((rows) => rows.map((row) => [row.mariadbId, row] as const)),
db.query.mongo
.findMany({
where: (fields, { inArray }) =>
inArray(fields.externalPort, publishedPorts),
})
.then((rows) => rows.map((row) => [row.mongoId, row] as const)),
db.query.redis
.findMany({
where: (fields, { inArray }) =>
inArray(fields.externalPort, publishedPorts),
})
.then((rows) => rows.map((row) => [row.redisId, row] as const)),
db.query.libsql
.findMany({
where: (fields, { inArray }) =>
inArray(fields.externalPort, publishedPorts),
})
.then((rows) => rows.map((row) => [row.libsqlId, row] as const)),
]);
for (const [existingId, row] of dbRows.flat()) {
if (row.externalPort === null) continue;
usedPorts.push({
existingId,
port: row.externalPort,
serverId: row.serverId ?? null,
});
}
return usedPorts;
};
const detectConflicts = async (
db: TransferDb,
snapshot: Snapshot,
@@ -263,20 +346,17 @@ const detectConflicts = async (
const publishedPorts = [
...new Set([...portUsage.values()].map((v) => v.publishedPort)),
];
const existingPorts = await db.query.ports.findMany({
where: (fields, { inArray }) =>
inArray(fields.publishedPort, publishedPorts),
with: { application: { columns: { serverId: true } } },
});
for (const row of existingPorts) {
const rowServerId = row.application?.serverId ?? null;
const key = `${rowServerId ?? "local"}:${row.publishedPort}`;
const usedPorts = await findUsedPorts(db, publishedPorts);
const seen = new Set<string>();
for (const { existingId, port, serverId } of usedPorts) {
const key = `${serverId ?? "local"}:${port}`;
const entry = portUsage.get(key);
if (!entry) continue;
if (!entry || seen.has(key)) continue;
seen.add(key);
conflicts.push({
kind: "port",
value: String(row.publishedPort),
existingId: row.portId,
value: String(port),
existingId,
usedBy: entry.usedBy,
options: CONFLICT_OPTIONS.port,
});
@@ -287,11 +367,19 @@ const detectConflicts = async (
if (requirement.kind !== "network") continue;
const key = requirementKey(requirement);
if (resolutions[key]?.action !== "create") continue;
const destServerId = networkDestServerId(
requirement,
buildServiceServerMap(snapshot),
serverMap,
);
const existingNetwork = await db.query.network.findFirst({
where: (fields, { and, eq }) =>
where: (fields, { and, eq, isNull }) =>
and(
eq(fields.organizationId, organizationId),
eq(fields.name, requirement.name),
destServerId
? eq(fields.serverId, destServerId)
: isNull(fields.serverId),
),
});
if (existingNetwork) {
@@ -403,8 +491,35 @@ export const buildPlan = async (
}
}
const appNameMap: Record<string, string> = {};
const serviceServerMapForAppNames = buildServiceServerMap(snapshot);
for (const requirement of snapshot.requirements) {
if (requirement.kind !== "network") continue;
const key = requirementKey(requirement);
const resolution = resolutions[key];
if (
userChoices.resolutions?.[key] ||
resolution?.action !== "reuse" ||
!resolution.targetId
) {
continue;
}
const targetId = resolution.targetId;
const existing = await db.query.network.findFirst({
where: (fields, { eq }) => eq(fields.networkId, targetId),
columns: { serverId: true },
});
if (
(existing?.serverId ?? null) !==
networkDestServerId(requirement, serviceServerMapForAppNames, serverMap)
) {
resolutions[key] = {
requirementKey: key,
action: requirement.configIncluded ? "create" : "skip",
};
}
}
const appNameMap: Record<string, string> = {};
for (const service of snapshot.resources.services) {
const serviceType = service.kind as ServiceType;
const oldAppName = service.appName as string | undefined;
@@ -522,6 +637,93 @@ export type ValidatePlanOptions = {
probeVersion?: RunVersionProbe;
};
const checkConflictRenames = async (
db: TransferDb,
snapshot: Snapshot,
plan: Plan,
organizationId: string,
): Promise<PlanValidationCheck[]> => {
const renames = Object.entries(conflictResolutionsOf(plan)).flatMap(
([key, resolution]) => {
const [kind, ...rest] = key.split(":");
return resolution.action === "rename" && resolution.value
? [{ kind, original: rest.join(":"), value: resolution.value.trim() }]
: [];
},
);
const checks: PlanValidationCheck[] = [];
const domainRenames = renames.filter((rename) => rename.kind === "domain");
if (domainRenames.length > 0) {
const importedHosts = collectDomainHosts(snapshot, {
appNameMap: plan.appNameMap,
serverMap: plan.serverMap,
serverIpMap: await buildServerIpMap(db, plan.serverMap, organizationId),
domainStrategy: plan.domainStrategy,
});
const existingHosts = new Set(
(
await db.query.domains.findMany({
where: (fields, { inArray }) =>
inArray(
fields.host,
domainRenames.map((rename) => rename.value),
),
columns: { host: true },
})
).map((row) => row.host),
);
for (const rename of domainRenames) {
const taken =
existingHosts.has(rename.value) ||
importedHosts.has(rename.value) ||
domainRenames.some(
(other) => other !== rename && other.value === rename.value,
);
checks.push({
check: `rename:domain:${rename.original}`,
ok: !taken,
detail: taken ? `${rename.value} is already in use` : undefined,
});
}
}
const portRenames = renames.filter((rename) => rename.kind === "port");
if (portRenames.length > 0) {
const portUsage = [...collectPortUsage(snapshot, plan.serverMap).values()];
const usedPorts = await findUsedPorts(
db,
portRenames.map((rename) => Number(rename.value)),
);
for (const rename of portRenames) {
const port = Number(rename.value);
const servers = new Set(
portUsage
.filter((usage) => String(usage.publishedPort) === rename.original)
.map((usage) => usage.destServerId),
);
const taken =
usedPorts.some(
(used) => used.port === port && servers.has(used.serverId),
) ||
portUsage.some(
(usage) =>
usage.publishedPort === port && servers.has(usage.destServerId),
) ||
portRenames.some(
(other) => other !== rename && Number(other.value) === port,
);
checks.push({
check: `rename:port:${rename.original}`,
ok: !taken,
detail: taken ? `Port ${port} is already in use` : undefined,
});
}
}
return checks;
};
export const validatePlan = async (
db: TransferDb,
snapshot: Snapshot,
@@ -675,6 +877,19 @@ export const validatePlan = async (
...row,
serverId: probeServerIdForRequirement(key) ?? undefined,
});
} else if (kind === "network") {
const row = await db.query.network.findFirst({
where: (fields, { eq }) =>
eq(fields.networkId, resolution.targetId as string),
});
if (!row || row.organizationId !== organizationId) {
throw new Error("Not found in this organization");
}
if ((row.serverId ?? null) !== probeServerIdForRequirement(key)) {
throw new Error(
`Network ${row.name} is on a different server than the services that use it`,
);
}
} else {
continue;
}
@@ -688,6 +903,10 @@ export const validatePlan = async (
}
}
checks.push(
...(await checkConflictRenames(db, snapshot, plan, organizationId)),
);
return checks;
};
@@ -4,8 +4,10 @@ import { findApplicationById } from "@dokploy/server/services/application";
import type { Certificate } from "@dokploy/server/services/certificate";
import { createCertificateFiles } from "@dokploy/server/services/certificate";
import { createFileMount } from "@dokploy/server/services/mount";
import { recreateNetwork } from "@dokploy/server/services/network";
import { manageDomain } from "@dokploy/server/utils/traefik/domain";
import { replaceAppNames } from "./appname";
import { conflictResolutionsOf } from "./conflicts";
import { assertSafeBundlePath } from "./deserialize";
import type { TransferDb } from "./queries";
import {
@@ -27,6 +29,7 @@ import type {
Snapshot,
TransferResult,
} from "./types";
import { conflictKey } from "./types";
export type DeployCandidate = {
serviceId: string;
@@ -97,6 +100,41 @@ const copyComposeFiles = async (
emit(`Copied compose files for ${newAppName}`);
};
const createImportedNetworks = async (
snapshot: Snapshot,
plan: Plan,
result: TransferResult,
emit: Emit,
): Promise<RestoreError[]> => {
const errors: RestoreError[] = [];
const conflictResolutions = conflictResolutionsOf(plan);
for (const requirement of snapshot.requirements) {
if (requirement.kind !== "network") continue;
if (plan.resolutions[requirementKey(requirement)]?.action !== "create") {
continue;
}
const conflict =
conflictResolutions[
conflictKey({ kind: "network", value: requirement.name })
];
if (conflict?.action === "keep" || conflict?.action === "skip") continue;
if (!requirement.originalId) continue;
const newNetworkId = result.idMap.network?.[requirement.originalId];
if (!newNetworkId) continue;
try {
await recreateNetwork(newNetworkId);
emit(`Created Docker network ${requirement.name}`);
} catch (error) {
errors.push({
serviceId: newNetworkId,
serviceName: requirement.name,
error: safeErrorMessage(error),
});
}
}
return errors;
};
const writeCertificateFiles = async (
db: TransferDb,
snapshot: Snapshot,
@@ -142,10 +180,17 @@ const writeRawTraefikConfig = async (
return true;
};
const domainKeys = (rows: { host: string; enabled?: boolean | null }[]) =>
rows
.map((row) => `${row.host}|${row.enabled ?? true}`)
.sort()
.join(",");
const registerTraefikDomains = async (
oldAppName: string,
newAppName: string,
newApplicationId: string,
sourceDomains: { host: string; enabled?: boolean | null }[],
appNameMap: Record<string, string>,
workDir: Location,
pathsFn: typeof paths,
@@ -154,14 +199,24 @@ const registerTraefikDomains = async (
const app = await findApplicationById(newApplicationId);
const serverId = app.serverId ?? null;
const usedRawConfig = await writeRawTraefikConfig(
oldAppName,
newAppName,
appNameMap,
serverId,
workDir,
pathsFn,
);
// The raw file still routes the source hosts, so it is only safe when the import kept them.
const domainsUnchanged =
domainKeys(
sourceDomains.map((domain) => ({
...domain,
host: replaceAppNames(domain.host, appNameMap),
})),
) === domainKeys(app.domains);
const usedRawConfig =
domainsUnchanged &&
(await writeRawTraefikConfig(
oldAppName,
newAppName,
appNameMap,
serverId,
workDir,
pathsFn,
));
if (usedRawConfig) {
emit(`Wrote raw Traefik config for ${newAppName}`);
return;
@@ -248,6 +303,7 @@ export const postApply = async (
oldAppName,
newAppName,
newId,
(service.domains as { host: string; enabled?: boolean }[]) ?? [],
plan.appNameMap,
workDir,
pathsFn,
@@ -278,6 +334,9 @@ export const postApply = async (
}
}
postApplyErrors.push(
...(await createImportedNetworks(snapshot, plan, result, emit)),
);
await writeCertificateFiles(db, snapshot, plan, result, emit);
await writeFileMounts(db, snapshot, result, emit);
@@ -1,4 +1,4 @@
import { createHash } from "node:crypto";
import { createHash, X509Certificate } from "node:crypto";
import { VAULT_REF_REGEX } from "@dokploy/server/utils/vault";
import type { ProjectTree, TransferDb } from "./queries";
import {
@@ -21,7 +21,7 @@ export const detectRequirements = async (
detectDestinationRequirements(db, services, scope),
detectRegistryRequirements(db, services, scope),
detectNetworkRequirements(db, services, scope),
detectCertificateRequirements(db, services, scope),
detectCertificateRequirements(db, tree, services, scope),
]);
return [
@@ -403,11 +403,15 @@ const detectNetworkRequirements = async (
const detectCertificateRequirements = async (
db: TransferDb,
tree: ProjectTree,
services: TaggedService[],
scope: ExportScope,
): Promise<Requirement[]> => {
const usedByPerResolver = new Map<string, UsedBy[]>();
const httpsDomains: {
host: string;
serverId: string | null;
usedBy: UsedBy;
}[] = [];
for (const { serviceType, environmentId, service } of services) {
const domains =
(service.domains as Record<string, unknown>[] | undefined) ?? [];
@@ -415,45 +419,59 @@ const detectCertificateRequirements = async (
serviceType === "application" ? "applicationId" : `${serviceType}Id`
] as string;
for (const domain of domains) {
if (domain.certificateType !== "custom") {
continue;
}
const resolver = domain.customCertResolver as string | null;
if (!resolver) {
continue;
}
const list = usedByPerResolver.get(resolver) ?? [];
list.push(
buildUsedBy(
// Uploaded certificates are picked by Traefik through SNI, so these domains use certificateType "none".
if (!domain.https || domain.certificateType !== "none") continue;
httpsDomains.push({
host: domain.host as string,
serverId: (service.serverId as string | null) ?? null,
usedBy: buildUsedBy(
environmentId,
serviceType,
service,
serviceId,
`domain:${domain.domainId}`,
),
);
usedByPerResolver.set(resolver, list);
});
}
}
if (usedByPerResolver.size === 0) {
if (httpsDomains.length === 0) {
return [];
}
const resolvers = [...usedByPerResolver.keys()];
const rows = await db.query.certificates.findMany({
where: (fields, { inArray }) => inArray(fields.certificatePath, resolvers),
where: (fields, { eq }) => eq(fields.organizationId, tree.organizationId),
});
const configIncluded = scope.shared.certificate === "value";
return rows.map((row) => ({
kind: "certificate" as const,
originalId: row.certificateId,
fingerprint: createHash("sha256").update(row.certificateData).digest("hex"),
configIncluded,
usedBy: usedByPerResolver.get(row.certificatePath) ?? [],
serverId: row.serverId ?? null,
}));
return rows.flatMap((row) => {
let certificate: X509Certificate;
try {
certificate = new X509Certificate(row.certificateData);
} catch {
return [];
}
const usedBy = httpsDomains
.filter(
(domain) =>
domain.serverId === (row.serverId ?? null) &&
certificate.checkHost(domain.host) !== undefined,
)
.map((domain) => domain.usedBy);
if (usedBy.length === 0) return [];
return [
{
kind: "certificate" as const,
originalId: row.certificateId,
fingerprint: createHash("sha256")
.update(row.certificateData)
.digest("hex"),
configIncluded,
usedBy,
serverId: row.serverId ?? null,
},
];
});
};
const detectServerRequirements = (services: TaggedService[]): Requirement[] => {
@@ -142,7 +142,13 @@ export const buildRotationPlan = async (
if (!row) continue;
const oldPassword = row.databasePassword as string | undefined;
if (oldPassword && !isSafeToRotate(oldPassword, row)) {
const oldRoot = ROOT_PASSWORD_KINDS.includes(kind)
? (row.databaseRootPassword as string | undefined)
: undefined;
if (
(oldPassword && !isSafeToRotate(oldPassword, row)) ||
(oldRoot && !isSafeToRotate(oldRoot, row))
) {
unsafe.push(usedByOf(service, newId));
continue;
}
@@ -156,21 +162,17 @@ export const buildRotationPlan = async (
newPassword,
});
}
if (ROOT_PASSWORD_KINDS.includes(kind)) {
const oldRoot = row.databaseRootPassword as string | undefined;
if (oldRoot) {
const rootNewPassword =
if (oldRoot) {
specs.push({
serviceId: newId,
serviceType: kind,
field: "databaseRootPassword",
oldPassword: oldRoot,
newPassword:
oldRoot === oldPassword && newPassword
? newPassword
: generateRotatedPassword();
specs.push({
serviceId: newId,
serviceType: kind,
field: "databaseRootPassword",
oldPassword: oldRoot,
newPassword: rootNewPassword,
});
}
: generateRotatedPassword(),
});
}
}