From 5624dff5c1d4ea72f80cee649c93634212673219 Mon Sep 17 00:00:00 2001 From: Mauricio Siu Date: Tue, 1 Sep 2026 01:56:58 -0600 Subject: [PATCH] fix: verify swarm task convergence before marking db deploys done --- packages/server/src/services/libsql.ts | 6 ++- packages/server/src/services/mariadb.ts | 6 ++- packages/server/src/services/mongo.ts | 6 ++- packages/server/src/services/mysql.ts | 6 ++- packages/server/src/services/postgres.ts | 7 +++- packages/server/src/services/redis.ts | 6 ++- packages/server/src/utils/docker/utils.ts | 51 +++++++++++++++++++++++ 7 files changed, 82 insertions(+), 6 deletions(-) diff --git a/packages/server/src/services/libsql.ts b/packages/server/src/services/libsql.ts index 5a362789a..f250727bb 100644 --- a/packages/server/src/services/libsql.ts +++ b/packages/server/src/services/libsql.ts @@ -7,7 +7,10 @@ import { } from "@dokploy/server/db/schema"; import { generatePassword } from "@dokploy/server/templates"; import { buildLibsql } from "@dokploy/server/utils/databases/libsql"; -import { pullImage } from "@dokploy/server/utils/docker/utils"; +import { + pullImage, + waitForSwarmServiceConvergence, +} from "@dokploy/server/utils/docker/utils"; import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { eq, getTableColumns } from "drizzle-orm"; @@ -149,6 +152,7 @@ export const deployLibsql = async ( } await buildLibsql(libsql); + await waitForSwarmServiceConvergence(libsql.appName, libsql.serverId); await updateLibsqlById(libsqlId, { applicationStatus: "done", }); diff --git a/packages/server/src/services/mariadb.ts b/packages/server/src/services/mariadb.ts index 54ffb34ba..4c6fdade9 100644 --- a/packages/server/src/services/mariadb.ts +++ b/packages/server/src/services/mariadb.ts @@ -7,7 +7,10 @@ import { } from "@dokploy/server/db/schema"; import { generatePassword } from "@dokploy/server/templates"; import { buildMariadb } from "@dokploy/server/utils/databases/mariadb"; -import { pullImage } from "@dokploy/server/utils/docker/utils"; +import { + pullImage, + waitForSwarmServiceConvergence, +} from "@dokploy/server/utils/docker/utils"; import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { eq, getTableColumns } from "drizzle-orm"; @@ -154,6 +157,7 @@ export const deployMariadb = async ( } await buildMariadb(mariadb); + await waitForSwarmServiceConvergence(mariadb.appName, mariadb.serverId); await updateMariadbById(mariadbId, { applicationStatus: "done", }); diff --git a/packages/server/src/services/mongo.ts b/packages/server/src/services/mongo.ts index c6ff19f1e..3da007c2f 100644 --- a/packages/server/src/services/mongo.ts +++ b/packages/server/src/services/mongo.ts @@ -8,7 +8,10 @@ import { } from "@dokploy/server/db/schema"; import { generatePassword } from "@dokploy/server/templates"; import { buildMongo } from "@dokploy/server/utils/databases/mongo"; -import { pullImage } from "@dokploy/server/utils/docker/utils"; +import { + pullImage, + waitForSwarmServiceConvergence, +} from "@dokploy/server/utils/docker/utils"; import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { eq, getTableColumns } from "drizzle-orm"; @@ -169,6 +172,7 @@ export const deployMongo = async ( } await buildMongo(mongo); + await waitForSwarmServiceConvergence(mongo.appName, mongo.serverId); await updateMongoById(mongoId, { applicationStatus: "done", }); diff --git a/packages/server/src/services/mysql.ts b/packages/server/src/services/mysql.ts index 288c6fc28..c2df01167 100644 --- a/packages/server/src/services/mysql.ts +++ b/packages/server/src/services/mysql.ts @@ -7,7 +7,10 @@ import { } from "@dokploy/server/db/schema"; import { generatePassword } from "@dokploy/server/templates"; import { buildMysql } from "@dokploy/server/utils/databases/mysql"; -import { pullImage } from "@dokploy/server/utils/docker/utils"; +import { + pullImage, + waitForSwarmServiceConvergence, +} from "@dokploy/server/utils/docker/utils"; import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { eq, getTableColumns } from "drizzle-orm"; @@ -152,6 +155,7 @@ export const deployMySql = async ( } await buildMysql(mysql); + await waitForSwarmServiceConvergence(mysql.appName, mysql.serverId); await updateMySqlById(mysqlId, { applicationStatus: "done", }); diff --git a/packages/server/src/services/postgres.ts b/packages/server/src/services/postgres.ts index 3a9754125..2d3715c46 100644 --- a/packages/server/src/services/postgres.ts +++ b/packages/server/src/services/postgres.ts @@ -7,7 +7,10 @@ import { } from "@dokploy/server/db/schema"; import { generatePassword } from "@dokploy/server/templates"; import { buildPostgres } from "@dokploy/server/utils/databases/postgres"; -import { pullImage } from "@dokploy/server/utils/docker/utils"; +import { + pullImage, + waitForSwarmServiceConvergence, +} from "@dokploy/server/utils/docker/utils"; import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { eq, getTableColumns } from "drizzle-orm"; @@ -165,6 +168,8 @@ export const deployPostgres = async ( await buildPostgres(postgres); + await waitForSwarmServiceConvergence(postgres.appName, postgres.serverId); + await updatePostgresById(postgresId, { applicationStatus: "done", }); diff --git a/packages/server/src/services/redis.ts b/packages/server/src/services/redis.ts index da2822b13..d757f7111 100644 --- a/packages/server/src/services/redis.ts +++ b/packages/server/src/services/redis.ts @@ -6,7 +6,10 @@ import { } from "@dokploy/server/db/schema"; import { generatePassword } from "@dokploy/server/templates"; import { buildRedis } from "@dokploy/server/utils/databases/redis"; -import { pullImage } from "@dokploy/server/utils/docker/utils"; +import { + pullImage, + waitForSwarmServiceConvergence, +} from "@dokploy/server/utils/docker/utils"; import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; @@ -119,6 +122,7 @@ export const deployRedis = async ( } await buildRedis(redis); + await waitForSwarmServiceConvergence(redis.appName, redis.serverId); await updateRedisById(redisId, { applicationStatus: "done", }); diff --git a/packages/server/src/utils/docker/utils.ts b/packages/server/src/utils/docker/utils.ts index e19d7d2fd..e342eb450 100644 --- a/packages/server/src/utils/docker/utils.ts +++ b/packages/server/src/utils/docker/utils.ts @@ -934,6 +934,57 @@ const getSwarmServiceContainerId = async ( } }; +export class ServiceConvergenceError extends Error {} + +export const waitForSwarmServiceConvergence = async ( + appName: string, + serverId?: string | null, + options?: { timeoutMs?: number; intervalMs?: number }, +): Promise => { + const timeoutMs = options?.timeoutMs ?? 45_000; + const intervalMs = options?.intervalMs ?? 2_000; + const remoteDocker = await getRemoteDocker(serverId); + const service = remoteDocker.getService(appName); + const deadline = Date.now() + timeoutMs; + + let lastState = "unknown"; + while (true) { + const info = await service.inspect(); + const desiredTasksCount = info.Spec?.Mode?.Replicated?.Replicas ?? 1; + + const tasks = await remoteDocker.listTasks({ + filters: JSON.stringify({ service: [appName] }), + }); + const currentTasks = tasks.filter( + (task) => task.DesiredState === "running", + ); + const runningTasksCount = currentTasks.filter( + (task) => task.Status?.State === "running", + ).length; + + if (runningTasksCount >= desiredTasksCount) { + return; + } + + const failedTask = currentTasks.find((task) => + ["failed", "rejected"].includes(task.Status?.State ?? ""), + ); + lastState = + failedTask?.Status?.Err ?? + failedTask?.Status?.State ?? + currentTasks[0]?.Status?.State ?? + lastState; + + if (Date.now() >= deadline) { + throw new ServiceConvergenceError( + `Service ${appName} did not converge within ${timeoutMs}ms: ${runningTasksCount}/${desiredTasksCount} tasks running (last state: ${lastState})`, + ); + } + + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } +}; + export const checkPostgresHealth = async (): Promise => { const serviceCheck = await checkSwarmServiceRunning("dokploy-postgres"); if (serviceCheck.status === "unhealthy") {