From 2598a6277153e32638b4bf31e365f9b5f1e833b2 Mon Sep 17 00:00:00 2001 From: Aditya Nandlal <73009776+bestmaa@users.noreply.github.com> Date: Fri, 14 Aug 2026 11:40:21 +0000 Subject: [PATCH 1/2] fix(volume-backups): restart services after backup failure --- .../backups/volume-backup-restart.test.ts | 49 ++++++++++++++ .../server/src/utils/volume-backups/backup.ts | 65 ++++++++++++++----- 2 files changed, 98 insertions(+), 16 deletions(-) create mode 100644 apps/dokploy/__test__/backups/volume-backup-restart.test.ts diff --git a/apps/dokploy/__test__/backups/volume-backup-restart.test.ts b/apps/dokploy/__test__/backups/volume-backup-restart.test.ts new file mode 100644 index 000000000..26b59ed2d --- /dev/null +++ b/apps/dokploy/__test__/backups/volume-backup-restart.test.ts @@ -0,0 +1,49 @@ +import { spawnSync } from "node:child_process"; +import { createRestartSafeBackupCommand } from "@dokploy/server/utils/volume-backups/backup"; +import { describe, expect, it } from "vitest"; + +const runCommand = (command: string) => + spawnSync("bash", ["-c", command], { + encoding: "utf8", + }); + +const outputLines = (stdout: string) => + stdout + .trim() + .split("\n") + .filter((line) => line.length > 0); + +describe("createRestartSafeBackupCommand", () => { + it("restarts the service and preserves the backup error status", () => { + const result = runCommand( + createRestartSafeBackupCommand({ + stopCommand: 'echo "stop"', + backupCommand: 'echo "backup"; exit 23', + startCommand: 'echo "start"', + uploadCommand: 'echo "upload"', + }), + ); + + expect(result.status).toBe(23); + expect(outputLines(result.stdout)).toEqual(["stop", "backup", "start"]); + }); + + it("uploads only after a successful backup and service restart", () => { + const result = runCommand( + createRestartSafeBackupCommand({ + stopCommand: 'echo "stop"', + backupCommand: 'echo "backup"', + startCommand: 'echo "start"', + uploadCommand: 'echo "upload"', + }), + ); + + expect(result.status).toBe(0); + expect(outputLines(result.stdout)).toEqual([ + "stop", + "backup", + "start", + "upload", + ]); + }); +}); diff --git a/packages/server/src/utils/volume-backups/backup.ts b/packages/server/src/utils/volume-backups/backup.ts index a9240f2dd..deb4f0c23 100644 --- a/packages/server/src/utils/volume-backups/backup.ts +++ b/packages/server/src/utils/volume-backups/backup.ts @@ -9,6 +9,33 @@ import { normalizeS3Path, } from "../backups/utils"; +interface RestartSafeBackupCommandOptions { + stopCommand: string; + backupCommand: string; + startCommand: string; + uploadCommand: string; +} + +export const createRestartSafeBackupCommand = ({ + stopCommand, + backupCommand, + startCommand, + uploadCommand, +}: RestartSafeBackupCommandOptions) => ` + ${stopCommand} + set +e + ( + ${backupCommand} + ) + DOKPLOY_VOLUME_BACKUP_STATUS=$? + set -e + ${startCommand} + if [ "$DOKPLOY_VOLUME_BACKUP_STATUS" -ne 0 ]; then + exit "$DOKPLOY_VOLUME_BACKUP_STATUS" + fi + ${uploadCommand} +`; + export const getVolumeServiceAppName = ( volumeBackup: Awaited>, ): string => { @@ -117,16 +144,20 @@ export const backupVolume = async ( ); if (serviceType === "application") { - return lockWrapper(` - echo "Stopping application to 0 replicas" - ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}") - echo "Actual replicas: $ACTUAL_REPLICAS" - docker service update --replicas=0 ${volumeBackup.application?.appName} - ${backupCommand} - echo "Starting application to $ACTUAL_REPLICAS replicas" - docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName} - ${uploadCommand} - `); + return lockWrapper( + createRestartSafeBackupCommand({ + stopCommand: ` + echo "Stopping application to 0 replicas" + ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}") + echo "Actual replicas: $ACTUAL_REPLICAS" + docker service update --replicas=0 ${volumeBackup.application?.appName}`, + backupCommand, + startCommand: ` + echo "Starting application to $ACTUAL_REPLICAS replicas" + docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName}`, + uploadCommand, + }), + ); } if (serviceType === "compose") { const compose = await findComposeById( @@ -158,11 +189,13 @@ export const backupVolume = async ( echo "Compose container started" `; } - return lockWrapper(` - ${stopCommand} - ${backupCommand} - ${startCommand} - ${uploadCommand} - `); + return lockWrapper( + createRestartSafeBackupCommand({ + stopCommand, + backupCommand, + startCommand, + uploadCommand, + }), + ); } }; From c6ab76fe09093dfacd6d11ddde36e3b7f77719ae Mon Sep 17 00:00:00 2001 From: Aditya Nandlal <73009776+bestmaa@users.noreply.github.com> Date: Fri, 14 Aug 2026 12:51:28 +0000 Subject: [PATCH 2/2] fix(volume-backups): preserve backup failure status --- .../backups/volume-backup-restart.test.ts | 33 +++++++++++++++++++ .../server/src/utils/volume-backups/backup.ts | 12 ++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/apps/dokploy/__test__/backups/volume-backup-restart.test.ts b/apps/dokploy/__test__/backups/volume-backup-restart.test.ts index 26b59ed2d..ba2fc081e 100644 --- a/apps/dokploy/__test__/backups/volume-backup-restart.test.ts +++ b/apps/dokploy/__test__/backups/volume-backup-restart.test.ts @@ -28,6 +28,39 @@ describe("createRestartSafeBackupCommand", () => { expect(outputLines(result.stdout)).toEqual(["stop", "backup", "start"]); }); + it("preserves the backup error status when the restart also fails", () => { + const result = runCommand( + createRestartSafeBackupCommand({ + stopCommand: 'echo "stop"', + backupCommand: 'echo "backup"; exit 23', + startCommand: 'echo "start"; exit 17', + uploadCommand: 'echo "upload"', + }), + ); + + expect(result.status).toBe(23); + expect(outputLines(result.stdout)).toEqual([ + "stop", + "backup", + "start", + "Service restart also failed with exit code 17", + ]); + }); + + it("returns the restart error status when the backup succeeds", () => { + const result = runCommand( + createRestartSafeBackupCommand({ + stopCommand: 'echo "stop"', + backupCommand: 'echo "backup"', + startCommand: 'echo "start"; exit 17', + uploadCommand: 'echo "upload"', + }), + ); + + expect(result.status).toBe(17); + expect(outputLines(result.stdout)).toEqual(["stop", "backup", "start"]); + }); + it("uploads only after a successful backup and service restart", () => { const result = runCommand( createRestartSafeBackupCommand({ diff --git a/packages/server/src/utils/volume-backups/backup.ts b/packages/server/src/utils/volume-backups/backup.ts index deb4f0c23..3172c9c21 100644 --- a/packages/server/src/utils/volume-backups/backup.ts +++ b/packages/server/src/utils/volume-backups/backup.ts @@ -28,11 +28,21 @@ export const createRestartSafeBackupCommand = ({ ${backupCommand} ) DOKPLOY_VOLUME_BACKUP_STATUS=$? + ( + set -e + ${startCommand} + ) + DOKPLOY_VOLUME_RESTART_STATUS=$? set -e - ${startCommand} if [ "$DOKPLOY_VOLUME_BACKUP_STATUS" -ne 0 ]; then + if [ "$DOKPLOY_VOLUME_RESTART_STATUS" -ne 0 ]; then + echo "Service restart also failed with exit code $DOKPLOY_VOLUME_RESTART_STATUS" + fi exit "$DOKPLOY_VOLUME_BACKUP_STATUS" fi + if [ "$DOKPLOY_VOLUME_RESTART_STATUS" -ne 0 ]; then + exit "$DOKPLOY_VOLUME_RESTART_STATUS" + fi ${uploadCommand} `;