diff --git a/packages/server/src/db/schema/audit-log.ts b/packages/server/src/db/schema/audit-log.ts index a1cec1910..60ea01d20 100644 --- a/packages/server/src/db/schema/audit-log.ts +++ b/packages/server/src/db/schema/audit-log.ts @@ -94,4 +94,5 @@ export type AuditResourceType = | "compose" | "network" | "vaultProvider" - | "dnsProvider"; + | "dnsProvider" + | "sandbox"; diff --git a/packages/server/src/db/schema/sandbox.ts b/packages/server/src/db/schema/sandbox.ts index 0090750f4..addb5dc73 100644 --- a/packages/server/src/db/schema/sandbox.ts +++ b/packages/server/src/db/schema/sandbox.ts @@ -55,12 +55,8 @@ export const sandboxes = pgTable("sandbox", { status: sandboxStatus("status").notNull().default("creating"), cpu: real("cpu").notNull().default(SANDBOX_DEFAULTS.cpu), memoryMb: integer("memoryMb").notNull().default(SANDBOX_DEFAULTS.memoryMb), - pidsLimit: integer("pidsLimit") - .notNull() - .default(SANDBOX_DEFAULTS.pidsLimit), - timeoutMs: integer("timeoutMs") - .notNull() - .default(SANDBOX_DEFAULTS.timeoutMs), + pidsLimit: integer("pidsLimit").notNull().default(SANDBOX_DEFAULTS.pidsLimit), + timeoutMs: integer("timeoutMs").notNull().default(SANDBOX_DEFAULTS.timeoutMs), expiresAt: timestamp("expiresAt", { withTimezone: true }), lastActivityAt: timestamp("lastActivityAt", { withTimezone: true }), networkMode: sandboxNetworkMode("networkMode").notNull().default("isolated"), @@ -137,7 +133,9 @@ export const apiExecSandbox = z.object({ sandboxId: z.string().min(1), cmd: z.string().min(1).max(65_536), cwd: absolutePath.optional(), - env: z.record(z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/), z.string()).optional(), + env: z + .record(z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/), z.string()) + .optional(), timeoutMs: z.number().int().min(1000).max(600_000).optional(), }); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index cd7bef41f..2c3490007 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -48,6 +48,7 @@ export * from "./services/redirect"; export * from "./services/redis"; export * from "./services/registry"; export * from "./services/rollbacks"; +export * from "./services/sandbox"; export * from "./services/schedule"; export * from "./services/security"; export * from "./services/server"; @@ -131,6 +132,12 @@ export * from "./utils/providers/github"; export * from "./utils/providers/github"; export * from "./utils/providers/gitlab"; export * from "./utils/providers/raw"; +export * from "./utils/sandbox/container"; +export * from "./utils/sandbox/exec"; +export * from "./utils/sandbox/files"; +export * from "./utils/sandbox/reaper"; +export * from "./utils/sandbox/tar"; +export * from "./utils/sandbox/templates"; export * from "./utils/schedules/index"; export * from "./utils/schedules/utils"; export * from "./utils/servers/remote-docker"; diff --git a/packages/server/src/services/sandbox.ts b/packages/server/src/services/sandbox.ts new file mode 100644 index 000000000..89484ddae --- /dev/null +++ b/packages/server/src/services/sandbox.ts @@ -0,0 +1,485 @@ +import { posix } from "node:path"; +import { db } from "@dokploy/server/db"; +import { + type apiCreateSandbox, + environments, + SANDBOX_DEFAULTS, + sandboxes, +} from "@dokploy/server/db/schema"; +import { getRemoteDocker } from "@dokploy/server/utils/servers/remote-docker"; +import { TRPCError } from "@trpc/server"; +import type Dockerode from "dockerode"; +import { desc, eq, inArray } from "drizzle-orm"; +import { quote } from "shell-quote"; +import type { z } from "zod"; +import { + buildSandboxContainerOptions, + getSandboxNetworkName, + parseSandboxUser, + SANDBOX_LABEL, + type SandboxNetworkMode, +} from "../utils/sandbox/container"; +import { runSandboxExec, type SandboxExecResult } from "../utils/sandbox/exec"; +import { + buildSandboxFindCommand, + buildSandboxLsCommand, + parseSandboxFindOutput, + parseSandboxLsOutput, +} from "../utils/sandbox/files"; +import { + buildSandboxDirectoryTar, + buildSandboxFileTar, + extractSandboxFile, + SandboxFileTooLargeError, + SandboxNotRegularFileError, +} from "../utils/sandbox/tar"; +import { SANDBOX_TEMPLATES } from "../utils/sandbox/templates"; + +export type Sandbox = typeof sandboxes.$inferSelect; + +export const SANDBOX_MAX_FILE_BYTES = 10 * 1024 * 1024; +const INTERNAL_EXEC_TIMEOUT_MS = 15_000; + +const statusCodeOf = (error: unknown) => + (error as { statusCode?: number } | null)?.statusCode; + +const isNotFound = (error: unknown) => statusCodeOf(error) === 404; + +export const findSandboxById = async (sandboxId: string) => { + const result = await db.query.sandboxes.findFirst({ + where: eq(sandboxes.sandboxId, sandboxId), + with: { + environment: { with: { project: true } }, + server: { columns: { serverId: true, name: true, serverStatus: true } }, + }, + }); + if (!result) { + throw new TRPCError({ code: "NOT_FOUND", message: "Sandbox not found" }); + } + return result; +}; + +export const findSandboxesByEnvironmentId = (environmentId: string) => + db.query.sandboxes.findMany({ + where: eq(sandboxes.environmentId, environmentId), + orderBy: desc(sandboxes.createdAt), + with: { server: { columns: { serverId: true, name: true } } }, + }); + +export const findSandboxesByProjectId = async (projectId: string) => { + const projectEnvironments = await db.query.environments.findMany({ + where: eq(environments.projectId, projectId), + columns: { environmentId: true }, + }); + if (projectEnvironments.length === 0) return []; + return db.query.sandboxes.findMany({ + where: inArray( + sandboxes.environmentId, + projectEnvironments.map((env) => env.environmentId), + ), + orderBy: desc(sandboxes.createdAt), + with: { server: { columns: { serverId: true, name: true } } }, + }); +}; + +export const updateSandboxById = async ( + sandboxId: string, + data: Partial, +) => { + const [result] = await db + .update(sandboxes) + .set(data) + .where(eq(sandboxes.sandboxId, sandboxId)) + .returning(); + return result; +}; + +export const removeSandboxById = async (sandboxId: string) => { + const [result] = await db + .delete(sandboxes) + .where(eq(sandboxes.sandboxId, sandboxId)) + .returning(); + return result; +}; + +export const ensureSandboxImage = async (docker: Dockerode, image: string) => { + try { + await docker.getImage(image).inspect(); + return; + } catch (error) { + if (!isNotFound(error)) throw error; + } + await new Promise((resolve, reject) => { + docker.pull(image, {}, (err, stream) => { + if (err) { + reject(err); + return; + } + docker.modem.followProgress(stream, (progressError: Error | null) => { + if (progressError) reject(progressError); + else resolve(); + }); + }); + }); +}; + +export const ensureSandboxNetwork = async ( + docker: Dockerode, + mode: SandboxNetworkMode, +) => { + const name = getSandboxNetworkName(mode); + try { + await docker.getNetwork(name).inspect(); + return name; + } catch (error) { + if (!isNotFound(error)) throw error; + } + try { + await docker.createNetwork({ + Name: name, + Driver: "bridge", + Internal: mode === "isolated", + CheckDuplicate: true, + Labels: { [SANDBOX_LABEL]: "true" }, + }); + } catch (error) { + // Another create raced us; the network exists now. + if (statusCodeOf(error) !== 409) throw error; + } + return name; +}; + +const resolveSandboxUser = async ( + docker: Dockerode, + containerId: string, + fallback: string | null, +) => { + try { + const result = await runSandboxExec(docker, containerId, { + cmd: "id -u && id -g", + timeoutMs: INTERNAL_EXEC_TIMEOUT_MS, + }); + const [uid, gid] = result.stdout.trim().split(/\s+/); + if (result.exitCode === 0 && uid && gid) return `${uid}:${gid}`; + } catch {} + return fallback; +}; + +const prepareSandboxWorkdir = async ( + docker: Dockerode, + containerId: string, + workdir: string, + user: string | null, +) => { + const parent = posix.dirname(workdir); + const name = posix.basename(workdir); + if (!name) return; + if (parent !== "/") { + await runSandboxExec(docker, containerId, { + cmd: `mkdir -p ${quote([parent])}`, + user: "root", + timeoutMs: INTERNAL_EXEC_TIMEOUT_MS, + }).catch(() => {}); + } + const ids = parseSandboxUser(user) ?? { uid: 0, gid: 0 }; + // putArchive runs in the daemon, so it can create the directory owned by the + // sandbox user even though every capability (CAP_CHOWN included) is dropped. + const tar = await buildSandboxDirectoryTar({ name, ...ids }); + await docker.getContainer(containerId).putArchive(tar, { path: parent }); +}; + +const startSandboxContainer = async (sandbox: Sandbox, projectId: string) => { + const docker = await getRemoteDocker(sandbox.serverId); + await ensureSandboxImage(docker, sandbox.image); + await ensureSandboxNetwork(docker, sandbox.networkMode); + const container = await docker.createContainer( + buildSandboxContainerOptions({ ...sandbox, projectId }), + ); + try { + await container.start(); + const user = await resolveSandboxUser(docker, container.id, sandbox.user); + await prepareSandboxWorkdir(docker, container.id, sandbox.workdir, user); + const now = new Date(); + return await updateSandboxById(sandbox.sandboxId, { + containerId: container.id, + user, + status: "running", + expiresAt: new Date(now.getTime() + sandbox.timeoutMs), + lastActivityAt: now, + }); + } catch (error) { + await container.remove({ force: true }).catch(() => {}); + throw error; + } +}; + +export const createSandbox = async ( + input: z.infer, +) => { + const environment = await db.query.environments.findFirst({ + where: eq(environments.environmentId, input.environmentId), + columns: { environmentId: true, projectId: true }, + }); + if (!environment) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Environment not found", + }); + } + + const template = input.template ? SANDBOX_TEMPLATES[input.template] : null; + const image = input.image ?? template?.image; + if (!image) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Either template or image is required", + }); + } + + const [sandbox] = await db + .insert(sandboxes) + .values({ + name: input.name ?? `sandbox-${Date.now().toString(36)}`, + environmentId: input.environmentId, + serverId: input.serverId ?? null, + image, + template: input.template ?? null, + cpu: input.cpu ?? SANDBOX_DEFAULTS.cpu, + memoryMb: input.memoryMb ?? SANDBOX_DEFAULTS.memoryMb, + pidsLimit: input.pidsLimit ?? SANDBOX_DEFAULTS.pidsLimit, + timeoutMs: input.timeoutMs ?? SANDBOX_DEFAULTS.timeoutMs, + networkMode: input.networkMode ?? "isolated", + envVars: input.envVars ?? null, + workdir: input.workdir ?? template?.workdir ?? SANDBOX_DEFAULTS.workdir, + user: template?.user ?? null, + status: "creating", + }) + .returning(); + if (!sandbox) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Error inserting sandbox", + }); + } + + try { + const running = await startSandboxContainer(sandbox, environment.projectId); + if (!running) throw new Error("Sandbox row disappeared during creation"); + return running; + } catch (error) { + await updateSandboxById(sandbox.sandboxId, { status: "error" }); + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: `Error creating sandbox: ${error instanceof Error ? error.message : error}`, + cause: error, + }); + } +}; + +type RunningSandbox = Sandbox & { containerId: string }; + +export const assertSandboxRunning: ( + sandbox: Sandbox, +) => asserts sandbox is RunningSandbox = (sandbox) => { + if (sandbox.status !== "running" || !sandbox.containerId) { + throw new TRPCError({ + code: "PRECONDITION_FAILED", + message: `Sandbox is ${sandbox.status}`, + }); + } +}; + +export const touchSandbox = async (sandbox: Sandbox) => { + const now = new Date(); + return updateSandboxById(sandbox.sandboxId, { + lastActivityAt: now, + expiresAt: new Date(now.getTime() + sandbox.timeoutMs), + }); +}; + +export interface ExecInSandboxOptions { + cmd: string; + cwd?: string; + env?: Record; + timeoutMs?: number; + onStdout?: (chunk: string) => void; + onStderr?: (chunk: string) => void; +} + +export const execInSandbox = async ( + sandbox: Sandbox, + options: ExecInSandboxOptions, +): Promise => { + assertSandboxRunning(sandbox); + await touchSandbox(sandbox); + const docker = await getRemoteDocker(sandbox.serverId); + const result = await runSandboxExec(docker, sandbox.containerId, { + cmd: options.cmd, + cwd: options.cwd ?? sandbox.workdir, + env: options.env, + timeoutMs: options.timeoutMs ?? SANDBOX_DEFAULTS.execTimeoutMs, + onStdout: options.onStdout, + onStderr: options.onStderr, + }); + if (result.containerKilled) { + await updateSandboxById(sandbox.sandboxId, { status: "error" }); + } + return result; +}; + +export const writeSandboxFile = async ( + sandbox: Sandbox, + options: { path: string; content: Buffer; mode?: number }, +) => { + assertSandboxRunning(sandbox); + if (options.content.length > SANDBOX_MAX_FILE_BYTES) { + throw new TRPCError({ + code: "PAYLOAD_TOO_LARGE", + message: `File exceeds the ${SANDBOX_MAX_FILE_BYTES} byte limit`, + }); + } + const dir = posix.dirname(options.path); + const name = posix.basename(options.path); + if (!name) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Path must point to a file", + }); + } + await touchSandbox(sandbox); + const docker = await getRemoteDocker(sandbox.serverId); + const mkdir = await runSandboxExec(docker, sandbox.containerId, { + cmd: `mkdir -p ${quote([dir])}`, + timeoutMs: INTERNAL_EXEC_TIMEOUT_MS, + }); + if (mkdir.exitCode !== 0) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: mkdir.stderr.trim() || `Unable to create directory ${dir}`, + }); + } + const ids = parseSandboxUser(sandbox.user) ?? { uid: 0, gid: 0 }; + const tar = await buildSandboxFileTar({ + name, + content: options.content, + mode: options.mode, + ...ids, + }); + await docker.getContainer(sandbox.containerId).putArchive(tar, { path: dir }); + return { path: options.path, size: options.content.length }; +}; + +export const readSandboxFile = async (sandbox: Sandbox, path: string) => { + assertSandboxRunning(sandbox); + await touchSandbox(sandbox); + const docker = await getRemoteDocker(sandbox.serverId); + let archive: NodeJS.ReadableStream; + try { + archive = await docker + .getContainer(sandbox.containerId) + .getArchive({ path }); + } catch (error) { + if (isNotFound(error)) { + throw new TRPCError({ code: "NOT_FOUND", message: "File not found" }); + } + throw error; + } + try { + const { content } = await extractSandboxFile( + archive, + SANDBOX_MAX_FILE_BYTES, + ); + return { path, size: content.length, content }; + } catch (error) { + if (error instanceof SandboxFileTooLargeError) { + throw new TRPCError({ + code: "PAYLOAD_TOO_LARGE", + message: error.message, + }); + } + if (error instanceof SandboxNotRegularFileError) { + throw new TRPCError({ code: "BAD_REQUEST", message: error.message }); + } + throw error; + } +}; + +export const listSandboxFiles = async (sandbox: Sandbox, path?: string) => { + assertSandboxRunning(sandbox); + await touchSandbox(sandbox); + const target = path ?? sandbox.workdir; + const docker = await getRemoteDocker(sandbox.serverId); + const find = await runSandboxExec(docker, sandbox.containerId, { + cmd: buildSandboxFindCommand(target), + timeoutMs: INTERNAL_EXEC_TIMEOUT_MS, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (find.exitCode === 0) { + return { path: target, entries: parseSandboxFindOutput(find.stdout) }; + } + // busybox find has no -printf; fall back to a name-only listing. + const ls = await runSandboxExec(docker, sandbox.containerId, { + cmd: buildSandboxLsCommand(target), + timeoutMs: INTERNAL_EXEC_TIMEOUT_MS, + maxOutputBytes: 4 * 1024 * 1024, + }); + if (ls.exitCode === 0) { + return { path: target, entries: parseSandboxLsOutput(ls.stdout) }; + } + throw new TRPCError({ + code: "BAD_REQUEST", + message: (ls.stderr || find.stderr).trim() || "Unable to list files", + }); +}; + +export const setSandboxTimeout = async ( + sandbox: Sandbox, + timeoutMs: number, +) => { + assertSandboxRunning(sandbox); + const now = new Date(); + return updateSandboxById(sandbox.sandboxId, { + timeoutMs, + lastActivityAt: now, + expiresAt: new Date(now.getTime() + timeoutMs), + }); +}; + +export const removeSandboxContainer = async ( + serverId: string | null, + containerId: string, +) => { + const docker = await getRemoteDocker(serverId); + const container = docker.getContainer(containerId); + await container.stop({ t: 2 }).catch((error: unknown) => { + // 304 = already stopped, 404 = already gone + if (statusCodeOf(error) !== 304 && !isNotFound(error)) throw error; + }); + await container.remove({ force: true }).catch((error: unknown) => { + if (!isNotFound(error)) throw error; + }); +}; + +export const killSandbox = async (sandboxId: string) => { + const sandbox = await findSandboxById(sandboxId); + if (sandbox.status === "killed") return sandbox; + if (sandbox.containerId) { + try { + await removeSandboxContainer(sandbox.serverId, sandbox.containerId); + } catch (error) { + console.error( + `[Sandbox] Failed to remove container for ${sandboxId}:`, + error instanceof Error ? error.message : error, + ); + } + } + return updateSandboxById(sandboxId, { + status: "killed", + killedAt: new Date(), + }); +}; + +export const removeSandbox = async (sandboxId: string) => { + await killSandbox(sandboxId); + return removeSandboxById(sandboxId); +}; diff --git a/packages/server/src/utils/sandbox/container.ts b/packages/server/src/utils/sandbox/container.ts new file mode 100644 index 000000000..c7da55e5d --- /dev/null +++ b/packages/server/src/utils/sandbox/container.ts @@ -0,0 +1,85 @@ +import type Dockerode from "dockerode"; + +export const SANDBOX_LABEL = "dokploy.sandbox"; +export const SANDBOX_ID_LABEL = "dokploy.sandboxId"; +export const SANDBOX_PROJECT_LABEL = "dokploy.projectId"; +export const SANDBOX_ENVIRONMENT_LABEL = "dokploy.environmentId"; +export const SANDBOX_ISOLATED_NETWORK = "dokploy-sandboxes-isolated"; +export const SANDBOX_INTERNET_NETWORK = "dokploy-sandboxes"; + +export type SandboxNetworkMode = "isolated" | "internet"; + +export interface SandboxContainerSpec { + sandboxId: string; + projectId: string; + environmentId: string; + image: string; + cpu: number; + memoryMb: number; + pidsLimit: number; + networkMode: SandboxNetworkMode; + envVars?: string | null; + workdir: string; + user?: string | null; +} + +export const getSandboxNetworkName = (mode: SandboxNetworkMode) => + mode === "internet" ? SANDBOX_INTERNET_NETWORK : SANDBOX_ISOLATED_NETWORK; + +export const getSandboxContainerName = (sandboxId: string) => + `dokploy-sandbox-${sandboxId}`; + +export const parseSandboxEnvVars = (envVars?: string | null): string[] => { + if (!envVars) return []; + return envVars + .split("\n") + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith("#") && line.includes("=")); +}; + +export const parseSandboxUser = ( + user?: string | null, +): { uid: number; gid: number } | null => { + if (!user) return null; + const [uidPart, gidPart] = user.split(":"); + const uid = Number.parseInt(uidPart ?? "", 10); + const gid = Number.parseInt(gidPart ?? uidPart ?? "", 10); + if (!Number.isInteger(uid) || uid < 0) return null; + return { uid, gid: Number.isInteger(gid) && gid >= 0 ? gid : uid }; +}; + +export const buildSandboxContainerOptions = ( + spec: SandboxContainerSpec, +): Dockerode.ContainerCreateOptions => { + const memory = Math.round(spec.memoryMb * 1024 * 1024); + return { + name: getSandboxContainerName(spec.sandboxId), + Image: spec.image, + // [""] resets the image entrypoint so Cmd always becomes PID 1. + Entrypoint: [""], + Cmd: ["tail", "-f", "/dev/null"], + WorkingDir: spec.workdir, + Env: [`HOME=${spec.workdir}`, ...parseSandboxEnvVars(spec.envVars)], + ...(spec.user ? { User: spec.user } : {}), + Labels: { + [SANDBOX_LABEL]: "true", + [SANDBOX_ID_LABEL]: spec.sandboxId, + [SANDBOX_PROJECT_LABEL]: spec.projectId, + [SANDBOX_ENVIRONMENT_LABEL]: spec.environmentId, + }, + HostConfig: { + NanoCpus: Math.round(spec.cpu * 1e9), + Memory: memory, + MemorySwap: memory, + PidsLimit: spec.pidsLimit, + CapDrop: ["ALL"], + SecurityOpt: ["no-new-privileges"], + RestartPolicy: { Name: "no" }, + LogConfig: { + Type: "json-file", + Config: { "max-size": "10m", "max-file": "1" }, + }, + NetworkMode: getSandboxNetworkName(spec.networkMode), + }, + }; +}; diff --git a/packages/server/src/utils/sandbox/exec.ts b/packages/server/src/utils/sandbox/exec.ts new file mode 100644 index 000000000..24ad2ddb7 --- /dev/null +++ b/packages/server/src/utils/sandbox/exec.ts @@ -0,0 +1,162 @@ +import { Writable } from "node:stream"; +import { StringDecoder } from "node:string_decoder"; +import type Dockerode from "dockerode"; +import { nanoid } from "nanoid"; +import { quote } from "shell-quote"; + +export const SANDBOX_EXEC_TIMEOUT_EXIT_CODE = 124; +export const SANDBOX_EXEC_MARKER_ENV = "DOKPLOY_EXEC_ID"; +export const SANDBOX_EXEC_MAX_OUTPUT_BYTES = 1024 * 1024; +const KILL_GRACE_MS = 2000; + +export interface SandboxExecOptions { + cmd: string; + cwd?: string; + env?: Record | string[]; + user?: string; + timeoutMs: number; + maxOutputBytes?: number; + onStdout?: (chunk: string) => void; + onStderr?: (chunk: string) => void; +} + +export interface SandboxExecResult { + stdout: string; + stderr: string; + exitCode: number; + timedOut: boolean; + truncated: boolean; + containerKilled: boolean; +} + +export const toSandboxEnvArray = (env?: Record | string[]) => + Array.isArray(env) + ? env + : Object.entries(env ?? {}).map(([key, value]) => `${key}=${value}`); + +// Finds every process started by the exec (children included) through the +// marker env var and kills it, without relying on pkill/procps being installed. +export const buildSandboxKillCommand = (marker: string) => + `for p in /proc/[0-9]*; do if tr '\\0' '\\n' < "$p/environ" 2>/dev/null | grep -qxF ${quote([`${SANDBOX_EXEC_MARKER_ENV}=${marker}`])}; then kill -9 "\${p#/proc/}" 2>/dev/null; fi; done; true`; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const createSink = ( + maxBytes: number, + onChunk: ((chunk: string) => void) | undefined, + state: { truncated: boolean }, +) => { + const chunks: Buffer[] = []; + const decoder = new StringDecoder("utf8"); + let bytes = 0; + const sink = new Writable({ + write(chunk: Buffer, _encoding, next) { + onChunk?.(decoder.write(chunk)); + if (bytes < maxBytes) { + const slice = chunk.subarray(0, maxBytes - bytes); + chunks.push(slice); + bytes += slice.length; + if (slice.length < chunk.length) state.truncated = true; + } else { + state.truncated = true; + } + next(); + }, + }); + return { sink, text: () => Buffer.concat(chunks).toString("utf8") }; +}; + +const killByMarker = async ( + docker: Dockerode, + containerId: string, + marker: string, +) => { + const exec = await docker.getContainer(containerId).exec({ + Cmd: ["sh", "-c", buildSandboxKillCommand(marker)], + AttachStdout: false, + AttachStderr: false, + }); + await exec.start({ Detach: true }); +}; + +export const runSandboxExec = async ( + docker: Dockerode, + containerId: string, + options: SandboxExecOptions, +): Promise => { + const container = docker.getContainer(containerId); + const marker = nanoid(); + const exec = await container.exec({ + Cmd: ["sh", "-c", options.cmd], + AttachStdout: true, + AttachStderr: true, + Tty: false, + WorkingDir: options.cwd, + Env: [ + ...toSandboxEnvArray(options.env), + `${SANDBOX_EXEC_MARKER_ENV}=${marker}`, + ], + ...(options.user ? { User: options.user } : {}), + }); + const stream = await exec.start({ hijack: true, stdin: false }); + + const maxBytes = options.maxOutputBytes ?? SANDBOX_EXEC_MAX_OUTPUT_BYTES; + const state = { truncated: false }; + const stdout = createSink(maxBytes, options.onStdout, state); + const stderr = createSink(maxBytes, options.onStderr, state); + + const finished = new Promise((resolve, reject) => { + stream.once("end", resolve); + stream.once("close", resolve); + stream.once("error", reject); + }); + docker.modem.demuxStream(stream, stdout.sink, stderr.sink); + + let timedOut = false; + let containerKilled = false; + let timeoutTask: Promise | null = null; + const timer = setTimeout(() => { + timedOut = true; + timeoutTask = (async () => { + await killByMarker(docker, containerId, marker).catch(() => {}); + const ended = await Promise.race([ + finished.then( + () => true, + () => true, + ), + sleep(KILL_GRACE_MS).then(() => false), + ]); + if (!ended) { + await container + .kill({ signal: "SIGKILL" }) + .then(() => { + containerKilled = true; + }) + .catch(() => {}); + stream.destroy(); + } + })(); + }, options.timeoutMs); + + try { + await finished; + } finally { + clearTimeout(timer); + } + if (timeoutTask) await timeoutTask; + + let exitCode = SANDBOX_EXEC_TIMEOUT_EXIT_CODE; + if (!timedOut) { + const info = await exec.inspect(); + exitCode = info.ExitCode ?? 1; + } + + return { + stdout: stdout.text(), + stderr: stderr.text(), + exitCode, + timedOut, + truncated: state.truncated, + containerKilled, + }; +}; diff --git a/packages/server/src/utils/sandbox/files.ts b/packages/server/src/utils/sandbox/files.ts new file mode 100644 index 000000000..92df92111 --- /dev/null +++ b/packages/server/src/utils/sandbox/files.ts @@ -0,0 +1,68 @@ +import { quote } from "shell-quote"; + +export type SandboxFileType = "file" | "directory" | "symlink" | "other"; + +export interface SandboxFileEntry { + name: string; + type: SandboxFileType; + size: number | null; + mode: string | null; + modifiedAt: string | null; +} + +export const buildSandboxFindCommand = (path: string) => + `find ${quote([path])} -mindepth 1 -maxdepth 1 -printf '%y\\t%s\\t%T@\\t%m\\t%f\\n'`; + +export const buildSandboxLsCommand = (path: string) => + `ls -1Ap ${quote([path])}`; + +const FIND_TYPES: Record = { + f: "file", + d: "directory", + l: "symlink", +}; + +const sortEntries = (a: SandboxFileEntry, b: SandboxFileEntry) => { + if (a.type !== b.type) { + if (a.type === "directory") return -1; + if (b.type === "directory") return 1; + } + return a.name.localeCompare(b.name); +}; + +export const parseSandboxFindOutput = (stdout: string): SandboxFileEntry[] => + stdout + .split("\n") + .filter(Boolean) + .flatMap((line) => { + const [type, size, mtime, mode, ...rest] = line.split("\t"); + const name = rest.join("\t"); + if (!name) return []; + const seconds = Number.parseFloat(mtime ?? ""); + const bytes = Number.parseInt(size ?? "", 10); + return [ + { + name, + type: FIND_TYPES[type ?? ""] ?? "other", + size: Number.isInteger(bytes) ? bytes : null, + mode: mode || null, + modifiedAt: Number.isFinite(seconds) + ? new Date(Math.floor(seconds * 1000)).toISOString() + : null, + }, + ]; + }) + .sort(sortEntries); + +export const parseSandboxLsOutput = (stdout: string): SandboxFileEntry[] => + stdout + .split("\n") + .filter(Boolean) + .map((entry) => ({ + name: entry.endsWith("/") ? entry.slice(0, -1) : entry, + type: (entry.endsWith("/") ? "directory" : "file") as SandboxFileType, + size: null, + mode: null, + modifiedAt: null, + })) + .sort(sortEntries); diff --git a/packages/server/src/utils/sandbox/reaper.ts b/packages/server/src/utils/sandbox/reaper.ts new file mode 100644 index 000000000..90cf40732 --- /dev/null +++ b/packages/server/src/utils/sandbox/reaper.ts @@ -0,0 +1,182 @@ +import { IS_CLOUD } from "@dokploy/server/constants"; +import { db } from "@dokploy/server/db"; +import { sandboxes, server } from "@dokploy/server/db/schema"; +import { getRemoteDocker } from "@dokploy/server/utils/servers/remote-docker"; +import type Dockerode from "dockerode"; +import { and, eq, inArray, isNotNull, isNull, lte } from "drizzle-orm"; +import { killSandbox } from "../../services/sandbox"; +import { SANDBOX_ID_LABEL, SANDBOX_LABEL } from "./container"; + +export const SANDBOX_REAPER_INTERVAL_MS = 15_000; +const STUCK_CREATING_MS = 10 * 60 * 1000; + +let reaperTimer: NodeJS.Timeout | null = null; +let reaping = false; + +export const findExpiredSandboxes = < + T extends { status: string; expiresAt: Date | null }, +>( + list: T[], + now: Date, +) => + list.filter( + (sandbox) => + sandbox.status === "running" && + !!sandbox.expiresAt && + sandbox.expiresAt.getTime() <= now.getTime(), + ); + +export const reapExpiredSandboxes = async ( + now = new Date(), + kill: (sandboxId: string) => Promise = killSandbox, +) => { + const expired = await db.query.sandboxes.findMany({ + where: and(eq(sandboxes.status, "running"), lte(sandboxes.expiresAt, now)), + columns: { sandboxId: true, status: true, expiresAt: true }, + }); + const killed: string[] = []; + for (const sandbox of findExpiredSandboxes(expired, now)) { + try { + await kill(sandbox.sandboxId); + killed.push(sandbox.sandboxId); + } catch (error) { + console.error( + `[Sandbox] Failed to reap ${sandbox.sandboxId}:`, + error instanceof Error ? error.message : error, + ); + } + } + return killed; +}; + +export interface SandboxReconcilePlan { + removeContainers: string[]; + markError: string[]; +} + +export const planSandboxReconcile = ( + rows: { sandboxId: string; containerId: string | null }[], + containers: { Id: string; State: string; Labels: Record }[], +): SandboxReconcilePlan => { + const rowsById = new Map(rows.map((row) => [row.sandboxId, row])); + const liveContainerIds = new Set(); + const removeContainers: string[] = []; + + for (const container of containers) { + const sandboxId = container.Labels?.[SANDBOX_ID_LABEL]; + const row = sandboxId ? rowsById.get(sandboxId) : undefined; + if ( + !row || + row.containerId !== container.Id || + container.State !== "running" + ) { + removeContainers.push(container.Id); + continue; + } + liveContainerIds.add(container.Id); + } + + const markError = rows + .filter((row) => !row.containerId || !liveContainerIds.has(row.containerId)) + .map((row) => row.sandboxId); + + return { removeContainers, markError }; +}; + +const listSandboxContainers = (docker: Dockerode) => + docker.listContainers({ + all: true, + filters: { label: [`${SANDBOX_LABEL}=true`] }, + }); + +const reconcileServer = async (serverId: string | null) => { + const docker = await getRemoteDocker(serverId); + const [containers, rows] = await Promise.all([ + listSandboxContainers(docker), + db.query.sandboxes.findMany({ + where: and( + eq(sandboxes.status, "running"), + serverId + ? eq(sandboxes.serverId, serverId) + : isNull(sandboxes.serverId), + ), + columns: { sandboxId: true, containerId: true }, + }), + ]); + const plan = planSandboxReconcile(rows, containers); + + for (const containerId of plan.removeContainers) { + await docker + .getContainer(containerId) + .remove({ force: true }) + .catch(() => {}); + } + if (plan.markError.length > 0) { + await db + .update(sandboxes) + .set({ status: "error" }) + .where(inArray(sandboxes.sandboxId, plan.markError)); + } + return plan; +}; + +export const reconcileSandboxes = async () => { + const stuckSince = new Date(Date.now() - STUCK_CREATING_MS); + await db + .update(sandboxes) + .set({ status: "error" }) + .where( + and( + eq(sandboxes.status, "creating"), + lte(sandboxes.createdAt, stuckSince.toISOString()), + ), + ); + + const serverIds: (string | null)[] = IS_CLOUD ? [] : [null]; + const remoteServers = await db.query.server.findMany({ + where: isNotNull(server.sshKeyId), + columns: { serverId: true }, + }); + serverIds.push(...remoteServers.map((remote) => remote.serverId)); + + for (const serverId of serverIds) { + try { + const plan = await reconcileServer(serverId); + if (plan.removeContainers.length || plan.markError.length) { + console.log( + `[Sandbox] Reconciled ${serverId ?? "local"}: removed ${plan.removeContainers.length} containers, errored ${plan.markError.length} sandboxes`, + ); + } + } catch (error) { + console.error( + `[Sandbox] Reconcile failed for ${serverId ?? "local"}:`, + error instanceof Error ? error.message : error, + ); + } + } +}; + +export const initSandboxReaper = async () => { + if (reaperTimer) return; + await reconcileSandboxes(); + reaperTimer = setInterval(async () => { + if (reaping) return; + reaping = true; + try { + await reapExpiredSandboxes(); + } catch (error) { + console.error( + "[Sandbox] Reaper failed:", + error instanceof Error ? error.message : error, + ); + } finally { + reaping = false; + } + }, SANDBOX_REAPER_INTERVAL_MS); + reaperTimer.unref(); +}; + +export const stopSandboxReaper = () => { + if (reaperTimer) clearInterval(reaperTimer); + reaperTimer = null; +}; diff --git a/packages/server/src/utils/sandbox/tar.ts b/packages/server/src/utils/sandbox/tar.ts new file mode 100644 index 000000000..36bcbba8e --- /dev/null +++ b/packages/server/src/utils/sandbox/tar.ts @@ -0,0 +1,124 @@ +import type { Readable } from "node:stream"; +import { extract, pack } from "tar-stream"; + +export interface SandboxTarFile { + name: string; + content: Buffer; + mode?: number; + uid?: number; + gid?: number; +} + +export interface SandboxTarDirectory { + name: string; + mode?: number; + uid?: number; + gid?: number; +} + +const toBuffer = (chunk: unknown) => + Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array | string); + +const collect = async (stream: AsyncIterable) => { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(toBuffer(chunk)); + } + return Buffer.concat(chunks); +}; + +export const buildSandboxFileTar = async (file: SandboxTarFile) => { + const tar = pack(); + tar.entry( + { + name: file.name, + type: "file", + size: file.content.length, + mode: file.mode ?? 0o644, + uid: file.uid ?? 0, + gid: file.gid ?? 0, + mtime: new Date(), + }, + file.content, + ); + tar.finalize(); + return collect(tar); +}; + +export const buildSandboxDirectoryTar = async (dir: SandboxTarDirectory) => { + const tar = pack(); + tar.entry({ + name: dir.name.endsWith("/") ? dir.name : `${dir.name}/`, + type: "directory", + mode: dir.mode ?? 0o755, + uid: dir.uid ?? 0, + gid: dir.gid ?? 0, + mtime: new Date(), + }); + tar.finalize(); + return collect(tar); +}; + +export class SandboxFileTooLargeError extends Error { + constructor(maxBytes: number) { + super(`File exceeds the ${maxBytes} byte limit`); + this.name = "SandboxFileTooLargeError"; + } +} + +export class SandboxNotRegularFileError extends Error { + constructor() { + super("Path is not a regular file"); + this.name = "SandboxNotRegularFileError"; + } +} + +export const extractSandboxFile = ( + source: NodeJS.ReadableStream, + maxBytes: number, +): Promise<{ name: string; content: Buffer }> => + new Promise((resolve, reject) => { + const extractor = extract(); + let found: { name: string; content: Buffer } | null = null; + let settled = false; + + const fail = (error: Error) => { + if (settled) return; + settled = true; + extractor.destroy(); + (source as Readable).destroy?.(); + reject(error); + }; + + extractor.on("entry", (header, entryStream, next) => { + if (found || header.type !== "file") { + entryStream.on("end", next); + entryStream.resume(); + return; + } + const chunks: Buffer[] = []; + let size = 0; + entryStream.on("data", (data: unknown) => { + const chunk = toBuffer(data); + size += chunk.length; + if (size > maxBytes) { + fail(new SandboxFileTooLargeError(maxBytes)); + return; + } + chunks.push(chunk); + }); + entryStream.on("end", () => { + found = { name: header.name, content: Buffer.concat(chunks) }; + next(); + }); + }); + extractor.on("finish", () => { + if (settled) return; + settled = true; + if (found) resolve(found); + else reject(new SandboxNotRegularFileError()); + }); + extractor.on("error", fail); + source.on("error", fail); + source.pipe(extractor); + }); diff --git a/packages/server/src/utils/sandbox/templates.ts b/packages/server/src/utils/sandbox/templates.ts new file mode 100644 index 000000000..0cd435437 --- /dev/null +++ b/packages/server/src/utils/sandbox/templates.ts @@ -0,0 +1,30 @@ +import type { SandboxTemplateName } from "@dokploy/server/db/schema"; + +export interface SandboxTemplate { + image: string; + user: string | null; + workdir: string; + description: string; +} + +// Placeholder images until the dokploy/sandbox-* images are published. +export const SANDBOX_TEMPLATES: Record = { + base: { + image: "ubuntu:24.04", + user: "1000:1000", + workdir: "/home/user", + description: "Ubuntu 24.04 with a POSIX shell and coreutils", + }, + python: { + image: "python:3.12-slim", + user: "1000:1000", + workdir: "/home/user", + description: "Python 3.12 (pip available)", + }, + node: { + image: "node:22-slim", + user: "1000:1000", + workdir: "/home/user", + description: "Node.js 22 (npm available)", + }, +};