diff --git a/app/server/config/integration/kubernetes.ts b/app/server/config/integration/kubernetes.ts index aa4ea42..fa201f7 100644 --- a/app/server/config/integration/kubernetes.ts +++ b/app/server/config/integration/kubernetes.ts @@ -1,265 +1,167 @@ -import { readdir, readFile } from 'node:fs/promises'; -import { platform } from 'node:os'; -import { join, resolve } from 'node:path'; -import { kill } from 'node:process'; -import { setTimeout } from 'node:timers/promises'; -import { CoreV1Api, KubeConfig } from '@kubernetes/client-node'; -import { type } from 'arktype'; -import type { RuntimeApiClient } from '~/server/headscale/api/endpoints'; -import log from '~/utils/log'; -import { Integration } from './abstract'; +import { CoreV1Api, KubeConfig } from "@kubernetes/client-node"; +import { type } from "arktype"; +import { readdir, readFile } from "node:fs/promises"; +import { platform } from "node:os"; +import { join } from "node:path"; + +import type { RuntimeApiClient } from "~/server/headscale/api/endpoints"; + +import log from "~/utils/log"; + +import { Integration } from "./abstract"; +import { findHeadscaleServe, signalAndWaitHealthy } from "./proc-helper"; // https://github.com/kubernetes-client/javascript/blob/055b83c6504dfd1b2a2d081efd974163c6cbb808/src/config.ts#L40 -const svcRoot = '/var/run/secrets/kubernetes.io/serviceaccount'; +const svcRoot = "/var/run/secrets/kubernetes.io/serviceaccount"; const svcCaPath = `${svcRoot}/ca.crt`; const svcTokenPath = `${svcRoot}/token`; const svcNamespacePath = `${svcRoot}/namespace`; const configSchema = { - full: type({ - enabled: 'boolean', - pod_name: 'string', - validate_manifest: 'boolean = true', - }), + full: type({ + enabled: "boolean", + pod_name: "string", + validate_manifest: "boolean = true", + }), - partial: type({ - enabled: 'boolean?', - pod_name: 'string?', - validate_manifest: 'boolean?', - }).partial(), + partial: type({ + enabled: "boolean?", + pod_name: "string?", + validate_manifest: "boolean?", + }).partial(), }; -export default class KubernetesIntegration extends Integration< - typeof configSchema.full.infer -> { - private pid: number | undefined; - private maxAttempts = 10; +export default class KubernetesIntegration extends Integration { + private pid: number | undefined; - get name() { - return 'Kubernetes (k8s)'; - } + get name() { + return "Kubernetes (k8s)"; + } - static get configSchema() { - return configSchema; - } + static get configSchema() { + return configSchema; + } - async isAvailable() { - if (platform() !== 'linux') { - log.error('config', 'Kubernetes is only available on Linux'); - return false; - } + async isAvailable() { + if (platform() !== "linux") { + log.error("config", "Kubernetes is only available on Linux"); + return false; + } - try { - log.debug('config', 'Checking Kubernetes service account at %s', svcRoot); - const files = await readdir(svcRoot); - if (files.length === 0) { - log.error('config', 'Kubernetes service account not found'); - return false; - } + try { + log.debug("config", "Checking Kubernetes service account at %s", svcRoot); + const files = await readdir(svcRoot); + if (files.length === 0) { + log.error("config", "Kubernetes service account not found"); + return false; + } - const mappedFiles = new Set(files.map((file) => join(svcRoot, file))); - const expectedFiles = [svcCaPath, svcTokenPath, svcNamespacePath]; + const mappedFiles = new Set(files.map((file) => join(svcRoot, file))); + const expectedFiles = [svcCaPath, svcTokenPath, svcNamespacePath]; - log.debug('config', 'Looking for %s', expectedFiles.join(', ')); - if (!expectedFiles.every((file) => mappedFiles.has(file))) { - log.error('config', 'Malformed Kubernetes service account'); - return false; - } - } catch (error) { - log.error('config', 'Failed to access %s: %s', svcRoot, error); - return false; - } + log.debug("config", "Looking for %s", expectedFiles.join(", ")); + if (!expectedFiles.every((file) => mappedFiles.has(file))) { + log.error("config", "Malformed Kubernetes service account"); + return false; + } + } catch (error) { + log.error("config", "Failed to access %s: %s", svcRoot, error); + return false; + } - log.debug('config', 'Reading Kubernetes service account at %s', svcRoot); - const namespace = await readFile(svcNamespacePath, 'utf8'); + log.debug("config", "Reading Kubernetes service account at %s", svcRoot); + const namespace = await readFile(svcNamespacePath, "utf8"); - // Some very ugly nesting but it's necessary - if (this.context.validate_manifest === false) { - log.warn('config', 'Skipping strict Pod status check'); - } else { - const pod = this.context.pod_name; - if (!pod) { - log.error('config', 'Missing POD_NAME variable'); - return false; - } + // Some very ugly nesting but it's necessary + if (this.context.validate_manifest === false) { + log.warn("config", "Skipping strict Pod status check"); + } else { + const pod = this.context.pod_name; + if (!pod) { + log.error("config", "Missing POD_NAME variable"); + return false; + } - if (pod.trim().length === 0) { - log.error('config', 'Pod name is empty'); - return false; - } + if (pod.trim().length === 0) { + log.error("config", "Pod name is empty"); + return false; + } - log.debug( - 'config', - 'Checking Kubernetes pod %s in namespace %s', - pod, - namespace, - ); + log.debug("config", "Checking Kubernetes pod %s in namespace %s", pod, namespace); - try { - log.debug('config', 'Attempgin to get cluster KubeConfig'); - const kc = new KubeConfig(); - kc.loadFromCluster(); + try { + log.debug("config", "Attempgin to get cluster KubeConfig"); + const kc = new KubeConfig(); + kc.loadFromCluster(); - const cluster = kc.getCurrentCluster(); - if (!cluster) { - log.error('config', 'Malformed kubeconfig'); - return false; - } + const cluster = kc.getCurrentCluster(); + if (!cluster) { + log.error("config", "Malformed kubeconfig"); + return false; + } - log.info( - 'config', - 'Service account connected to %s (%s)', - cluster.name, - cluster.server, - ); + log.info("config", "Service account connected to %s (%s)", cluster.name, cluster.server); - const kCoreV1Api = kc.makeApiClient(CoreV1Api); + const kCoreV1Api = kc.makeApiClient(CoreV1Api); - log.info('config', 'Checking pod %s in namespace %s', pod, namespace); - log.debug('config', 'Reading pod info for %s', pod); - const body = await kCoreV1Api.readNamespacedPod({ - name: pod, - namespace, - }); + log.info("config", "Checking pod %s in namespace %s", pod, namespace); + log.debug("config", "Reading pod info for %s", pod); + const body = await kCoreV1Api.readNamespacedPod({ + name: pod, + namespace, + }); - if (!body.spec) { - log.error( - 'config', - 'Missing spec in pod info for %s/%s', - pod, - namespace, - ); + if (!body.spec) { + log.error("config", "Missing spec in pod info for %s/%s", pod, namespace); - return false; - } + return false; + } - log.debug('config', 'Got pod info: %o', body.spec); - const shared = body.spec.shareProcessNamespace; - if (shared === undefined) { - log.error( - 'config', - 'Pod does not have spec.shareProcessNamespace set', - ); + log.debug("config", "Got pod info: %o", body.spec); + const shared = body.spec.shareProcessNamespace; + if (shared === undefined) { + log.error("config", "Pod does not have spec.shareProcessNamespace set"); - return false; - } + return false; + } - if (!shared) { - log.error( - 'config', - 'Pod has set but disabled spec.shareProcessNamespace', - ); + if (!shared) { + log.error("config", "Pod has set but disabled spec.shareProcessNamespace"); - return false; - } + return false; + } - log.info('config', 'Pod %s enabled shared processes', pod); - } catch (error) { - log.error('config', 'Failed to read pod info: %s', error); - return false; - } - } + log.info("config", "Pod %s enabled shared processes", pod); + } catch (error) { + log.error("config", "Failed to read pod info: %s", error); + return false; + } + } - log.debug('config', 'Looking for namespaced process in /proc'); - const dir = resolve('/proc'); - try { - const subdirs = await readdir(dir); - const promises = subdirs.map(async (dir) => { - const pid = Number.parseInt(dir, 10); + try { + const result = await findHeadscaleServe(); + if (!result) { + log.error("config", "Could not find headscale serve process"); + return false; + } - if (Number.isNaN(pid)) { - return; - } + this.pid = result; + log.info("config", "Found headscale serve (PID %d)", this.pid); + return true; + } catch (error) { + log.error("config", "Failed to scan /proc: %s", error); + return false; + } + } - const path = join('/proc', dir, 'comm'); - try { - log.debug('config', 'Reading %s', path); - const data = await readFile(path, 'utf8'); - if (data.trim() !== 'headscale') { - throw new Error( - `Found PID with unexpected command: ${data.trim()}`, - ); - } + async onConfigChange(client: RuntimeApiClient) { + if (!this.pid) { + return; + } - return pid; - } catch (error) { - log.debug('config', 'Failed to read %s: %s', path, error); - } - }); - - const results = await Promise.allSettled(promises); - const pids = []; - - for (const result of results) { - if (result.status === 'fulfilled' && result.value) { - pids.push(result.value); - } - } - - log.debug('config', 'Found Headscale processes: %o', pids); - if (pids.length > 1) { - log.error( - 'config', - 'Found %d Headscale processes: %s', - pids.length, - pids.join(', '), - ); - return false; - } - - if (pids.length === 0) { - log.error('config', 'Could not find Headscale process'); - return false; - } - - this.pid = pids[0]; - log.info('config', 'Found Headscale process with PID: %d', this.pid); - return true; - } catch { - log.error('config', 'Failed to read /proc'); - return false; - } - } - - async onConfigChange(client: RuntimeApiClient) { - if (!this.pid) { - return; - } - - try { - log.info('config', 'Sending SIGTERM to Headscale'); - kill(this.pid, 'SIGTERM'); - } catch (error) { - log.error('config', 'Failed to send SIGTERM to Headscale: %s', error); - log.debug('config', 'kill(1) error: %o', error); - } - - await setTimeout(1000); - let attempts = 0; - while (attempts <= this.maxAttempts) { - try { - log.debug('config', 'Checking Headscale status (attempt %d)', attempts); - const status = await client.isHealthy(); - if (status === false) { - throw new Error('Headscale is not running'); - } - - log.info('config', 'Headscale is up and running'); - return; - } catch { - if (attempts < this.maxAttempts) { - attempts++; - await setTimeout(1000); - continue; - } - - log.error( - 'config', - 'Missed restart deadline for Headscale (pid %d)', - this.pid, - ); - return; - } - } - } + await signalAndWaitHealthy(client, { + pid: this.pid, + signal: "SIGHUP", + }); + } } diff --git a/app/server/config/integration/proc-helper.ts b/app/server/config/integration/proc-helper.ts new file mode 100644 index 0000000..4432dde --- /dev/null +++ b/app/server/config/integration/proc-helper.ts @@ -0,0 +1,116 @@ +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { kill } from "node:process"; +import { setTimeout } from "node:timers/promises"; + +import type { RuntimeApiClient } from "~/server/headscale/api/endpoints"; + +import log from "~/utils/log"; + +/** + * Does a two-stage scan of /proc to find the headscale process that is running + * a "serve" subcommand. It first scans all processes' comm files to find + * headscale processes, then checks their cmdline files to see if "serve" is + * the second argument. + * + * @param procPath The path to the proc filesystem (default: /proc) + * @returns The PID of the headscale serve process, or undefined if not found + */ +export async function findHeadscaleServe(procPath = "/proc"): Promise { + const subdirs = await readdir(procPath); + const commResults = await Promise.allSettled( + subdirs.map(async (entry) => { + const pid = Number.parseInt(entry, 10); + if (Number.isNaN(pid)) { + return undefined; + } + + try { + const comm = await readFile(join(procPath, entry, "comm"), "utf8"); + return comm.trim() === "headscale" ? pid : undefined; + } catch { + return undefined; + } + }), + ); + + const headscalePids = commResults + .map((result) => { + if (result.status === "fulfilled" && result.value !== undefined) { + return result.value; + } + return undefined; + }) + .filter((pid): pid is number => pid !== undefined); + + if (headscalePids.length === 0) { + return undefined; + } + + log.debug("config", "Found %d headscale process(es), checking for serve", headscalePids.length); + for (const pid of headscalePids) { + try { + const cmdline = await readFile(join(procPath, pid.toString(), "cmdline"), "utf8"); + const args = cmdline.split("\0").filter(Boolean); + + if (args[1] === "serve") { + return pid; + } + } catch { + // Process may have exited between stages + } + } + + return undefined; +} + +/** + * Options for signaling the headscale process. + */ +export interface SignalHeadscaleOptions { + pid: number; + signal?: NodeJS.Signals; + maxAttempts?: number; + retryDelayMs?: number; +} + +/** + * Sends a signal to the headscale process and waits for it to become healthy. + * @param client The RuntimeApiClient to check health + * @param options Options for signaling and waiting + * @returns True if headscale became healthy, false otherwise + */ +export async function signalAndWaitHealthy( + client: RuntimeApiClient, + options: SignalHeadscaleOptions, +): Promise { + const { pid, signal = "SIGHUP", maxAttempts = 10, retryDelayMs = 1000 } = options; + + try { + kill(pid, signal); + log.info("config", "Sent %s to Headscale (PID %d)", signal, pid); + } catch (error) { + log.error("config", "Failed to send %s to PID %d: %s", signal, pid, error); + return false; + } + + await setTimeout(retryDelayMs); + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const healthy = await client.isHealthy(); + if (healthy) { + log.info("config", "Headscale is healthy after restart"); + return true; + } + } catch { + // Still restarting + } + + if (attempt < maxAttempts) { + await setTimeout(retryDelayMs); + } + } + + log.error("config", "Headscale did not become healthy after %d attempts", maxAttempts); + return false; +} diff --git a/app/server/config/integration/proc.ts b/app/server/config/integration/proc.ts index 1c51603..8c9a886 100644 --- a/app/server/config/integration/proc.ts +++ b/app/server/config/integration/proc.ts @@ -1,175 +1,64 @@ -import { readdir, readFile } from 'node:fs/promises'; -import { platform } from 'node:os'; -import { join, resolve } from 'node:path'; -import { kill } from 'node:process'; -import { setTimeout } from 'node:timers/promises'; -import { type } from 'arktype'; -import type { RuntimeApiClient } from '~/server/headscale/api/endpoints'; -import log from '~/utils/log'; -import type { HeadplaneConfig } from '../config-schema'; -import { Integration } from './abstract'; +import { type } from "arktype"; +import { platform } from "node:os"; + +import type { RuntimeApiClient } from "~/server/headscale/api/endpoints"; + +import log from "~/utils/log"; + +import { Integration } from "./abstract"; +import { findHeadscaleServe, signalAndWaitHealthy } from "./proc-helper"; const configSchema = { - full: type({ - enabled: 'boolean', - }), + full: type({ + enabled: "boolean", + }), - partial: type({ - enabled: 'boolean?', - }).partial(), + partial: type({ + enabled: "boolean?", + }).partial(), }; -export default class ProcIntegration extends Integration< - typeof configSchema.full.infer -> { - private pid: number | undefined; - private maxAttempts = 10; +export default class ProcIntegration extends Integration { + private pid: number | undefined; - get name() { - return 'Native Linux (/proc)'; - } + get name() { + return "Native Linux (/proc)"; + } - static get configSchema() { - return configSchema; - } + static get configSchema() { + return configSchema; + } - async isAvailable() { - if (platform() !== 'linux') { - log.error('config', '/proc is only available on Linux'); - return false; - } + async isAvailable() { + if (platform() !== "linux") { + log.error("config", "/proc is only available on Linux"); + return false; + } - log.debug('config', 'Checking /proc for Headscale process'); - const dir = resolve('/proc'); - try { - const subdirs = await readdir(dir); - const promises = subdirs.map(async (dir) => { - const pid = Number.parseInt(dir, 10); + try { + const result = await findHeadscaleServe(); + if (!result) { + log.error("config", "Could not find headscale serve process"); + return false; + } - if (Number.isNaN(pid)) { - return; - } + this.pid = result; + log.info("config", "Found headscale serve (PID %d)", this.pid); + return true; + } catch (error) { + log.error("config", "Failed to scan /proc: %s", error); + return false; + } + } - const path = join('/proc', dir, 'comm'); - try { - log.debug('config', 'Reading %s', path); - const data = await readFile(path, 'utf8'); - if (data.trim() !== 'headscale') { - throw new Error( - `Found PID with unexpected command: ${data.trim()}`, - ); - } + async onConfigChange(client: RuntimeApiClient) { + if (!this.pid) { + return; + } - return pid; - } catch (error) { - log.debug('config', 'Failed to read %s: %s', path, error); - } - }); - - const results = await Promise.allSettled(promises); - const pids = []; - - for (const result of results) { - if (result.status === 'fulfilled' && result.value) { - pids.push(result.value); - } - } - - log.debug('config', 'Found Headscale processes: %o', pids); - if (pids.length > 1) { - log.warn( - 'config', - 'Found %d Headscale processes: %s', - pids.length, - pids.join(', '), - ); - - log.debug( - 'config', - 'Checking if any of them have Parent PID = 1, assuming thats the correct PID', - ); - const ppidRegex = /(?:PPid:\s)(\d+)(?:\n?)/; - for (const pid of pids) { - const pidStatusPath = join('/proc', pid.toString(), 'status'); - try { - log.debug('config', 'Reading %s', pidStatusPath); - const pidData = await readFile(pidStatusPath, 'utf8'); - const ppidResult = pidData.match(ppidRegex); - - if (ppidResult !== null) { - const potentialPPid = Number.parseInt(ppidResult[1], 10); - if (potentialPPid === 1) { - this.pid = pid; - log.info( - 'config', - 'Found potential Headscale process with PID: %d based on Parent PID = 1', - this.pid, - ); - return true; - } - } - } catch (error) { - log.error('config', 'Failed to read %s: %s', pidStatusPath, error); - } - } - - return false; - } - - if (pids.length === 0) { - log.error('config', 'Could not find Headscale process'); - return false; - } - - this.pid = pids[0]; - log.info('config', 'Found Headscale process with PID: %d', this.pid); - return true; - } catch { - log.error('config', 'Failed to read /proc'); - return false; - } - } - - async onConfigChange(client: RuntimeApiClient) { - if (!this.pid) { - return; - } - - try { - log.info('config', 'Sending SIGTERM to Headscale'); - kill(this.pid, 'SIGTERM'); - } catch (error) { - log.error('config', 'Failed to send SIGTERM to Headscale: %s', error); - log.debug('config', 'kill(1) error: %o', error); - } - - await setTimeout(1000); - let attempts = 0; - while (attempts <= this.maxAttempts) { - try { - log.debug('config', 'Checking Headscale status (attempt %d)', attempts); - const status = await client.isHealthy(); - if (status === false) { - log.error('config', 'Headscale is not running'); - return; - } - - log.info('config', 'Headscale is up and running'); - return; - } catch { - if (attempts < this.maxAttempts) { - attempts++; - await setTimeout(1000); - continue; - } - - log.error( - 'config', - 'Missed restart deadline for Headscale (pid %d)', - this.pid, - ); - return; - } - } - } + await signalAndWaitHealthy(client, { + pid: this.pid, + signal: "SIGHUP", + }); + } }