fix: strengthen the proc scanner for headscale

This commit is contained in:
Aarnav Tale
2026-01-16 00:20:26 -05:00
parent 15d145bf01
commit 2618cdd9f6
3 changed files with 297 additions and 390 deletions
+130 -228
View File
@@ -1,265 +1,167 @@
import { readdir, readFile } from 'node:fs/promises';
import { platform } from 'node:os';
import { join, resolve } from 'node:path';
import { kill } from 'node:process';
import { setTimeout } from 'node:timers/promises';
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
import { type } from 'arktype';
import type { RuntimeApiClient } from '~/server/headscale/api/endpoints';
import log from '~/utils/log';
import { Integration } from './abstract';
import { CoreV1Api, KubeConfig } from "@kubernetes/client-node";
import { type } from "arktype";
import { readdir, readFile } from "node:fs/promises";
import { platform } from "node:os";
import { join } from "node:path";
import type { RuntimeApiClient } from "~/server/headscale/api/endpoints";
import log from "~/utils/log";
import { Integration } from "./abstract";
import { findHeadscaleServe, signalAndWaitHealthy } from "./proc-helper";
// https://github.com/kubernetes-client/javascript/blob/055b83c6504dfd1b2a2d081efd974163c6cbb808/src/config.ts#L40
const svcRoot = '/var/run/secrets/kubernetes.io/serviceaccount';
const svcRoot = "/var/run/secrets/kubernetes.io/serviceaccount";
const svcCaPath = `${svcRoot}/ca.crt`;
const svcTokenPath = `${svcRoot}/token`;
const svcNamespacePath = `${svcRoot}/namespace`;
const configSchema = {
full: type({
enabled: 'boolean',
pod_name: 'string',
validate_manifest: 'boolean = true',
}),
full: type({
enabled: "boolean",
pod_name: "string",
validate_manifest: "boolean = true",
}),
partial: type({
enabled: 'boolean?',
pod_name: 'string?',
validate_manifest: 'boolean?',
}).partial(),
partial: type({
enabled: "boolean?",
pod_name: "string?",
validate_manifest: "boolean?",
}).partial(),
};
export default class KubernetesIntegration extends Integration<
typeof configSchema.full.infer
> {
private pid: number | undefined;
private maxAttempts = 10;
export default class KubernetesIntegration extends Integration<typeof configSchema.full.infer> {
private pid: number | undefined;
get name() {
return 'Kubernetes (k8s)';
}
get name() {
return "Kubernetes (k8s)";
}
static get configSchema() {
return configSchema;
}
static get configSchema() {
return configSchema;
}
async isAvailable() {
if (platform() !== 'linux') {
log.error('config', 'Kubernetes is only available on Linux');
return false;
}
async isAvailable() {
if (platform() !== "linux") {
log.error("config", "Kubernetes is only available on Linux");
return false;
}
try {
log.debug('config', 'Checking Kubernetes service account at %s', svcRoot);
const files = await readdir(svcRoot);
if (files.length === 0) {
log.error('config', 'Kubernetes service account not found');
return false;
}
try {
log.debug("config", "Checking Kubernetes service account at %s", svcRoot);
const files = await readdir(svcRoot);
if (files.length === 0) {
log.error("config", "Kubernetes service account not found");
return false;
}
const mappedFiles = new Set(files.map((file) => join(svcRoot, file)));
const expectedFiles = [svcCaPath, svcTokenPath, svcNamespacePath];
const mappedFiles = new Set(files.map((file) => join(svcRoot, file)));
const expectedFiles = [svcCaPath, svcTokenPath, svcNamespacePath];
log.debug('config', 'Looking for %s', expectedFiles.join(', '));
if (!expectedFiles.every((file) => mappedFiles.has(file))) {
log.error('config', 'Malformed Kubernetes service account');
return false;
}
} catch (error) {
log.error('config', 'Failed to access %s: %s', svcRoot, error);
return false;
}
log.debug("config", "Looking for %s", expectedFiles.join(", "));
if (!expectedFiles.every((file) => mappedFiles.has(file))) {
log.error("config", "Malformed Kubernetes service account");
return false;
}
} catch (error) {
log.error("config", "Failed to access %s: %s", svcRoot, error);
return false;
}
log.debug('config', 'Reading Kubernetes service account at %s', svcRoot);
const namespace = await readFile(svcNamespacePath, 'utf8');
log.debug("config", "Reading Kubernetes service account at %s", svcRoot);
const namespace = await readFile(svcNamespacePath, "utf8");
// Some very ugly nesting but it's necessary
if (this.context.validate_manifest === false) {
log.warn('config', 'Skipping strict Pod status check');
} else {
const pod = this.context.pod_name;
if (!pod) {
log.error('config', 'Missing POD_NAME variable');
return false;
}
// Some very ugly nesting but it's necessary
if (this.context.validate_manifest === false) {
log.warn("config", "Skipping strict Pod status check");
} else {
const pod = this.context.pod_name;
if (!pod) {
log.error("config", "Missing POD_NAME variable");
return false;
}
if (pod.trim().length === 0) {
log.error('config', 'Pod name is empty');
return false;
}
if (pod.trim().length === 0) {
log.error("config", "Pod name is empty");
return false;
}
log.debug(
'config',
'Checking Kubernetes pod %s in namespace %s',
pod,
namespace,
);
log.debug("config", "Checking Kubernetes pod %s in namespace %s", pod, namespace);
try {
log.debug('config', 'Attempgin to get cluster KubeConfig');
const kc = new KubeConfig();
kc.loadFromCluster();
try {
log.debug("config", "Attempgin to get cluster KubeConfig");
const kc = new KubeConfig();
kc.loadFromCluster();
const cluster = kc.getCurrentCluster();
if (!cluster) {
log.error('config', 'Malformed kubeconfig');
return false;
}
const cluster = kc.getCurrentCluster();
if (!cluster) {
log.error("config", "Malformed kubeconfig");
return false;
}
log.info(
'config',
'Service account connected to %s (%s)',
cluster.name,
cluster.server,
);
log.info("config", "Service account connected to %s (%s)", cluster.name, cluster.server);
const kCoreV1Api = kc.makeApiClient(CoreV1Api);
const kCoreV1Api = kc.makeApiClient(CoreV1Api);
log.info('config', 'Checking pod %s in namespace %s', pod, namespace);
log.debug('config', 'Reading pod info for %s', pod);
const body = await kCoreV1Api.readNamespacedPod({
name: pod,
namespace,
});
log.info("config", "Checking pod %s in namespace %s", pod, namespace);
log.debug("config", "Reading pod info for %s", pod);
const body = await kCoreV1Api.readNamespacedPod({
name: pod,
namespace,
});
if (!body.spec) {
log.error(
'config',
'Missing spec in pod info for %s/%s',
pod,
namespace,
);
if (!body.spec) {
log.error("config", "Missing spec in pod info for %s/%s", pod, namespace);
return false;
}
return false;
}
log.debug('config', 'Got pod info: %o', body.spec);
const shared = body.spec.shareProcessNamespace;
if (shared === undefined) {
log.error(
'config',
'Pod does not have spec.shareProcessNamespace set',
);
log.debug("config", "Got pod info: %o", body.spec);
const shared = body.spec.shareProcessNamespace;
if (shared === undefined) {
log.error("config", "Pod does not have spec.shareProcessNamespace set");
return false;
}
return false;
}
if (!shared) {
log.error(
'config',
'Pod has set but disabled spec.shareProcessNamespace',
);
if (!shared) {
log.error("config", "Pod has set but disabled spec.shareProcessNamespace");
return false;
}
return false;
}
log.info('config', 'Pod %s enabled shared processes', pod);
} catch (error) {
log.error('config', 'Failed to read pod info: %s', error);
return false;
}
}
log.info("config", "Pod %s enabled shared processes", pod);
} catch (error) {
log.error("config", "Failed to read pod info: %s", error);
return false;
}
}
log.debug('config', 'Looking for namespaced process in /proc');
const dir = resolve('/proc');
try {
const subdirs = await readdir(dir);
const promises = subdirs.map(async (dir) => {
const pid = Number.parseInt(dir, 10);
try {
const result = await findHeadscaleServe();
if (!result) {
log.error("config", "Could not find headscale serve process");
return false;
}
if (Number.isNaN(pid)) {
return;
}
this.pid = result;
log.info("config", "Found headscale serve (PID %d)", this.pid);
return true;
} catch (error) {
log.error("config", "Failed to scan /proc: %s", error);
return false;
}
}
const path = join('/proc', dir, 'comm');
try {
log.debug('config', 'Reading %s', path);
const data = await readFile(path, 'utf8');
if (data.trim() !== 'headscale') {
throw new Error(
`Found PID with unexpected command: ${data.trim()}`,
);
}
async onConfigChange(client: RuntimeApiClient) {
if (!this.pid) {
return;
}
return pid;
} catch (error) {
log.debug('config', 'Failed to read %s: %s', path, error);
}
});
const results = await Promise.allSettled(promises);
const pids = [];
for (const result of results) {
if (result.status === 'fulfilled' && result.value) {
pids.push(result.value);
}
}
log.debug('config', 'Found Headscale processes: %o', pids);
if (pids.length > 1) {
log.error(
'config',
'Found %d Headscale processes: %s',
pids.length,
pids.join(', '),
);
return false;
}
if (pids.length === 0) {
log.error('config', 'Could not find Headscale process');
return false;
}
this.pid = pids[0];
log.info('config', 'Found Headscale process with PID: %d', this.pid);
return true;
} catch {
log.error('config', 'Failed to read /proc');
return false;
}
}
async onConfigChange(client: RuntimeApiClient) {
if (!this.pid) {
return;
}
try {
log.info('config', 'Sending SIGTERM to Headscale');
kill(this.pid, 'SIGTERM');
} catch (error) {
log.error('config', 'Failed to send SIGTERM to Headscale: %s', error);
log.debug('config', 'kill(1) error: %o', error);
}
await setTimeout(1000);
let attempts = 0;
while (attempts <= this.maxAttempts) {
try {
log.debug('config', 'Checking Headscale status (attempt %d)', attempts);
const status = await client.isHealthy();
if (status === false) {
throw new Error('Headscale is not running');
}
log.info('config', 'Headscale is up and running');
return;
} catch {
if (attempts < this.maxAttempts) {
attempts++;
await setTimeout(1000);
continue;
}
log.error(
'config',
'Missed restart deadline for Headscale (pid %d)',
this.pid,
);
return;
}
}
}
await signalAndWaitHealthy(client, {
pid: this.pid,
signal: "SIGHUP",
});
}
}
@@ -0,0 +1,116 @@
import { readdir, readFile } from "node:fs/promises";
import { join } from "node:path";
import { kill } from "node:process";
import { setTimeout } from "node:timers/promises";
import type { RuntimeApiClient } from "~/server/headscale/api/endpoints";
import log from "~/utils/log";
/**
* Does a two-stage scan of /proc to find the headscale process that is running
* a "serve" subcommand. It first scans all processes' comm files to find
* headscale processes, then checks their cmdline files to see if "serve" is
* the second argument.
*
* @param procPath The path to the proc filesystem (default: /proc)
* @returns The PID of the headscale serve process, or undefined if not found
*/
export async function findHeadscaleServe(procPath = "/proc"): Promise<number | undefined> {
const subdirs = await readdir(procPath);
const commResults = await Promise.allSettled(
subdirs.map(async (entry) => {
const pid = Number.parseInt(entry, 10);
if (Number.isNaN(pid)) {
return undefined;
}
try {
const comm = await readFile(join(procPath, entry, "comm"), "utf8");
return comm.trim() === "headscale" ? pid : undefined;
} catch {
return undefined;
}
}),
);
const headscalePids = commResults
.map((result) => {
if (result.status === "fulfilled" && result.value !== undefined) {
return result.value;
}
return undefined;
})
.filter((pid): pid is number => pid !== undefined);
if (headscalePids.length === 0) {
return undefined;
}
log.debug("config", "Found %d headscale process(es), checking for serve", headscalePids.length);
for (const pid of headscalePids) {
try {
const cmdline = await readFile(join(procPath, pid.toString(), "cmdline"), "utf8");
const args = cmdline.split("\0").filter(Boolean);
if (args[1] === "serve") {
return pid;
}
} catch {
// Process may have exited between stages
}
}
return undefined;
}
/**
* Options for signaling the headscale process.
*/
export interface SignalHeadscaleOptions {
pid: number;
signal?: NodeJS.Signals;
maxAttempts?: number;
retryDelayMs?: number;
}
/**
* Sends a signal to the headscale process and waits for it to become healthy.
* @param client The RuntimeApiClient to check health
* @param options Options for signaling and waiting
* @returns True if headscale became healthy, false otherwise
*/
export async function signalAndWaitHealthy(
client: RuntimeApiClient,
options: SignalHeadscaleOptions,
): Promise<boolean> {
const { pid, signal = "SIGHUP", maxAttempts = 10, retryDelayMs = 1000 } = options;
try {
kill(pid, signal);
log.info("config", "Sent %s to Headscale (PID %d)", signal, pid);
} catch (error) {
log.error("config", "Failed to send %s to PID %d: %s", signal, pid, error);
return false;
}
await setTimeout(retryDelayMs);
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const healthy = await client.isHealthy();
if (healthy) {
log.info("config", "Headscale is healthy after restart");
return true;
}
} catch {
// Still restarting
}
if (attempt < maxAttempts) {
await setTimeout(retryDelayMs);
}
}
log.error("config", "Headscale did not become healthy after %d attempts", maxAttempts);
return false;
}
+51 -162
View File
@@ -1,175 +1,64 @@
import { readdir, readFile } from 'node:fs/promises';
import { platform } from 'node:os';
import { join, resolve } from 'node:path';
import { kill } from 'node:process';
import { setTimeout } from 'node:timers/promises';
import { type } from 'arktype';
import type { RuntimeApiClient } from '~/server/headscale/api/endpoints';
import log from '~/utils/log';
import type { HeadplaneConfig } from '../config-schema';
import { Integration } from './abstract';
import { type } from "arktype";
import { platform } from "node:os";
import type { RuntimeApiClient } from "~/server/headscale/api/endpoints";
import log from "~/utils/log";
import { Integration } from "./abstract";
import { findHeadscaleServe, signalAndWaitHealthy } from "./proc-helper";
const configSchema = {
full: type({
enabled: 'boolean',
}),
full: type({
enabled: "boolean",
}),
partial: type({
enabled: 'boolean?',
}).partial(),
partial: type({
enabled: "boolean?",
}).partial(),
};
export default class ProcIntegration extends Integration<
typeof configSchema.full.infer
> {
private pid: number | undefined;
private maxAttempts = 10;
export default class ProcIntegration extends Integration<typeof configSchema.full.infer> {
private pid: number | undefined;
get name() {
return 'Native Linux (/proc)';
}
get name() {
return "Native Linux (/proc)";
}
static get configSchema() {
return configSchema;
}
static get configSchema() {
return configSchema;
}
async isAvailable() {
if (platform() !== 'linux') {
log.error('config', '/proc is only available on Linux');
return false;
}
async isAvailable() {
if (platform() !== "linux") {
log.error("config", "/proc is only available on Linux");
return false;
}
log.debug('config', 'Checking /proc for Headscale process');
const dir = resolve('/proc');
try {
const subdirs = await readdir(dir);
const promises = subdirs.map(async (dir) => {
const pid = Number.parseInt(dir, 10);
try {
const result = await findHeadscaleServe();
if (!result) {
log.error("config", "Could not find headscale serve process");
return false;
}
if (Number.isNaN(pid)) {
return;
}
this.pid = result;
log.info("config", "Found headscale serve (PID %d)", this.pid);
return true;
} catch (error) {
log.error("config", "Failed to scan /proc: %s", error);
return false;
}
}
const path = join('/proc', dir, 'comm');
try {
log.debug('config', 'Reading %s', path);
const data = await readFile(path, 'utf8');
if (data.trim() !== 'headscale') {
throw new Error(
`Found PID with unexpected command: ${data.trim()}`,
);
}
async onConfigChange(client: RuntimeApiClient) {
if (!this.pid) {
return;
}
return pid;
} catch (error) {
log.debug('config', 'Failed to read %s: %s', path, error);
}
});
const results = await Promise.allSettled(promises);
const pids = [];
for (const result of results) {
if (result.status === 'fulfilled' && result.value) {
pids.push(result.value);
}
}
log.debug('config', 'Found Headscale processes: %o', pids);
if (pids.length > 1) {
log.warn(
'config',
'Found %d Headscale processes: %s',
pids.length,
pids.join(', '),
);
log.debug(
'config',
'Checking if any of them have Parent PID = 1, assuming thats the correct PID',
);
const ppidRegex = /(?:PPid:\s)(\d+)(?:\n?)/;
for (const pid of pids) {
const pidStatusPath = join('/proc', pid.toString(), 'status');
try {
log.debug('config', 'Reading %s', pidStatusPath);
const pidData = await readFile(pidStatusPath, 'utf8');
const ppidResult = pidData.match(ppidRegex);
if (ppidResult !== null) {
const potentialPPid = Number.parseInt(ppidResult[1], 10);
if (potentialPPid === 1) {
this.pid = pid;
log.info(
'config',
'Found potential Headscale process with PID: %d based on Parent PID = 1',
this.pid,
);
return true;
}
}
} catch (error) {
log.error('config', 'Failed to read %s: %s', pidStatusPath, error);
}
}
return false;
}
if (pids.length === 0) {
log.error('config', 'Could not find Headscale process');
return false;
}
this.pid = pids[0];
log.info('config', 'Found Headscale process with PID: %d', this.pid);
return true;
} catch {
log.error('config', 'Failed to read /proc');
return false;
}
}
async onConfigChange(client: RuntimeApiClient) {
if (!this.pid) {
return;
}
try {
log.info('config', 'Sending SIGTERM to Headscale');
kill(this.pid, 'SIGTERM');
} catch (error) {
log.error('config', 'Failed to send SIGTERM to Headscale: %s', error);
log.debug('config', 'kill(1) error: %o', error);
}
await setTimeout(1000);
let attempts = 0;
while (attempts <= this.maxAttempts) {
try {
log.debug('config', 'Checking Headscale status (attempt %d)', attempts);
const status = await client.isHealthy();
if (status === false) {
log.error('config', 'Headscale is not running');
return;
}
log.info('config', 'Headscale is up and running');
return;
} catch {
if (attempts < this.maxAttempts) {
attempts++;
await setTimeout(1000);
continue;
}
log.error(
'config',
'Missed restart deadline for Headscale (pid %d)',
this.pid,
);
return;
}
}
}
await signalAndWaitHealthy(client, {
pid: this.pid,
signal: "SIGHUP",
});
}
}