feat: respect context in server

server/context/app.ts

@@ -0,0 +1,12 @@
+import { hp_getConfig } from './loader';
+import { HeadplaneConfig } from './parser';
+
+export interface AppContext {
+	context: HeadplaneConfig;
+}
+
+export default function appContext() {
+	return {
+		context: hp_getConfig(),
+	};
+}

server/context/loader.ts

@@ -0,0 +1,237 @@
+import { constants, access, readFile } from 'node:fs/promises';
+import { type } from 'arktype';
+import { parseDocument } from 'yaml';
+import { testOidc } from '~/utils/oidc';
+import log, { hpServer_loadLogger } from '~server/utils/log';
+import mutex from '~server/utils/mutex';
+import { HeadplaneConfig, coalesceConfig, validateConfig } from './parser';
+
+const envBool = type('string | undefined').pipe((v) => {
+	return ['1', 'true', 'yes', 'on'].includes(v?.toLowerCase() ?? '');
+});
+
+const rootEnvs = type({
+	HEADPLANE_DEBUG_LOG: envBool,
+	HEADPLANE_LOAD_ENV_FILE: envBool,
+	HEADPLANE_LOAD_ENV_OVERRIDES: envBool,
+	HEADPLANE_CONFIG_PATH: 'string | undefined',
+}).onDeepUndeclaredKey('reject');
+
+const HEADPLANE_DEFAULT_CONFIG_PATH = '/etc/headplane/config.yaml';
+let runtimeConfig: HeadplaneConfig | undefined = undefined;
+const runtimeLock = mutex();
+
+// We need to acquire here to ensure that the configuration is loaded
+// properly. We can't request a configuration if its in the process
+// of being updated.
+export function hp_getConfig() {
+	runtimeLock.acquire();
+	if (!runtimeConfig) {
+		runtimeLock.release();
+		// This shouldn't be possible, we NEED to have a configuration
+		throw new Error('Configuration not loaded');
+	}
+
+	const config = runtimeConfig;
+
+	runtimeLock.release();
+	return config;
+}
+
+// hp_loadConfig should ONLY be called when we explicitly need to reload
+// the configuration. This should be done when the configuration file
+// changes and we ignore environment variable changes.
+//
+// To read the config hp_getConfig should be used.
+// TODO: File watching for hp_loadConfig()
+export async function hp_loadConfig() {
+	runtimeLock.acquire();
+	let path = HEADPLANE_DEFAULT_CONFIG_PATH;
+
+	const envs = rootEnvs({
+		HEADPLANE_DEBUG_LOG: process.env.HEADPLANE_DEBUG_LOG,
+		HEADPLANE_CONFIG_PATH: process.env.HEADPLANE_CONFIG_PATH,
+		HEADPLANE_LOAD_ENV_FILE: process.env.HEADPLANE_LOAD_ENV_FILE,
+		HEADPLANE_LOAD_ENV_OVERRIDES: process.env.HEADPLANE_LOAD_ENV_OVERRIDES,
+	});
+
+	if (envs instanceof type.errors) {
+		log.error('CFGX', 'Error parsing environment variables:');
+		for (const [number, error] of envs.entries()) {
+			log.error('CFGX', ` (${number}): ${error.toString()}`);
+		}
+
+		return;
+	}
+
+	// Load our debug based logger before ANYTHING
+	hpServer_loadLogger(envs.HEADPLANE_DEBUG_LOG);
+
+	if (envs.HEADPLANE_CONFIG_PATH) {
+		path = envs.HEADPLANE_CONFIG_PATH;
+	}
+
+	await validateConfigPath(path);
+	const rawConfig = await loadConfigFile(path);
+	if (!rawConfig) {
+		log.error('CFGX', 'Failed to load Headplane configuration file');
+		process.exit(1);
+	}
+
+	let config = validateConfig({
+		...rawConfig,
+		debug: envs.HEADPLANE_DEBUG_LOG,
+	});
+
+	if (envs.HEADPLANE_LOAD_ENV_FILE) {
+		log.info('CFGX', 'Loading a .env file if one exists');
+		await import('dotenv/config');
+	}
+
+	if (config && envs.HEADPLANE_LOAD_ENV_OVERRIDES) {
+		log.info(
+			'CFGX',
+			'Loading environment variables to override the configuration',
+		);
+		config = coalesceEnv(config);
+	}
+
+	if (!config) {
+		runtimeLock.release();
+		log.error('CFGX', 'Fatal error encountered with configuration');
+		process.exit(1);
+	}
+
+	if (config.oidc?.strict_validation) {
+		testOidc(config.oidc);
+	}
+
+	runtimeConfig = config;
+	runtimeLock.release();
+}
+
+async function validateConfigPath(path: string) {
+	log.debug('CFGX', `Validating Headplane configuration file at ${path}`);
+	try {
+		await access(path, constants.F_OK | constants.R_OK);
+		log.info('CFGX', `Headplane configuration found at ${path}`);
+		return true;
+	} catch (e) {
+		log.error('CFGX', `Headplane configuration not readable at ${path}`);
+		log.error('CFGX', `${e}`);
+		return false;
+	}
+}
+
+async function loadConfigFile(path: string) {
+	log.debug('CFGX', `Loading Headplane configuration file at ${path}`);
+	try {
+		const data = await readFile(path, 'utf8');
+		const configYaml = parseDocument(data);
+		if (configYaml.errors.length > 0) {
+			log.error(
+				'CFGX',
+				`Error parsing Headplane configuration file at ${path}`,
+			);
+			for (const error of configYaml.errors) {
+				log.error('CFGX', `  ${error.toString()}`);
+			}
+
+			return;
+		}
+
+		if (configYaml.warnings.length > 0) {
+			log.warn(
+				'CFGX',
+				`Warnings parsing Headplane configuration file at ${path}`,
+			);
+			for (const warning of configYaml.warnings) {
+				log.warn('CFGX', `  ${warning.toString()}`);
+			}
+		}
+
+		return configYaml.toJSON() as unknown;
+	} catch (e) {
+		log.error('CFGX', `Error reading Headplane configuration file at ${path}`);
+		log.error('CFGX', `${e}`);
+		return;
+	}
+}
+
+function coalesceEnv(config: HeadplaneConfig) {
+	const envConfig: Record<string, unknown> = {};
+	const rootKeys: string[] = rootEnvs.props.map((prop) => prop.key);
+
+	// Typescript is still insanely stupid at nullish filtering
+	const vars = Object.entries(process.env).filter(([key, value]) => {
+		if (!value) {
+			return false;
+		}
+
+		if (!key.startsWith('HEADPLANE_')) {
+			return false;
+		}
+
+		// Filter out the rootEnv configurations
+		if (rootKeys.includes(key)) {
+			return false;
+		}
+
+		return true;
+	}) as [string, string][];
+
+	log.debug('CFGX', `Coalescing ${vars.length} environment variables`);
+	for (const [key, value] of vars) {
+		const configPath = key.replace('HEADPLANE_', '').toLowerCase().split('__');
+		log.debug('CFGX', `  ${key}=${new Array(value.length).fill('*').join('')}`);
+
+		let current = envConfig;
+		while (configPath.length > 1) {
+			const path = configPath.shift() as string;
+			if (!(path in current)) {
+				current[path] = {};
+			}
+
+			current = current[path] as Record<string, unknown>;
+		}
+
+		current[configPath[0]] = value;
+	}
+
+	const toMerge = coalesceConfig(envConfig);
+	if (!toMerge) {
+		return;
+	}
+
+	// Deep merge the environment variables into the configuration
+	// This will overwrite any existing values in the configuration
+	return deepMerge(config, toMerge);
+}
+
+type DeepPartial<T> =
+	| {
+			[P in keyof T]?: DeepPartial<T[P]>;
+	  }
+	| undefined;
+
+function deepMerge<T>(target: T, source: DeepPartial<T>): T {
+	if (typeof target !== 'object' || typeof source !== 'object')
+		return source as T;
+	const result = { ...target } as T;
+
+	for (const key in source) {
+		const val = source[key];
+		if (val === undefined) {
+			continue;
+		}
+
+		if (typeof val === 'object') {
+			result[key] = deepMerge(result[key], val);
+			continue;
+		}
+
+		result[key] = val;
+	}
+
+	return result;
+}

server/context/parser.ts

@@ -0,0 +1,76 @@
+import { type } from 'arktype';
+import log from '~server/utils/log';
+
+export type HeadplaneConfig = typeof headplaneConfig.infer;
+const stringToBool = type('string | boolean').pipe((v) => Boolean(v));
+const serverConfig = type({
+	host: 'string.ip',
+	port: type('string | number.integer').pipe((v) => Number(v)),
+	cookie_secret: '32 <= string <= 32',
+	cookie_secure: stringToBool,
+});
+
+const oidcConfig = type({
+	issuer: 'string.url',
+	client_id: 'string',
+	client_secret: 'string',
+	token_endpoint_auth_method:
+		'"client_secret_basic" | "client_secret_post" | "client_secret_jwt"',
+	redirect_uri: 'string.url?',
+	disable_api_key_login: stringToBool,
+	headscale_api_key: 'string',
+	strict_validation: stringToBool.default(true),
+}).onDeepUndeclaredKey('reject');
+
+const headscaleConfig = type({
+	url: 'string.url',
+	public_url: 'string.url?',
+	config_path: 'string?',
+	config_strict: stringToBool,
+}).onDeepUndeclaredKey('reject');
+
+const headplaneConfig = type({
+	debug: stringToBool,
+	server: serverConfig,
+	'oidc?': oidcConfig,
+	headscale: headscaleConfig,
+}).onDeepUndeclaredKey('reject');
+
+const partialHeadplaneConfig = type({
+	debug: stringToBool,
+	server: serverConfig.partial(),
+	'oidc?': oidcConfig.partial(),
+	headscale: headscaleConfig.partial(),
+}).partial();
+
+export function validateConfig(config: unknown) {
+	log.debug('CFGX', 'Validating Headplane configuration...');
+	const out = headplaneConfig(config);
+	if (out instanceof type.errors) {
+		log.error('CFGX', 'Error parsing Headplane configuration:');
+		for (const [number, error] of out.entries()) {
+			log.error('CFGX', ` (${number}): ${error.toString()}`);
+		}
+
+		return;
+	}
+
+	log.debug('CFGX', 'Headplane configuration is valid.');
+	return out;
+}
+
+export function coalesceConfig(config: unknown) {
+	log.debug('CFGX', 'Validating coalescing vars for configuration...');
+	const out = partialHeadplaneConfig(config);
+	if (out instanceof type.errors) {
+		log.error('CFGX', 'Error parsing variables:');
+		for (const [number, error] of out.entries()) {
+			log.error('CFGX', ` (${number}): ${error.toString()}`);
+		}
+
+		return;
+	}
+
+	log.debug('CFGX', 'Coalescing variables is valid.');
+	return out;
+}

server/dev/dev-handler.ts

@@ -1,7 +1,7 @@
-import { createRequestHandler } from 'react-router'
+import { createRequestHandler } from 'react-router';
 
 export default createRequestHandler(
 	// @ts-expect-error: React Router Vite plugin
 	() => import('virtual:react-router/server-build'),
-	'development'
+	'development',
 );

server/dev/hot-server.ts

@@ -1,23 +1,23 @@
-import log from '~server/log';
-import { createServer, type ViteDevServer } from 'vite';
-import { type createRequestHandler } from 'react-router';
+import { type ViteDevServer, createServer } from 'vite';
+import log from '~server/utils/log';
 
+// TODO: Remove env.NODE_ENV
 let server: ViteDevServer | undefined;
 export async function loadDevtools() {
-	log.info('DEVX', 'Starting Vite Development server')
+	log.info('DEVX', 'Starting Vite Development server');
 	process.env.NODE_ENV = 'development';
 
 	// This is loading the ROOT vite.config.ts
 	server = await createServer({
 		server: {
 			middlewareMode: true,
-		}
+		},
 	});
 
 	// We can't just do ssrLoadModule for virtual:react-router/server-build
 	// because for hot reload to work server side it needs to be imported
 	// using builtin import in its own file.
-	const handler = await server.ssrLoadModule('./server/dev-handler.ts');
+	const handler = await server.ssrLoadModule('./server/dev/dev-handler.ts');
 	return {
 		server,
 		handler: handler.default,
@@ -26,11 +26,11 @@ export async function loadDevtools() {
 
 export async function stacksafeTry(
 	devtools: {
-		server: ViteDevServer,
-		handler: any, // import() is dynamic
+		server: ViteDevServer;
+		handler: (req: Request, context: unknown) => Promise<Response>;
 	},
 	req: Request,
-	context: unknown
+	context: unknown,
 ) {
 	try {
 		const result = await devtools.handler(req, context);
@@ -38,7 +38,6 @@ export async function stacksafeTry(
 	} catch (error) {
 		log.error('DEVX', 'Error in request handler', error);
 		if (typeof error === 'object' && error instanceof Error) {
-			console.log('got error');
 			devtools.server.ssrFixStacktrace(error);
 		}
 

server/entry.ts

@@ -1,8 +1,9 @@
+// import { initWebsocket } from '~server/ws';
+import { constants, access } from 'node:fs/promises';
 import { createServer } from 'node:http';
+import { hp_getConfig, hp_loadConfig } from '~server/context/loader';
 import { listener } from '~server/listener';
-import { initWebsocket } from '~server/ws';
-import { access, constants } from 'node:fs/promises';
-import log from '~server/log';
+import log from '~server/utils/log';
 
 log.info('SRVX', 'Running Node.js %s', process.versions.node);
 
@@ -15,21 +16,26 @@ try {
 	process.exit(1);
 }
 
+await hp_loadConfig();
 const server = createServer(listener);
-const port = process.env.PORT || 3000;
-const host = process.env.HOST || '0.0.0.0';
 
-const ws = initWebsocket();
-if (ws) {
-	server.on('upgrade', (req, socket, head) => {
-		ws.handleUpgrade(req, socket, head, (ws) => {
-			ws.emit('connection', ws, req);
-		});
-	});
-}
+// const ws = initWebsocket();
+// if (ws) {
+// 	server.on('upgrade', (req, socket, head) => {
+// 		ws.handleUpgrade(req, socket, head, (ws) => {
+// 			ws.emit('connection', ws, req);
+// 		});
+// 	});
+// }
 
-server.listen(Number(port), host, () => {
-	log.info('SRVX', 'Running on %s:%s', host, port);
+const context = hp_getConfig();
+server.listen(context.server.port, context.server.host, () => {
+	log.info(
+		'SRVX',
+		'Running on %s:%s',
+		context.server.host,
+		context.server.port,
+	);
 });
 
 if (import.meta.hot) {
@@ -41,5 +47,3 @@ if (import.meta.hot) {
 		server.close();
 	});
 }
-
-// export const app = listener;

server/listener.ts

@@ -1,17 +1,13 @@
-import { type RequestListener } from 'node:http';
-import { resolve, join } from 'node:path'
-import { createServer } from 'vite'
-import { createRequestHandler } from 'react-router'
-import { access, constants } from 'node:fs/promises';
 import { createReadStream, existsSync, statSync } from 'node:fs';
+import { type RequestListener } from 'node:http';
+import { join, resolve } from 'node:path';
 import {
 	createReadableStreamFromReadable,
 	writeReadableStreamToWritable,
 } from '@react-router/node';
-import mime from 'mime/lite'
-
-import { loadDevtools, stacksafeTry } from '~server/dev';
-import { appContext } from '~server/ws';
+import mime from 'mime/lite';
+import appContext from '~server/context/app';
+import { loadDevtools, stacksafeTry } from '~server/dev/hot-server';
 import prodBuild from '~server/prod-handler';
 
 declare global {
@@ -19,13 +15,9 @@ declare global {
 	const __hp_prefix: string;
 }
 
-const devtools = import.meta.env.DEV
-	? await loadDevtools()
-	: undefined;
+const devtools = import.meta.env.DEV ? await loadDevtools() : undefined;
 
-const prodHandler = import.meta.env.PROD
-	? await prodBuild()
-	: undefined;
+const prodHandler = import.meta.env.PROD ? await prodBuild() : undefined;
 
 const buildPath = process.env.BUILD_PATH ?? './build';
 const baseDir = resolve(join(buildPath, 'client'));
@@ -113,15 +105,22 @@ export const listener: RequestListener = async (req, res) => {
 		// If we have a body, we set the duplex and load it
 		...(req.method !== 'GET' && req.method !== 'HEAD'
 			? {
-				body: createReadableStreamFromReadable(req),
-				duplex: 'half',
-			} : {}),
+					body: createReadableStreamFromReadable(req),
+					duplex: 'half',
+				}
+			: {}),
 	});
 
 	const response = devtools
 		? await stacksafeTry(devtools, frameworkReq, appContext())
 		: await prodHandler?.(frameworkReq, appContext());
 
+	if (!response) {
+		res.writeHead(404);
+		res.end();
+		return;
+	}
+
 	res.statusCode = response.status;
 	res.statusMessage = response.statusText;
 
@@ -134,4 +133,4 @@ export const listener: RequestListener = async (req, res) => {
 	}
 
 	res.end();
-}
+};

server/prod-handler.ts

@@ -1,9 +1,9 @@
-import { createRequestHandler } from 'react-router'
-import { access, constants } from 'node:fs/promises';
+import { constants, access } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
-import log from '~server/log';
+import { createRequestHandler } from 'react-router';
+import log from '~server/utils/log';
 
-export default async function() {
+export default async function () {
 	const buildPath = process.env.BUILD_PATH ?? './build';
 	const server = resolve(join(buildPath, 'server'));
 
@@ -12,7 +12,10 @@ export default async function() {
 		log.info('SRVX', 'Using build directory %s', resolve(buildPath));
 	} catch (error) {
 		log.error('SRVX', 'No build found. Please refer to the documentation');
-		log.error('SRVX', 'https://github.com/tale/headplane/blob/main/docs/integration/Native.md');
+		log.error(
+			'SRVX',
+			'https://github.com/tale/headplane/blob/main/docs/integration/Native.md',
+		);
 		console.error(error);
 		process.exit(1);
 	}

server/utils/log.ts

@@ -1,4 +1,22 @@
-export default {
+export function hpServer_loadLogger(debug: boolean) {
+	if (debug) {
+		log.debug = (category: string, message: string, ...args: unknown[]) => {
+			defaultLog('DEBG', category, message, ...args);
+		};
+
+		log.info('CFGX', 'Debug logging enabled');
+		log.info(
+			'CFGX',
+			'This is very verbose and should only be used for debugging purposes',
+		);
+		log.info(
+			'CFGX',
+			'If you run this in production, your storage COULD fill up quickly',
+		);
+	}
+}
+
+const log = {
 	info: (category: string, message: string, ...args: unknown[]) => {
 		defaultLog('INFO', category, message, ...args);
 	},
@@ -11,11 +29,7 @@ export default {
 		defaultLog('ERRO', category, message, ...args);
 	},
 
-	debug: (category: string, message: string, ...args: unknown[]) => {
-		if (process.env.DEBUG === 'true') {
-			defaultLog('DEBG', category, message, ...args);
-		}
-	},
+	debug: (category: string, message: string, ...args: unknown[]) => {},
 };
 
 function defaultLog(
@@ -27,3 +41,5 @@ function defaultLog(
 	const date = new Date().toISOString();
 	console.log(`${date} (${level}) [${category}] ${message}`, ...args);
 }
+
+export default log;

server/utils/mutex.ts

@@ -0,0 +1,32 @@
+class Mutex {
+	private locked = false;
+	private queue: (() => void)[] = [];
+
+	constructor(locked: boolean) {
+		this.locked = locked;
+	}
+
+	acquire() {
+		return new Promise<void>((resolve) => {
+			if (!this.locked) {
+				this.locked = true;
+				resolve();
+			} else {
+				this.queue.push(resolve);
+			}
+		});
+	}
+
+	release() {
+		if (this.queue.length > 0) {
+			const next = this.queue.shift();
+			next?.();
+		} else {
+			this.locked = false;
+		}
+	}
+}
+
+export default function mutex(locked = false) {
+	return new Mutex(locked);
+}

server/utils/ws.ts

@@ -1,5 +1,5 @@
-import WebSocket, { WebSocketServer } from 'ws'
-import log from '~server/log'
+import WebSocket, { WebSocketServer } from 'ws';
+import log from '~server/utils/log';
 
 const server = new WebSocketServer({ noServer: true });
 export function initWebsocket() {
@@ -13,6 +13,7 @@ export function initWebsocket() {
 
 	log.info('CACH', 'Initializing agent WebSocket');
 	server.on('connection', (ws, req) => {
+		// biome-ignore lint: this file is not USED
 		const auth = req.headers['authorization'];
 		if (auth !== `Bearer ${key}`) {
 			log.warn('CACH', 'Invalid agent WebSocket connection');
@@ -20,7 +21,6 @@ export function initWebsocket() {
 			return;
 		}
 
-
 		const nodeID = req.headers['x-headplane-ts-node-id'];
 		if (!nodeID) {
 			log.warn('CACH', 'Invalid agent WebSocket connection');
@@ -46,7 +46,7 @@ export function initWebsocket() {
 			log.error('CACH', 'Closing agent WebSocket connection');
 			log.error('CACH', 'Agent WebSocket error: %s', error);
 			ws.close(1011, 'ERR_INTERNAL_ERROR');
-		})
+		});
 	});
 
 	return server;
@@ -56,5 +56,5 @@ export function appContext() {
 	return {
 		ws: server,
 		wsAuthKey: process.env.LOCAL_AGENT_AUTHKEY,
-	}
+	};
 }