feat: make api calls more resilient and stuff

app/server/config/integration/abstract.ts

@@ -1,4 +1,4 @@
-import type { HeadscaleClient } from "~/server/headscale/api";
+import type { Headscale } from "~/server/headscale/api";
 
 export abstract class Integration<T> {
   protected context: NonNullable<T>;
@@ -11,6 +11,6 @@ export abstract class Integration<T> {
   }
 
   abstract isAvailable(): Promise<boolean> | boolean;
-  abstract onConfigChange(client: HeadscaleClient): Promise<void> | void;
+  abstract onConfigChange(headscale: Headscale): Promise<void> | void;
   abstract get name(): string;
 }

app/server/config/integration/docker.ts

@@ -4,7 +4,7 @@ import { setTimeout } from "node:timers/promises";
 import { type } from "arktype";
 import { Client } from "undici";
 
-import type { HeadscaleClient } from "~/server/headscale/api";
+import type { Headscale } from "~/server/headscale/api";
 import log from "~/utils/log";
 
 import { Integration } from "./abstract";
@@ -255,7 +255,7 @@ export default class DockerIntegration extends Integration<typeof configSchema.f
     return this.client !== undefined && this.containerId !== undefined;
   }
 
-  async onConfigChange(client: HeadscaleClient) {
+  async onConfigChange(headscale: Headscale) {
     if (!this.client) {
       return;
     }
@@ -290,7 +290,7 @@ export default class DockerIntegration extends Integration<typeof configSchema.f
     while (attempts <= this.maxAttempts) {
       try {
         log.debug("config", "Checking Headscale status (attempt %d)", attempts);
-        const status = await client.isHealthy();
+        const status = await headscale.health();
         if (status === false) {
           throw new Error("Headscale is not running");
         }

app/server/config/integration/kubernetes.ts

@@ -5,7 +5,7 @@ import { join } from "node:path";
 import { CoreV1Api, KubeConfig } from "@kubernetes/client-node";
 import { type } from "arktype";
 
-import type { HeadscaleClient } from "~/server/headscale/api";
+import type { Headscale } from "~/server/headscale/api";
 import log from "~/utils/log";
 
 import { Integration } from "./abstract";
@@ -154,12 +154,12 @@ export default class KubernetesIntegration extends Integration<typeof configSche
     }
   }
 
-  async onConfigChange(client: HeadscaleClient) {
+  async onConfigChange(headscale: Headscale) {
     if (!this.pid) {
       return;
     }
 
-    await signalAndWaitHealthy(client, {
+    await signalAndWaitHealthy(headscale, {
       pid: this.pid,
       signal: "SIGHUP",
     });

app/server/config/integration/proc-helper.ts

@@ -3,7 +3,7 @@ import { join } from "node:path";
 import { kill } from "node:process";
 import { setTimeout } from "node:timers/promises";
 
-import type { HeadscaleClient } from "~/server/headscale/api";
+import type { Headscale } from "~/server/headscale/api";
 import log from "~/utils/log";
 
 /**
@@ -75,12 +75,12 @@ export interface SignalHeadscaleOptions {
 
 /**
  * Sends a signal to the headscale process and waits for it to become healthy.
- * @param client The HeadscaleClient to check health
+ * @param headscale The Headscale instance to health-check
  * @param options Options for signaling and waiting
  * @returns True if headscale became healthy, false otherwise
  */
 export async function signalAndWaitHealthy(
-  client: HeadscaleClient,
+  headscale: Headscale,
   options: SignalHeadscaleOptions,
 ): Promise<boolean> {
   const { pid, signal = "SIGHUP", maxAttempts = 10, retryDelayMs = 1000 } = options;
@@ -96,7 +96,7 @@ export async function signalAndWaitHealthy(
   await setTimeout(retryDelayMs);
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     try {
-      const healthy = await client.isHealthy();
+      const healthy = await headscale.health();
       if (healthy) {
         log.info("config", "Headscale is healthy after restart");
         return true;

app/server/config/integration/proc.ts

@@ -2,7 +2,7 @@ import { platform } from "node:os";
 
 import { type } from "arktype";
 
-import type { HeadscaleClient } from "~/server/headscale/api";
+import type { Headscale } from "~/server/headscale/api";
 import log from "~/utils/log";
 
 import { Integration } from "./abstract";
@@ -51,12 +51,12 @@ export default class ProcIntegration extends Integration<typeof configSchema.ful
     }
   }
 
-  async onConfigChange(client: HeadscaleClient) {
+  async onConfigChange(headscale: Headscale) {
     if (!this.pid) {
       return;
     }
 
-    await signalAndWaitHealthy(client, {
+    await signalAndWaitHealthy(headscale, {
       pid: this.pid,
       signal: "SIGHUP",
     });

app/server/headscale/api/index.ts

@@ -15,7 +15,7 @@ import { makePolicyApi, type PolicyApi } from "./resources/policy";
 import { makePreAuthKeyApi, type PreAuthKeyApi } from "./resources/pre-auth-keys";
 import { makeUserApi, type UserApi } from "./resources/users";
 import { formatServerVersion, parseServerVersion, type ServerVersion } from "./server-version";
-import { createTransport, type Transport } from "./transport";
+import { createTransport } from "./transport";
 
 export interface Headscale {
   readonly version: ServerVersion;
@@ -34,48 +34,84 @@ export interface HeadscaleClient {
   policy: PolicyApi;
   preAuthKeys: PreAuthKeyApi;
   apiKeys: ApiKeyApi;
-  /**
-   * Convenience passthrough to `Headscale.health()`. Headscale's
-   * `/health` endpoint is unauthenticated so callers that already
-   * have a client (loaders/actions) don't need to also reach for
-   * the top-level `Headscale` value.
-   */
-  isHealthy(): Promise<boolean>;
 }
 
 export interface CreateHeadscaleOptions {
   url: string;
   certPath?: string;
+  /**
+   * How often to retry `/version` while Headscale is unreachable.
+   * Defaults to 30 seconds. Exposed for tests.
+   */
+  retryIntervalMs?: number;
 }
 
+const DEFAULT_RETRY_INTERVAL_MS = 30_000;
+
 export async function createHeadscale(opts: CreateHeadscaleOptions): Promise<Headscale> {
   const transport = await createTransport({ url: opts.url, certPath: opts.certPath });
+  const retryIntervalMs = opts.retryIntervalMs ?? DEFAULT_RETRY_INTERVAL_MS;
 
-  const versionInfo = await transport.getPublic<{ version: string }>("/version");
-  const version = parseServerVersion(versionInfo.version);
-  const capabilities = capabilitiesFor(version);
+  let version: ServerVersion = parseServerVersion("unreachable");
+  let capabilities: Capabilities = capabilitiesFor(version);
+  let detected = false;
+  let retryTimer: ReturnType<typeof setTimeout> | undefined;
+  let disposed = false;
 
-  if (version.unknown) {
-    log.warn(
-      "api",
-      "Could not parse Headscale version %s, assuming newest known capabilities",
-      versionInfo.version,
-    );
-  } else {
-    log.info("api", "Connected to Headscale %s", formatServerVersion(version));
+  async function detectOnce(): Promise<boolean> {
+    try {
+      const { version: raw } = await transport.getPublic<{ version: string }>("/version");
+      const parsed = parseServerVersion(raw);
+      version = parsed;
+      capabilities = capabilitiesFor(parsed);
+      detected = true;
+      if (parsed.unknown) {
+        log.warn(
+          "api",
+          "Could not parse Headscale version %s, assuming newest known capabilities",
+          raw,
+        );
+      } else {
+        log.info("api", "Connected to Headscale %s", formatServerVersion(parsed));
+      }
+      return true;
+    } catch (error) {
+      log.debug("api", "Headscale /version probe failed: %s", String(error));
+      return false;
+    }
   }
 
-  return makeHeadscale(transport, version, capabilities);
-}
+  function scheduleRetry() {
+    if (disposed || detected) return;
+    retryTimer = setTimeout(async () => {
+      retryTimer = undefined;
+      if (disposed) return;
+      if (await detectOnce()) return;
+      scheduleRetry();
+    }, retryIntervalMs);
+    // Don't keep the event loop alive on this timer alone — Headplane
+    // should still shut down cleanly while we're waiting to retry.
+    retryTimer.unref?.();
+  }
+
+  if (!(await detectOnce())) {
+    log.warn(
+      "api",
+      "Headscale unreachable at boot; defaulting to newest-known capabilities and retrying every %dms",
+      retryIntervalMs,
+    );
+    scheduleRetry();
+  }
 
-function makeHeadscale(
-  transport: Transport,
-  version: ServerVersion,
-  capabilities: Capabilities,
-): Headscale {
   return {
-    version,
-    capabilities,
+    // Getters so callers always observe the latest detected values
+    // without having to know about the retry loop.
+    get version() {
+      return version;
+    },
+    get capabilities() {
+      return capabilities;
+    },
     health: () => transport.health(),
     client(apiKey) {
       return {
@@ -84,9 +120,15 @@ function makeHeadscale(
         policy: makePolicyApi(transport, capabilities, apiKey),
         preAuthKeys: makePreAuthKeyApi(transport, capabilities, apiKey),
         apiKeys: makeApiKeyApi(transport, capabilities, apiKey),
-        isHealthy: () => transport.health(),
       };
     },
-    dispose: () => transport.dispose(),
+    async dispose() {
+      disposed = true;
+      if (retryTimer) {
+        clearTimeout(retryTimer);
+        retryTimer = undefined;
+      }
+      await transport.dispose();
+    },
   };
 }