test: move the remaining integration tests onto the harness

Supported versions now live in one place, and CI runs every one of them.

.github/workflows/build.yaml

@@ -52,6 +52,8 @@ jobs:
 
       - name: Run integration tests
         run: pnpm run test:integration
+        env:
+          HP_HEADSCALE_VERSIONS: all
 
   nix:
     name: Nix Flake Check

package.json

@@ -14,7 +14,7 @@
     "pretypecheck": "cn build -q --content 'app/**/*.{ts,tsx}' --css app/tailwind.css -o app/utils/cn-tables.ts",
     "typecheck": "react-router typegen && tsc",
     "test:unit": "vitest run --project unit",
-    "test:integration": "vitest run --project integration:*",
+    "test:integration": "vitest run --project integration --project integration:agent",
     "docs:dev": "cd docs && react-router dev",
     "docs:build": "cd docs && react-router build && node scripts/finish-build.ts",
     "lint": "oxlint",

tests/harness/dex.ts

@@ -1,12 +1,14 @@
 import tc from "testcontainers";
 
+import { DEX_IMAGE } from "./versions";
+
 export interface DexEnv {
   container: tc.StartedTestContainer;
   issuerUrl: string;
 }
 
 export async function startDex(): Promise<DexEnv> {
-  const container = await new tc.GenericContainer("dexidp/dex:v2.41.1")
+  const container = await new tc.GenericContainer(DEX_IMAGE)
     .withExposedPorts(5556)
     .withEnvironment({
       DEX_ISSUER: "http://0.0.0.0:5556",

tests/integration/agent/faults.test.ts

@@ -7,9 +7,11 @@ import { afterEach, describe, expect, inject, test, vi } from "vitest";
 
 import type { AgentManager } from "~/server/hp-agent";
 
+import { testedVersions } from "../../harness/versions";
 import { agentPids, agentProcessCount, createHarness, waitForSync } from "./harness";
 
-const HS_VERSIONS = ["0.28.0", "0.29.5"];
+// The agent needs tag-only keys, which arrived in 0.28.
+const HS_VERSIONS = testedVersions().filter((version) => version !== "0.27.1");
 
 const SYNC_DEADLINE_MS = 45_000;
 

tests/integration/agent/harness.ts

@@ -10,7 +10,8 @@ import { expect, inject, vi } from "vitest";
 import { createHeadscale } from "~/server/headscale/api";
 import { type AgentManager, createAgentManager } from "~/server/hp-agent";
 
-import { startHeadscale } from "../setup/start-headscale";
+import { startHeadscale } from "../../harness/headscale";
+import type { HeadscaleVersion } from "../../harness/versions";
 import { createFaultProxy } from "./fault-proxy";
 
 const HOST_NAME = "headplane-agent";
@@ -38,12 +39,12 @@ export interface AgentOverrides {
   workDir?: string;
 }
 
-export async function createHarness(version: string, options: HarnessOptions = {}) {
-  const hs = await startHeadscale(version);
-  const proxy = await createFaultProxy(new URL(hs.apiUrl));
+export async function createHarness(version: HeadscaleVersion, options: HarnessOptions = {}) {
+  const hs = await startHeadscale({ version, id: `agent-${process.pid}` });
+  const proxy = await createFaultProxy(new URL(hs.url));
 
-  const inspectorKey = await createApiKey(hs.container);
-  const inspector = await createHeadscale({ url: hs.apiUrl });
+  const inspectorKey = await hs.createApiKey();
+  const inspector = await createHeadscale({ url: hs.url });
   const api = inspector.client(inspectorKey);
 
   const headscale = await createHeadscale({ url: proxy.url, retryIntervalMs: 500 });
@@ -150,7 +151,7 @@ export async function createHarness(version: string, options: HarnessOptions = {
     await proxy.close();
     await headscale.dispose();
     await inspector.dispose();
-    await hs.container.stop({ remove: true, removeVolumes: true });
+    await hs.stop();
     await rm(workDir, { recursive: true, force: true });
   }
 
@@ -169,15 +170,6 @@ export async function createHarness(version: string, options: HarnessOptions = {
   };
 }
 
-async function createApiKey(container: Awaited<ReturnType<typeof startHeadscale>>["container"]) {
-  const exec = await container.exec(["headscale", "apikeys", "create", "-o", "json"]);
-  if (exec.exitCode !== 0) {
-    throw new Error(`headscale apikeys create failed:\n${exec.stderr}`);
-  }
-
-  return JSON.parse(exec.stdout.toString()) as string;
-}
-
 export async function waitForSync(manager: AgentManager, timeoutMs = 60_000) {
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {

tests/integration/agent/lifecycle.test.ts

@@ -1,8 +1,10 @@
 import { afterEach, describe, expect, test, vi } from "vitest";
 
+import { testedVersions } from "../../harness/versions";
 import { createHarness, waitForSync } from "./harness";
 
-const HS_VERSIONS = ["0.28.0", "0.29.5"];
+// The agent needs tag-only keys, which arrived in 0.28.
+const HS_VERSIONS = testedVersions().filter((version) => version !== "0.27.1");
 
 describe.each(HS_VERSIONS)("agent lifecycle on Headscale %s", (version) => {
   let harness: Awaited<ReturnType<typeof createHarness>> | undefined;

tests/integration/api/api-keys.test.ts

@@ -1,12 +0,0 @@
-import { describe, expect, test } from "vitest";
-
-import { getRuntimeClient, HS_VERSIONS } from "../setup/env";
-
-describe.for(HS_VERSIONS)("Headscale %s: API Keys", (version) => {
-  test("api keys can be fetched", async () => {
-    const client = await getRuntimeClient(version);
-    const apiKeys = await client.apiKeys.list();
-    expect(Array.isArray(apiKeys)).toBe(true);
-    expect(apiKeys.length).toBe(1);
-  });
-});

tests/integration/api/nodes.test.ts

@@ -1,143 +0,0 @@
-import { RouterContextProvider } from "react-router";
-import { describe, expect, test, vi } from "vitest";
-
-import { machineAction } from "~/routes/machines/machine-actions";
-import { authContext, headscaleLiveStoreContext, requestApiContext } from "~/server/context";
-import { Capabilities } from "~/server/web/roles";
-
-import { getBootstrapClient, getNode, getRuntimeClient, HS_VERSIONS } from "../setup/env";
-
-function registerRequest(registerKey: string) {
-  const form = new FormData();
-  form.set("intent", "register");
-  form.set("register_key", registerKey);
-  form.set("user", "node-reg@");
-
-  return new Request("http://headplane.test/machines", {
-    method: "POST",
-    body: form,
-  });
-}
-
-function actionContext(api: Awaited<ReturnType<typeof getRuntimeClient>>) {
-  const auth = { can: vi.fn(() => true) };
-  const liveStore = { refresh: vi.fn() };
-  const principal = { id: "integration-user" };
-  const context = new RouterContextProvider();
-
-  context.set(authContext, auth as never);
-  context.set(headscaleLiveStoreContext, liveStore as never);
-  context.set(requestApiContext, vi.fn(async () => ({ principal, api })) as never);
-
-  return { auth, context, liveStore };
-}
-
-describe.for(HS_VERSIONS)("Headscale %s: Users", { concurrent: false }, (version) => {
-  let workingNodeId: string;
-
-  test("nodes can register from a Tailscale registration URL", async () => {
-    const client = await getRuntimeClient(version);
-    const tailnetNode = await getNode(version);
-    const { auth, context, liveStore } = actionContext(client);
-
-    const user = await client.users.create({ name: "node-reg@" });
-    expect(user.name).toBe("node-reg@");
-
-    const response = await machineAction({
-      request: registerRequest(tailnetNode.registerUrl),
-      context,
-      params: {},
-    } as never);
-
-    expect(auth.can).toHaveBeenCalledWith(expect.anything(), Capabilities.write_machines);
-    expect(liveStore.refresh).toHaveBeenCalledOnce();
-    expect(response).toBeInstanceOf(Response);
-    expect((response as Response).status).toBe(302);
-
-    const nodes = await client.nodes.list();
-    const node = nodes.find((n) => n.name === tailnetNode.nodeName);
-    expect(node).toBeDefined();
-    expect(node?.registerMethod).toBe("REGISTER_METHOD_CLI");
-  });
-
-  test("nodes can be retrieved", async () => {
-    const client = await getRuntimeClient(version);
-    const { nodeName } = await getNode(version);
-    const nodes = await client.nodes.list();
-    const node = nodes.find((n) => n.name === nodeName);
-    expect(node).toBeDefined();
-    expect(node?.name).toBe(nodeName);
-
-    const fetchedNode = await client.nodes.get(node!.id);
-    expect(fetchedNode).toBeDefined();
-    expect(fetchedNode.id).toBe(node!.id);
-    workingNodeId = node!.id;
-  });
-
-  test("nodes can be renamed", async () => {
-    const client = await getRuntimeClient(version);
-    const { nodeName } = await getNode(version);
-    const newName = `${nodeName}-renamed`;
-
-    await client.nodes.rename(workingNodeId, newName);
-    const renamedNode = await client.nodes.get(workingNodeId);
-    expect(renamedNode).toBeDefined();
-    expect(renamedNode.givenName).toBe(newName);
-  });
-
-  test("nodes can be reassigned to another user", async (context) => {
-    const bootstrap = await getBootstrapClient(version);
-    // Reassigning a node owner was removed in 0.28.
-    if (bootstrap.capabilities.nodeOwnerIsImmutable) {
-      context.skip();
-    }
-
-    const client = await getRuntimeClient(version);
-    const user = await client.users.create({ name: "node-reassign@" });
-
-    // reassignUser is only defined on pre-0.28 clients, hence the guard above.
-    await client.nodes.reassignUser!(workingNodeId, user.id);
-    const reassignedNode = await client.nodes.get(workingNodeId);
-    expect(reassignedNode).toBeDefined();
-    expect(reassignedNode.user?.name).toBe(user.name);
-  });
-
-  test("nodes can be expired", async () => {
-    const client = await getRuntimeClient(version);
-    await client.nodes.expire(workingNodeId);
-
-    const expiredNode = await client.nodes.get(workingNodeId);
-    expect(expiredNode).toBeDefined();
-    expect(expiredNode.expiry).toBeDefined();
-  });
-
-  test("key expiry of nodes can be toggled", async (context) => {
-    const bootstrap = await getBootstrapClient(version);
-    // Key expiry was introduced in 0.29.0
-    if (!bootstrap.capabilities.keyExpiryCanBeDisabled) {
-      context.skip();
-    }
-
-    const client = await getRuntimeClient(version);
-    await client.nodes.toggleExpiry(workingNodeId, true);
-
-    const permanentNode = await client.nodes.get(workingNodeId);
-    expect(permanentNode).toBeDefined();
-    expect(permanentNode.expiry).toBeNull();
-
-    await client.nodes.toggleExpiry(workingNodeId, false);
-
-    const node = await client.nodes.get(workingNodeId);
-    expect(node).toBeDefined();
-    expect(node.expiry).not.toBeNull();
-  });
-
-  test("nodes can be deleted", async () => {
-    const client = await getRuntimeClient(version);
-    await client.nodes.delete(workingNodeId);
-
-    const nodes = await client.nodes.list();
-    const node = nodes.find((n) => n.id === workingNodeId);
-    expect(node).toBeUndefined();
-  });
-});

tests/integration/api/pre-auth-keys.test.ts

@@ -1,141 +0,0 @@
-import { describe, expect, test } from "vitest";
-
-import { getBootstrapClient, getRuntimeClient, HS_VERSIONS } from "../setup/env";
-
-describe.for(HS_VERSIONS)("Headscale %s: Pre-auth Keys", { concurrent: false }, (version) => {
-  test("pre-auth keys can be created", async () => {
-    const client = await getRuntimeClient(version);
-    const preAuthKeyUser = await client.users.create({ name: "preauthkeyuser@" });
-    expect(preAuthKeyUser).toBeDefined();
-    expect(preAuthKeyUser.name).toBe("preauthkeyuser@");
-
-    const expiry = new Date(Date.now() + 3600 * 1000);
-    const preAuthKey = await client.preAuthKeys.create({
-      user: preAuthKeyUser.id,
-      ephemeral: false,
-      reusable: false,
-      expiration: expiry,
-      aclTags: null,
-    });
-
-    expect(preAuthKey).toBeDefined();
-    expect(preAuthKey.user?.id).toBe(preAuthKeyUser.id);
-    expect(preAuthKey.ephemeral).toBe(false);
-    expect(preAuthKey.reusable).toBe(false);
-    expect(preAuthKey.aclTags).toEqual([]);
-    expect(new Date(preAuthKey.expiration).getTime()).toBeCloseTo(expiry.getTime(), -2);
-  });
-
-  test("pre-auth keys can be created with ACL tags", async () => {
-    const client = await getRuntimeClient(version);
-    const [preAuthKeyUser] = await client.users.list({ name: "preauthkeyuser@" });
-    expect(preAuthKeyUser).toBeDefined();
-    expect(preAuthKeyUser.name).toBe("preauthkeyuser@");
-
-    const aclTags = ["tag:test1", "tag:test2"];
-    const preAuthKey = await client.preAuthKeys.create({
-      user: preAuthKeyUser.id,
-      ephemeral: true,
-      reusable: true,
-      expiration: null,
-      aclTags,
-    });
-
-    expect(preAuthKey).toBeDefined();
-    expect(preAuthKey.user?.id).toBe(preAuthKeyUser.id);
-    expect(preAuthKey.ephemeral).toBe(true);
-    expect(preAuthKey.reusable).toBe(true);
-    expect(preAuthKey.aclTags.sort()).toEqual(aclTags.sort());
-  });
-
-  test("tag-only pre-auth keys (0.28+)", async (context) => {
-    const bootstrap = await getBootstrapClient(version);
-    if (!bootstrap.capabilities.preAuthKeysHaveStableIds) {
-      context.skip();
-    }
-
-    const client = await getRuntimeClient(version);
-    const aclTags = ["tag:server", "tag:prod"];
-    const preAuthKey = await client.preAuthKeys.create({
-      user: null,
-      ephemeral: false,
-      reusable: true,
-      expiration: null,
-      aclTags,
-    });
-
-    expect(preAuthKey).toBeDefined();
-    expect(preAuthKey.user).toBeNull();
-    expect(preAuthKey.ephemeral).toBe(false);
-    expect(preAuthKey.reusable).toBe(true);
-    expect(preAuthKey.aclTags.sort()).toEqual(aclTags.sort());
-  });
-
-  test("pre-auth keys can be listed", async () => {
-    const client = await getRuntimeClient(version);
-    const [preAuthKeyUser] = await client.users.list({ name: "preauthkeyuser@" });
-    expect(preAuthKeyUser).toBeDefined();
-    expect(preAuthKeyUser.name).toBe("preauthkeyuser@");
-
-    const preAuthKeys = await client.preAuthKeys.listForUser(preAuthKeyUser.id);
-    expect(Array.isArray(preAuthKeys)).toBe(true);
-    expect(preAuthKeys.length).toBeGreaterThanOrEqual(2);
-  });
-
-  test("all pre-auth keys can be listed without user filter (0.28+)", async (context) => {
-    const bootstrap = await getBootstrapClient(version);
-    if (!bootstrap.capabilities.preAuthKeysHaveStableIds) {
-      context.skip();
-    }
-
-    const client = await getRuntimeClient(version);
-    const [preAuthKeyUser] = await client.users.list({ name: "preauthkeyuser@" });
-    expect(preAuthKeyUser).toBeDefined();
-
-    const allKeys = await client.preAuthKeys.listAll!();
-    expect(Array.isArray(allKeys)).toBe(true);
-    expect(allKeys.length).toBeGreaterThanOrEqual(2);
-
-    const userSpecificKeys = await client.preAuthKeys.listForUser(preAuthKeyUser.id);
-    for (const userKey of userSpecificKeys) {
-      const found = allKeys.find((k) => k.key === userKey.key);
-      expect(found).toBeDefined();
-    }
-  });
-
-  test("listAll returns keys with correct structure (0.28+)", async (context) => {
-    const bootstrap = await getBootstrapClient(version);
-    if (!bootstrap.capabilities.preAuthKeysHaveStableIds) {
-      context.skip();
-    }
-
-    const client = await getRuntimeClient(version);
-    const allKeys = await client.preAuthKeys.listAll!();
-
-    for (const key of allKeys) {
-      expect(key.id).toBeDefined();
-      expect(key.key).toBeDefined();
-      expect(typeof key.reusable).toBe("boolean");
-      expect(typeof key.ephemeral).toBe("boolean");
-      expect(typeof key.used).toBe("boolean");
-      expect(key.expiration).toBeDefined();
-      expect(key.createdAt).toBeDefined();
-    }
-  });
-
-  test("pre-auth keys can be expired", async () => {
-    const client = await getRuntimeClient(version);
-    const [preAuthKeyUser] = await client.users.list({ name: "preauthkeyuser@" });
-    expect(preAuthKeyUser).toBeDefined();
-    expect(preAuthKeyUser.name).toBe("preauthkeyuser@");
-
-    const preAuthKeys = await client.preAuthKeys.listForUser(preAuthKeyUser.id);
-    expect(preAuthKeys.length).toBeGreaterThanOrEqual(2);
-    const preAuthKeyToExpire = preAuthKeys[0];
-    await client.preAuthKeys.expire(preAuthKeyToExpire);
-
-    const preAuthKeysAfterExpire = await client.preAuthKeys.listForUser(preAuthKeyUser.id);
-    const expiredKey = preAuthKeysAfterExpire.find((key) => key.key === preAuthKeyToExpire.key);
-    expect(expiredKey).toBeDefined();
-  });
-});

tests/integration/api/users.test.ts

@@ -1,74 +0,0 @@
-import { describe, expect, test } from "vitest";
-
-import { getRuntimeClient, HS_VERSIONS } from "../setup/env";
-
-describe.for(HS_VERSIONS)("Headscale %s: Users", { concurrent: false }, (version) => {
-  test("users can be created", async () => {
-    const client = await getRuntimeClient(version);
-    const user = await client.users.create({ name: "tale@" });
-    expect(user).toBeDefined();
-    expect(user.name).toBe("tale@");
-  });
-
-  test("users can be created with attributes", async () => {
-    const client = await getRuntimeClient(version);
-    const user = await client.users.create({
-      name: "test-user@",
-      email: "test-user@example.com",
-      displayName: "Test User",
-      pictureUrl: "https://github.com/tale.png",
-    });
-
-    expect(user).toBeDefined();
-    expect(user.name).toBe("test-user@");
-    expect(user.email).toBe("test-user@example.com");
-    expect(user.displayName).toBe("Test User");
-    expect(user.profilePicUrl).toBe("https://github.com/tale.png");
-  });
-
-  test("users can be listed", async () => {
-    const client = await getRuntimeClient(version);
-    const users = await client.users.list();
-    expect(Array.isArray(users)).toBe(true);
-    expect(users.length).toBeGreaterThanOrEqual(2);
-  });
-
-  test("users can be listed by name", async () => {
-    const client = await getRuntimeClient(version);
-    const users = await client.users.list({ name: "tale@" });
-    expect(Array.isArray(users)).toBe(true);
-    expect(users.length).toBe(1);
-    expect(users[0].name).toBe("tale@");
-  });
-
-  test("users can be listed by email", async () => {
-    const client = await getRuntimeClient(version);
-    const users = await client.users.list({ email: "test-user@example.com" });
-    expect(Array.isArray(users)).toBe(true);
-    expect(users.length).toBe(1);
-    expect(users[0].email).toBe("test-user@example.com");
-  });
-
-  test("users can be renamed", async () => {
-    const client = await getRuntimeClient(version);
-    const usersBefore = await client.users.list({ name: "tale@" });
-    expect(usersBefore.length).toBe(1);
-    const user = usersBefore[0];
-
-    await client.users.rename(user.id, "renamed-user@");
-    const usersAfter = await client.users.list({ name: "renamed-user@" });
-    expect(usersAfter.length).toBe(1);
-    expect(usersAfter[0].id).toBe(user.id);
-  });
-
-  test("users can be deleted", async () => {
-    const client = await getRuntimeClient(version);
-    const usersBefore = await client.users.list({ name: "test-user@" });
-    expect(usersBefore.length).toBe(1);
-    const user = usersBefore[0];
-
-    await client.users.delete(user.id);
-    const usersAfter = await client.users.list({ name: "test-user@" });
-    expect(usersAfter.length).toBe(0);
-  });
-});

tests/integration/api/versions.test.ts

@@ -1,40 +0,0 @@
-import { describe, expect, test } from "vitest";
-
-import { gte } from "~/server/headscale/api/server-version";
-
-import { getBootstrapClient, HS_VERSIONS, Version } from "../setup/env";
-
-describe.for(HS_VERSIONS)("Headscale %s: Runtime Client", (version) => {
-  test("the runtime client is usable", async () => {
-    const bootstrapper = await getBootstrapClient(version);
-    const runtimeClient = bootstrapper.client("test-api-key");
-    expect(runtimeClient).toBeDefined();
-  });
-
-  test("the server version reported by /version matches the running container", async () => {
-    const bootstrapper = await getBootstrapClient(version);
-    expect(bootstrapper.version.unknown).toBe(false);
-    const reported = `${bootstrapper.version.major}.${bootstrapper.version.minor}.${bootstrapper.version.patch}`;
-    expect(reported).toBe(version);
-  });
-
-  test("capabilities are derived correctly from the detected version", async (context) => {
-    const bootstrapper = await getBootstrapClient(version);
-    const v = bootstrapper.version;
-    expect(bootstrapper.capabilities.preAuthKeysHaveStableIds).toBe(gte(v, "0.28.0"));
-    expect(bootstrapper.capabilities.nodeTagsAreFlat).toBe(gte(v, "0.28.0"));
-    expect(bootstrapper.capabilities.nodeOwnerIsImmutable).toBe(gte(v, "0.28.0"));
-    // If a future version is added to HS_VERSIONS before this test is
-    // updated, surface that explicitly rather than passing silently.
-    const known: Version[] = ["0.27.0", "0.27.1", "0.28.0", "0.29.0", "0.29.1"];
-    if (!known.includes(version)) {
-      context.skip();
-    }
-  });
-
-  test("the health check endpoint works", async () => {
-    const bootstrapper = await getBootstrapClient(version);
-    const health = await bootstrapper.health();
-    expect(health).toBe(true);
-  });
-});

tests/integration/oidc/oidc.test.ts

@@ -2,7 +2,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest";
 
 import { createOidcService, type OidcConfig } from "~/server/oidc/provider";
 
-import { type DexEnv, startDex } from "./start-dex";
+import { type DexEnv, startDex } from "../../harness/dex";
 
 let dex: DexEnv;
 

tests/integration/platform/docker.test.ts

@@ -1,118 +1,75 @@
-import { Client } from "undici";
+import { getContainerRuntimeClient } from "testcontainers";
 import { afterAll, beforeAll, describe, expect, test } from "vitest";
 
 import DockerIntegration from "~/server/config/integration/docker";
+import { createHeadscale } from "~/server/headscale/api";
 
-import { type HeadscaleEnv, startHeadscale } from "../setup/start-headscale";
+import { type StartedHeadscale, startHeadscale } from "../../harness/headscale";
+import { LATEST_HEADSCALE } from "../../harness/versions";
 
-const TEST_LABEL_KEY = "me.tale.headplane.integration-test";
-const TEST_LABEL_VALUE = `docker-${Date.now()}`;
+const LABEL_KEY = "me.tale.headplane.integration-test";
+const LABEL_VALUE = `docker-${process.pid}`;
+const SOCKET = process.env.DOCKER_HOST ?? "unix:///var/run/docker.sock";
 
-function connect(socket: string) {
-  const url = new URL(socket);
-  if (url.protocol === "unix:") {
-    return new Client("http://localhost", { socketPath: url.pathname });
-  }
-
-  return new Client(url.href.replace(url.protocol, "http:"));
-}
-
-describe("DockerIntegration", () => {
-  let env: HeadscaleEnv;
-  let dockerSocket: string;
+describe("Docker integration", () => {
+  let headscale: StartedHeadscale;
 
   beforeAll(async () => {
-    dockerSocket = process.env.DOCKER_HOST ?? "unix:///var/run/docker.sock";
-    env = await startHeadscale("0.25.1", {
-      labels: { [TEST_LABEL_KEY]: TEST_LABEL_VALUE },
+    headscale = await startHeadscale({
+      version: LATEST_HEADSCALE,
+      id: LABEL_VALUE,
+      labels: { [LABEL_KEY]: LABEL_VALUE },
     });
-  }, 60_000);
+  });
 
   afterAll(async () => {
-    await env?.container.stop({ remove: true, removeVolumes: true });
+    await headscale?.stop();
   });
 
-  test("isAvailable finds the headscale container by label", async () => {
-    const integration = new DockerIntegration({
+  function integration(options: { label?: string; name?: string; socket?: string }) {
+    return new DockerIntegration({
       enabled: true,
-      container_label: `${TEST_LABEL_KEY}=${TEST_LABEL_VALUE}`,
-      socket: dockerSocket,
+      container_label: options.label ?? "unused=unused",
+      container_name: options.name,
+      socket: options.socket ?? SOCKET,
     });
+  }
 
-    expect(await integration.isAvailable()).toBe(true);
+  async function startedAt() {
+    const client = await getContainerRuntimeClient();
+    const info = await client.container.inspect(
+      client.container.getById(headscale.container.getId()),
+    );
+    return info.State.StartedAt;
+  }
+
+  test("finds Headscale by label or by name", async () => {
+    const name = headscale.container.getName().replace(/^\//, "");
+
+    expect(await integration({ label: `${LABEL_KEY}=${LABEL_VALUE}` }).isAvailable()).toBe(true);
+    expect(await integration({ name }).isAvailable()).toBe(true);
   });
 
-  test("isAvailable finds the headscale container by name", async () => {
-    const name = env.container.getName().replace(/^\//, "");
-    const integration = new DockerIntegration({
-      enabled: true,
-      container_name: name,
-      container_label: "unused=unused",
-      socket: dockerSocket,
-    });
-
-    expect(await integration.isAvailable()).toBe(true);
+  test("isn't available without a matching container or a socket", async () => {
+    expect(await integration({ label: `${LABEL_KEY}=nothing` }).isAvailable()).toBe(false);
+    expect(
+      await integration({
+        label: `${LABEL_KEY}=${LABEL_VALUE}`,
+        socket: "unix:///tmp/nonexistent-docker.sock",
+      }).isAvailable(),
+    ).toBe(false);
   });
 
-  test("isAvailable returns false for a non-existent label", async () => {
-    const integration = new DockerIntegration({
-      enabled: true,
-      container_label: "me.tale.headplane.nonexistent=true",
-      socket: dockerSocket,
-    });
+  test("a config change restarts Headscale and waits for it to come back", async () => {
+    const docker = integration({ label: `${LABEL_KEY}=${LABEL_VALUE}` });
+    expect(await docker.isAvailable()).toBe(true);
 
-    expect(await integration.isAvailable()).toBe(false);
-  });
+    const before = await startedAt();
+    const client = await createHeadscale({ url: headscale.url });
+    await docker.onConfigChange(client);
 
-  test("isAvailable returns false for an invalid socket", async () => {
-    const integration = new DockerIntegration({
-      enabled: true,
-      container_label: `${TEST_LABEL_KEY}=${TEST_LABEL_VALUE}`,
-      socket: "unix:///tmp/nonexistent-docker.sock",
-    });
-
-    expect(await integration.isAvailable()).toBe(false);
-  });
-
-  test("onConfigChange restarts the container", async () => {
-    const integration = new DockerIntegration({
-      enabled: true,
-      container_label: `${TEST_LABEL_KEY}=${TEST_LABEL_VALUE}`,
-      socket: dockerSocket,
-    });
-
-    expect(await integration.isAvailable()).toBe(true);
-
-    // Health check goes through the Docker socket to avoid stale port
-    // mappings after container restart.
-    const containerId = env.container.getId();
-    const dockerClient = connect(dockerSocket);
-
-    const versionRes = await dockerClient.request({
-      method: "GET",
-      path: "/version",
-    });
-    const versionInfo = (await versionRes.body.json()) as { ApiVersion?: string };
-    const apiVersion = versionInfo.ApiVersion ?? "1.24";
-
-    const mockHeadscale = {
-      health: async () => {
-        try {
-          const res = await dockerClient.request({
-            method: "GET",
-            path: `/v${apiVersion}/containers/${containerId}/json`,
-          });
-          const info = (await res.body.json()) as any;
-          return info.State?.Running === true;
-        } catch {
-          return false;
-        }
-      },
-    } as any;
-
-    await integration.onConfigChange(mockHeadscale);
-
-    const healthy = await mockHeadscale.health();
-    expect(healthy).toBe(true);
+    expect(await startedAt()).not.toBe(before);
+    expect(await client.health()).toBe(true);
+    await client.dispose();
   }, 60_000);
 });

tests/integration/platform/proc.test.ts

@@ -6,6 +6,10 @@ import { fileURLToPath } from "node:url";
 import tc from "testcontainers";
 import { build } from "vite";
 import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { stringify } from "yaml";
+
+import { headscaleConfig } from "../../harness/headscale";
+import { LATEST_HEADSCALE } from "../../harness/versions";
 
 // This is a little shim file that we use to execute the real proc-helper that
 // is bundled through Vite in order to test against a real Linux /proc FS.
@@ -68,13 +72,14 @@ describe("ProcIntegration", () => {
     await writeFile(
       join(tmpDir, "Dockerfile"),
       [
-        "FROM headscale/headscale:0.25.1-debug AS headscale",
+        `FROM headscale/headscale:${LATEST_HEADSCALE}-debug AS headscale`,
         "FROM node:24-alpine",
         "COPY --from=headscale /ko-app/headscale /usr/local/bin/headscale",
       ].join("\n"),
     );
 
-    const configPath = join(root, "tests/integration/setup/config.yaml");
+    const configPath = join(tmpDir, "config.yaml");
+    await writeFile(configPath, stringify(headscaleConfig()));
     const image = await tc.GenericContainer.fromDockerfile(tmpDir).build();
 
     container = await image

tests/integration/setup/config.yaml

@@ -1,73 +0,0 @@
-server_url: http://localhost:8080
-listen_addr: 0.0.0.0:8080
-metrics_listen_addr: 127.0.0.1:9090
-grpc_listen_addr: 127.0.0.1:50443
-grpc_allow_insecure: false
-noise:
-  private_key_path: /var/lib/headscale/noise_private.key
-prefixes:
-  v4: 100.64.0.0/10
-  v6: fd7a:115c:a1e0::/48
-  allocation: sequential
-derp:
-  server:
-    enabled: false
-    region_id: 999
-    region_code: "headscale"
-    region_name: "Headscale Embedded DERP"
-    verify_clients: true
-    stun_listen_addr: "0.0.0.0:3478"
-    private_key_path: /var/lib/headscale/derp_server_private.key
-    automatically_add_embedded_derp_region: true
-    ipv4: 198.51.100.1
-    ipv6: 2001:db8::1
-  urls:
-    - https://controlplane.tailscale.com/derpmap/default
-  paths: []
-  auto_update_enabled: true
-  update_frequency: 3h
-disable_check_updates: false
-ephemeral_node_inactivity_timeout: 30m
-database:
-  type: sqlite
-  debug: false
-  gorm:
-    prepare_stmt: true
-    parameterized_queries: true
-    skip_err_record_not_found: true
-    slow_threshold: 1000
-  sqlite:
-    path: /var/lib/headscale/db.sqlite
-    write_ahead_log: true
-    wal_autocheckpoint: 1000
-acme_url: https://acme-v02.api.letsencrypt.org/directory
-acme_email: ""
-tls_letsencrypt_hostname: ""
-tls_letsencrypt_cache_dir: /var/lib/headscale/cache
-tls_letsencrypt_challenge_type: HTTP-01
-tls_letsencrypt_listen: ":http"
-tls_cert_path: ""
-tls_key_path: ""
-log:
-  level: info
-  format: text
-policy:
-  mode: database
-  path: ""
-dns:
-  magic_dns: true
-  base_domain: example.com
-  override_local_dns: true
-  nameservers:
-    global:
-      - 1.1.1.1
-      - 1.0.0.1
-      - 2606:4700:4700::1111
-      - 2606:4700:4700::1001
-    split: {}
-  search_domains: []
-  extra_records: []
-unix_socket: /var/run/headscale/headscale.sock
-unix_socket_permission: "0770"
-logtail:
-  enabled: false

tests/integration/setup/env.ts

@@ -1,68 +0,0 @@
-import { createHeadscale, type Headscale } from "~/server/headscale/api";
-
-import { type HeadscaleEnv, startHeadscale } from "./start-headscale";
-import { startTailscaleNode, TailscaleNodeEnv } from "./start-tailscale";
-
-// The set of Headscale versions integration tests run against. Listed
-// explicitly (rather than derived from a generated manifest) so the
-// supported version matrix lives next to the code that uses it.
-export const HS_VERSIONS = ["0.27.0", "0.27.1", "0.28.0", "0.29.0", "0.29.1"] as const;
-export type Version = (typeof HS_VERSIONS)[number];
-
-interface VersionStateEntry {
-  env: HeadscaleEnv;
-  tailscaleNode: TailscaleNodeEnv;
-  bootstrap: Headscale;
-}
-
-const versionState = new Map<Version, VersionStateEntry>();
-async function ensureVersion(version: Version) {
-  if (versionState.has(version)) {
-    return versionState.get(version)!;
-  }
-
-  const env = await startHeadscale(version);
-  const tailscaleNode = await startTailscaleNode(version, env.container.getMappedPort(8080));
-  const bootstrap = await createHeadscale({ url: env.apiUrl });
-
-  const entry = { env, tailscaleNode, bootstrap };
-  versionState.set(version, entry);
-  return entry;
-}
-
-export async function getBootstrapClient(version: Version) {
-  const { bootstrap } = await ensureVersion(version);
-  return bootstrap;
-}
-
-export async function getRuntimeClient(version: Version) {
-  const { env, bootstrap } = await ensureVersion(version);
-  return bootstrap.client(env.apiKey);
-}
-
-export async function getNode(version: Version) {
-  const { tailscaleNode } = await ensureVersion(version);
-  return {
-    authCode: tailscaleNode.authCode,
-    registerUrl: tailscaleNode.registerUrl,
-    nodeName: tailscaleNode.nodeName,
-  };
-}
-
-export async function stopAllVersions() {
-  for (const { env, tailscaleNode, bootstrap } of versionState.values()) {
-    await bootstrap.dispose();
-
-    await env.container.stop({
-      remove: true,
-      removeVolumes: true,
-    });
-
-    await tailscaleNode.container.stop({
-      remove: true,
-      removeVolumes: true,
-    });
-  }
-
-  versionState.clear();
-}

tests/integration/setup/start-headscale.ts

@@ -1,54 +0,0 @@
-import { join } from "node:path";
-import { fileURLToPath } from "node:url";
-
-import tc from "testcontainers";
-
-export interface HeadscaleEnv {
-  container: tc.StartedTestContainer;
-  apiUrl: string;
-  apiKey: string;
-}
-
-export interface HeadscaleOptions {
-  labels?: Record<string, string>;
-}
-
-const cwd = fileURLToPath(import.meta.url);
-const config = join(cwd, "..", "config.yaml");
-
-export async function startHeadscale(
-  version: string,
-  options?: HeadscaleOptions,
-): Promise<HeadscaleEnv> {
-  let builder = new tc.GenericContainer(`headscale/headscale:${version}`)
-    .withExposedPorts(8080)
-    .withWaitStrategy(
-      tc.Wait.forHttp("/health", 8080).withStartupTimeout(30_000).forStatusCode(200),
-    )
-    .withCopyFilesToContainer([
-      {
-        source: config,
-        target: "/etc/headscale/config.yaml",
-      },
-    ])
-    .withCommand(["serve"]);
-
-  if (options?.labels) {
-    builder = builder.withLabels(options.labels);
-  }
-
-  const container = await builder.start();
-
-  const host = container.getHost();
-  const port = container.getMappedPort(8080);
-  const apiUrl = `http://${host}:${port}`;
-
-  const exec = await container.exec(["headscale", "apikeys", "create", "-o", "json"]);
-
-  if (exec.exitCode !== 0) {
-    throw new Error(`headscale apikeys create failed:\n${exec.stderr}`);
-  }
-
-  const apiKey = JSON.parse(exec.stdout.toString());
-  return { container, apiUrl, apiKey };
-}

tests/integration/setup/start-tailscale.ts

@@ -1,84 +0,0 @@
-import { createInterface } from "node:readline";
-
-import tc from "testcontainers";
-
-// Imported by `env.ts` which owns the canonical version list; keep this
-// loose to avoid an import cycle.
-export type Version = string;
-
-export interface TailscaleNodeEnv {
-  container: tc.StartedTestContainer;
-  authCode: string;
-  registerUrl: string;
-  nodeName: string;
-}
-
-export async function startTailscaleNode(
-  version: Version,
-  headscalePort: number,
-): Promise<TailscaleNodeEnv> {
-  let resolveAuthCode!: (code: string) => void;
-  let rejectAuthCode!: (err: Error) => void;
-  let authCodeResolved = false;
-
-  const authCodePromise = new Promise<string>((resolve, reject) => {
-    resolveAuthCode = resolve;
-    rejectAuthCode = reject;
-  });
-
-  const nodeName = `test-node-${version.replace(/\./g, "-")}`;
-  const prefix = `http://localhost:8080/register/`;
-  const container = await new tc.GenericContainer("tailscale/tailscale:latest")
-    .withNetworkMode("host")
-    .withEnvironment({
-      TS_STATE_DIR: "/tailscale-state",
-      TS_EXTRA_ARGS: [
-        `--login-server=http://localhost:${headscalePort}`,
-        `--hostname=${nodeName}`,
-        "--accept-dns=false",
-        "--accept-routes=false",
-      ].join(" "),
-    })
-    .withLogConsumer((stream) => {
-      const rl = createInterface({ input: stream });
-
-      rl.on("line", (line: string) => {
-        if (authCodeResolved) return;
-        const idx = line.indexOf(prefix);
-        if (idx === -1) return;
-
-        const after = line.slice(idx + prefix.length).trim();
-        if (!after) return;
-
-        const token = after.split(/\s+/)[0];
-        if (!token) return;
-
-        authCodeResolved = true;
-        resolveAuthCode(`${prefix}${token}`);
-        rl.close();
-      });
-
-      rl.on("close", () => {
-        if (!authCodeResolved) {
-          rejectAuthCode(
-            new Error("Tailscale container log stream closed before auth code was found"),
-          );
-        }
-      });
-    })
-    .withWaitStrategy(tc.Wait.forLogMessage(prefix).withStartupTimeout(30_000))
-    .start();
-
-  const registerUrl = await Promise.race<string>([
-    authCodePromise,
-    new Promise((_, reject) =>
-      setTimeout(
-        () => reject(new Error(`Timed out waiting for Tailscale auth URL on ${prefix}`)),
-        25_000,
-      ),
-    ),
-  ]);
-  const authCode = registerUrl.slice(prefix.length);
-
-  return { container, authCode, registerUrl, nodeName };
-}

tests/integration/setup/vitest-hook.ts

@@ -1,11 +0,0 @@
-import { getBootstrapClient, HS_VERSIONS, stopAllVersions } from "./env";
-
-export async function setup() {
-  for (const version of HS_VERSIONS) {
-    await getBootstrapClient(version);
-  }
-}
-
-export async function teardown() {
-  await stopAllVersions();
-}

vitest.config.ts

@@ -29,29 +29,13 @@ export default defineConfig({
       {
         extends: true,
         test: {
-          name: "integration:api",
-          include: ["tests/integration/api/**/*.test.ts"],
-          setupFiles: ["tests/integration/setup/vitest-hook.ts"],
-          testTimeout: 60_000,
-        },
-      },
-      {
-        extends: true,
-        test: {
-          name: "integration:harness",
-          include: ["tests/integration/{harness,headscale,headplane}/**/*.test.ts"],
+          name: "integration",
+          include: ["tests/integration/**/*.test.ts"],
+          exclude: ["node_modules/**", "build/**", "tests/integration/agent/**"],
           testTimeout: 120_000,
           hookTimeout: 120_000,
         },
       },
-      {
-        extends: true,
-        test: {
-          name: "integration:platform",
-          include: ["tests/integration/platform/**/*.test.ts"],
-          testTimeout: 60_000,
-        },
-      },
       {
         extends: true,
         test: {
@@ -64,14 +48,6 @@ export default defineConfig({
           hookTimeout: 180_000,
         },
       },
-      {
-        extends: true,
-        test: {
-          name: "integration:oidc",
-          include: ["tests/integration/oidc/**/*.test.ts"],
-          testTimeout: 60_000,
-        },
-      },
     ],
     env: {
       HEADPLANE_DEBUG_LOG: "true",