fix(api-tokens): harden rate limiting and surface list-load errors (#1292)

* fix(api-tokens): scope per-token rate limits to live tokens

Forged or token-shaped Authorization headers no longer mint their own
rate-limit budget. The key generator now grants a per-token budget only to
a real, active token and falls back to per-IP keying for anything else, so
a single source cannot evade the global limiter by rotating fake tokens.
The validated token is memoized on the request, so authentication reuses
it without a second database lookup.

Token validation (format, checksum, lookup, revocation, expiry) is now a
single shared helper used by the HTTP auth middleware, the WebSocket
upgrade handler, and the rate-limit key generator, replacing two
near-identical inline copies that could drift apart. The last-used
timestamp write is throttled so a busy token no longer writes to the
database on every request.

* fix(api-tokens): surface token list-load failures with a retry

A failed load of the API tokens list was swallowed: a server error
rendered the empty "no tokens yet" state with no sign that anything went
wrong. The list now shows an error card with a Retry action and raises a
toast on any non-ok response or network error, matching the create and
revoke flows. Adds a troubleshooting entry for the error.

* test(api-tokens): seed tokens via the shared test helper

The new hardening and WS-scope suites computed sha256 of a raw token
directly, which CodeQL flags as js/insufficient-password-hash (a false
positive: these are 256-bit CSPRNG opaque tokens, not passwords). Route
token creation through the existing apiTokenTestHelper and read the stored
token_hash back from the row, so the suites no longer hash anything
themselves. Also removes the duplicated createToken helpers.

* fix(api-tokens): key the rate limiter by the same credential auth uses

The rate-limit key generator checked the session cookie before the
Authorization bearer, while authMiddleware authenticates bearer-over-cookie
(bearerToken || cookieToken). A request could send a Bearer API token plus a
forged cookie and be keyed by the cookie's (forgeable, rotatable) username,
sidestepping the per-token / per-IP keying the limiter applies to API tokens:
a valid token would lose its own bucket, and a forged token-shaped bearer
would no longer collapse to per-IP.

Reorder the generator to mirror auth: process the bearer first (validate the
API token and key per-token or fall back to per-IP; otherwise decode the JWT
by username/sub), and consult the cookie only when there is no bearer.
Regression tests cover a valid and a forged sen_sk_ bearer, each sent with a
forged cookie.

backend/src/__tests__/api-token-hardening.test.ts

@@ -0,0 +1,220 @@
+/**
+ * Tests for the API-token hardening pass:
+ *   - validateApiToken: shared format/checksum/lookup/revocation/expiry result.
+ *   - touchApiTokenLastUsed: throttled last-used write.
+ *   - rateLimitKeyGenerator: per-token budget only for live tokens; forged,
+ *     bad-checksum, revoked, or expired token-shaped bearers fall back to
+ *     per-IP keying so they cannot fragment the limiter (H-1).
+ *
+ * Token rows are seeded through the shared apiTokenTestHelper and their stored
+ * hash is read back from the row, so this suite hashes nothing itself.
+ */
+import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
+import jwt from 'jsonwebtoken';
+import type { Request } from 'express';
+import type { ApiToken } from '../services/DatabaseService';
+import { setupTestDb, cleanupTestDb, TEST_JWT_SECRET } from './helpers/setupTestDb';
+import { COOKIE_NAME } from '../helpers/constants';
+import { createTestApiToken, unbackedApiToken } from './helpers/apiTokenTestHelper';
+import { validateApiToken, touchApiTokenLastUsed } from '../utils/apiTokenAuth';
+import { rateLimitKeyGenerator } from '../middleware/rateLimiters';
+
+let tmpDir: string;
+let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
+
+/** Seed an API token via the shared helper and return the raw value plus its stored row. */
+function createToken(
+  scope: 'read-only' | 'deploy-only' | 'full-admin',
+  opts: { expiresAt?: number | null; revoked?: boolean } = {},
+): { raw: string; row: ApiToken } {
+  const db = DatabaseService.getInstance();
+  const userId = db.getUserByUsername('testadmin')!.id;
+  const name = `hardening-${scope}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+  const raw = createTestApiToken({ db: DatabaseService, scope, userId, name, expiresAt: opts.expiresAt ?? null });
+  const row = db.getActiveApiTokenByNameAndUser(name, userId)!;
+  if (opts.revoked) db.revokeApiToken(row.id);
+  return { raw, row };
+}
+
+/** Minimal Express request carrying a Bearer token, for the key generator. */
+function bearerReq(token: string): Request {
+  return { cookies: {}, headers: { authorization: `Bearer ${token}` }, ip: '203.0.113.7' } as unknown as Request;
+}
+
+beforeAll(async () => {
+  tmpDir = await setupTestDb();
+  ({ DatabaseService } = await import('../services/DatabaseService'));
+});
+
+afterAll(() => {
+  cleanupTestDb(tmpDir);
+});
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+describe('validateApiToken', () => {
+  it('accepts a live token', () => {
+    const { raw } = createToken('read-only');
+    const result = validateApiToken(raw);
+    expect(result.ok).toBe(true);
+    if (result.ok) expect(result.token.scope).toBe('read-only');
+  });
+
+  it('rejects a non-token string', () => {
+    const result = validateApiToken('not-a-sencho-token');
+    expect(result).toEqual({ ok: false, reason: 'not-api-token' });
+  });
+
+  it('rejects a token-shaped string with a bad checksum', () => {
+    const result = validateApiToken('sen_sk_' + 'A'.repeat(49));
+    expect(result).toEqual({ ok: false, reason: 'checksum' });
+  });
+
+  it('rejects a well-formed token that is not in the database', () => {
+    const result = validateApiToken(unbackedApiToken());
+    expect(result).toEqual({ ok: false, reason: 'not-found' });
+  });
+
+  it('rejects a revoked token', () => {
+    const { raw } = createToken('full-admin', { revoked: true });
+    const result = validateApiToken(raw);
+    expect(result).toEqual({ ok: false, reason: 'revoked' });
+  });
+
+  it('rejects an expired token', () => {
+    const { raw } = createToken('deploy-only', { expiresAt: Date.now() - 1000 });
+    const result = validateApiToken(raw);
+    expect(result).toEqual({ ok: false, reason: 'expired' });
+  });
+});
+
+describe('touchApiTokenLastUsed', () => {
+  it('writes when last_used_at is null', () => {
+    const { row } = createToken('read-only');
+    const spy = vi.spyOn(DatabaseService.getInstance(), 'updateApiTokenLastUsed');
+    touchApiTokenLastUsed({ ...row, last_used_at: null });
+    expect(spy).toHaveBeenCalledWith(row.id);
+  });
+
+  it('writes when last_used_at is stale (older than the throttle window)', () => {
+    const { row } = createToken('read-only');
+    const spy = vi.spyOn(DatabaseService.getInstance(), 'updateApiTokenLastUsed');
+    touchApiTokenLastUsed({ ...row, last_used_at: Date.now() - 70_000 });
+    expect(spy).toHaveBeenCalledWith(row.id);
+  });
+
+  it('skips the write when last_used_at is within the throttle window', () => {
+    const { row } = createToken('read-only');
+    const spy = vi.spyOn(DatabaseService.getInstance(), 'updateApiTokenLastUsed');
+    touchApiTokenLastUsed({ ...row, last_used_at: Date.now() - 1000 });
+    expect(spy).not.toHaveBeenCalled();
+  });
+});
+
+describe('rateLimitKeyGenerator (API token branch)', () => {
+  it('keys a live token by its own hash and memoizes the row for auth reuse', () => {
+    const { raw, row } = createToken('read-only');
+    const spy = vi.spyOn(DatabaseService.getInstance(), 'getApiTokenByHash');
+    const req = bearerReq(raw);
+    expect(rateLimitKeyGenerator(req)).toBe(`user:sk:${row.token_hash.slice(0, 16)}`);
+    expect(req._apiToken?.token_hash).toBe(row.token_hash);
+    expect(spy).toHaveBeenCalledTimes(1);
+  });
+
+  it('reuses the memoized row on a second pass without another lookup', () => {
+    const { raw, row } = createToken('read-only');
+    const req = bearerReq(raw);
+    rateLimitKeyGenerator(req);
+    const spy = vi.spyOn(DatabaseService.getInstance(), 'getApiTokenByHash');
+    expect(rateLimitKeyGenerator(req)).toBe(`user:sk:${row.token_hash.slice(0, 16)}`);
+    expect(spy).not.toHaveBeenCalled();
+  });
+
+  it('falls back to per-IP keying for a bad-checksum token without a DB lookup', () => {
+    const spy = vi.spyOn(DatabaseService.getInstance(), 'getApiTokenByHash');
+    const req = bearerReq('sen_sk_' + 'A'.repeat(49));
+    const key = rateLimitKeyGenerator(req);
+    expect(key).not.toMatch(/^user:sk:/);
+    expect(key).toContain('203.0.113.7');
+    expect(req._apiToken).toBeUndefined();
+    expect(spy).not.toHaveBeenCalled();
+  });
+
+  it('falls back to per-IP keying for a well-formed but unknown token', () => {
+    const req = bearerReq(unbackedApiToken());
+    const key = rateLimitKeyGenerator(req);
+    expect(key).not.toMatch(/^user:sk:/);
+    expect(key).toContain('203.0.113.7');
+    expect(req._apiToken).toBeUndefined();
+  });
+
+  it('falls back to per-IP keying for a revoked token', () => {
+    const { raw } = createToken('full-admin', { revoked: true });
+    const key = rateLimitKeyGenerator(bearerReq(raw));
+    expect(key).not.toMatch(/^user:sk:/);
+    expect(key).toContain('203.0.113.7');
+  });
+
+  it('falls back to per-IP keying for an expired token', () => {
+    const { raw } = createToken('deploy-only', { expiresAt: Date.now() - 1000 });
+    const key = rateLimitKeyGenerator(bearerReq(raw));
+    expect(key).not.toMatch(/^user:sk:/);
+    expect(key).toContain('203.0.113.7');
+  });
+
+  it('collapses distinct forged tokens from one IP into the same anonymous bucket (no fragmentation)', () => {
+    // The H-1 property: a single source cannot mint a fresh per-token budget by
+    // rotating forged token-shaped bearers. Two different well-formed tokens
+    // that are not in the DB, from one IP, must share one key, and that key must
+    // be the same per-IP bucket an unauthenticated request from that IP gets.
+    const ip = '198.51.100.42';
+    const forged = (token: string) =>
+      ({ cookies: {}, headers: { authorization: `Bearer ${token}` }, ip } as unknown as Request);
+    const keyA = rateLimitKeyGenerator(forged(unbackedApiToken()));
+    const keyB = rateLimitKeyGenerator(forged(unbackedApiToken()));
+    const keyAnon = rateLimitKeyGenerator({ cookies: {}, headers: {}, ip } as unknown as Request);
+    expect(keyA).not.toMatch(/^user:sk:/);
+    expect(keyA).toBe(keyB);
+    expect(keyA).toBe(keyAnon);
+  });
+
+  it('keys a live API-token bearer by the token even when a cookie is present (bearer precedence)', () => {
+    // authMiddleware prefers the Bearer token over the cookie, so the limiter
+    // must key off the same credential; a forged cookie must not override the
+    // token's bucket.
+    const { raw, row } = createToken('read-only');
+    const forgedCookie = jwt.sign({ username: 'someone-else' }, 'attacker-secret');
+    const req = {
+      cookies: { [COOKIE_NAME]: forgedCookie },
+      headers: { authorization: `Bearer ${raw}` },
+      ip: '203.0.113.7',
+    } as unknown as Request;
+    expect(rateLimitKeyGenerator(req)).toBe(`user:sk:${row.token_hash.slice(0, 16)}`);
+  });
+
+  it('does not let a forged cookie rescue a forged API-token bearer from per-IP keying', () => {
+    const forgedCookie = jwt.sign({ username: 'rotated-1' }, 'attacker-secret');
+    const req = {
+      cookies: { [COOKIE_NAME]: forgedCookie },
+      headers: { authorization: `Bearer ${unbackedApiToken()}` },
+      ip: '203.0.113.7',
+    } as unknown as Request;
+    const key = rateLimitKeyGenerator(req);
+    expect(key).not.toMatch(/^user:/);
+    expect(key).toContain('203.0.113.7');
+  });
+
+  it('still keys a JWT session bearer by username (non-token branch intact)', () => {
+    const token = jwt.sign({ username: 'ci-bot' }, TEST_JWT_SECRET);
+    const req = { cookies: {}, headers: { authorization: `Bearer ${token}` }, ip: '203.0.113.7' } as unknown as Request;
+    expect(rateLimitKeyGenerator(req)).toBe('user:ci-bot');
+  });
+
+  it('still keys a session cookie by username (cookie branch intact)', () => {
+    const token = jwt.sign({ username: 'cookie-user' }, TEST_JWT_SECRET);
+    const req = { cookies: { [COOKIE_NAME]: token }, headers: {}, ip: '203.0.113.7' } as unknown as Request;
+    expect(rateLimitKeyGenerator(req)).toBe('user:cookie-user');
+  });
+});

backend/src/__tests__/api-token-ws-scope.test.ts

@@ -0,0 +1,79 @@
+/**
+ * Integration tests for API-token scope enforcement on the WebSocket upgrade.
+ * Restricted scopes (read-only, deploy-only) may reach only stack logs and
+ * notifications; every other WS path (host console, generic) is 403'd by the
+ * scope gate before dispatch. Driven through a real listening server, no mocks.
+ */
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
+import WebSocket from 'ws';
+import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
+import { createTestApiToken } from './helpers/apiTokenTestHelper';
+
+let tmpDir: string;
+let server: import('http').Server;
+let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
+
+function createToken(scope: 'read-only' | 'deploy-only' | 'full-admin'): string {
+  const db = DatabaseService.getInstance();
+  return createTestApiToken({ db: DatabaseService, scope, userId: db.getUserByUsername('testadmin')!.id });
+}
+
+beforeAll(async () => {
+  vi.restoreAllMocks();
+  tmpDir = await setupTestDb();
+  ({ DatabaseService } = await import('../services/DatabaseService'));
+  const mod = await import('../index');
+  server = mod.server;
+  await new Promise<void>((resolve) => server.listen(0, resolve));
+});
+
+afterAll(async () => {
+  await new Promise<void>((resolve) => server.close(() => resolve()));
+  cleanupTestDb(tmpDir);
+});
+
+function wsUrl(path: string): string {
+  const addr = server.address();
+  if (!addr || typeof addr === 'string') throw new Error('Server not listening');
+  return `ws://127.0.0.1:${addr.port}${path}`;
+}
+
+/** Resolve to the rejected-upgrade HTTP status, or 200 if the socket opens. */
+function upgradeStatus(token: string, path: string): Promise<number> {
+  return new Promise<number>((resolve) => {
+    const ws = new WebSocket(wsUrl(path), { headers: { Authorization: `Bearer ${token}` } });
+    ws.on('unexpected-response', (_req, res) => { ws.close(); resolve(res.statusCode ?? 0); });
+    ws.on('open', () => { ws.close(); resolve(200); });
+    ws.on('error', () => resolve(0));
+  });
+}
+
+describe('WebSocket API-token scope enforcement', () => {
+  it('blocks a read-only token from the host console (403)', async () => {
+    expect(await upgradeStatus(createToken('read-only'), '/api/system/host-console')).toBe(403);
+  });
+
+  it('blocks a deploy-only token from the host console (403)', async () => {
+    expect(await upgradeStatus(createToken('deploy-only'), '/api/system/host-console')).toBe(403);
+  });
+
+  it('blocks a read-only token from a generic socket (403)', async () => {
+    expect(await upgradeStatus(createToken('read-only'), '/ws')).toBe(403);
+  });
+
+  it('does not scope-block a read-only token from notifications', async () => {
+    expect(await upgradeStatus(createToken('read-only'), '/ws/notifications')).not.toBe(403);
+  });
+
+  it('does not scope-block a deploy-only token from notifications', async () => {
+    expect(await upgradeStatus(createToken('deploy-only'), '/ws/notifications')).not.toBe(403);
+  });
+
+  it('does not scope-block a read-only token from stack logs', async () => {
+    expect(await upgradeStatus(createToken('read-only'), '/api/stacks/test-stack/logs')).not.toBe(403);
+  });
+
+  it('does not scope-block a full-admin token from a generic socket', async () => {
+    expect(await upgradeStatus(createToken('full-admin'), '/ws')).not.toBe(403);
+  });
+});

backend/src/middleware/auth.ts

@@ -1,6 +1,5 @@
 import type { Request, Response, NextFunction, RequestHandler } from 'express';
 import jwt from 'jsonwebtoken';
-import crypto from 'crypto';
 import {
   DatabaseService,
   API_TOKEN_SCOPE_TO_ROLE,
@@ -23,7 +22,8 @@ import {
   MFA_PENDING_TTL_MS,
 } from '../helpers/constants';
 import { getCookieOptions } from '../helpers/cookies';
-import { looksLikeApiToken, verifyApiTokenChecksum } from '../utils/apiTokenFormat';
+import { looksLikeApiToken } from '../utils/apiTokenFormat';
+import { validateApiToken, touchApiTokenLastUsed, type ApiTokenValidation } from '../utils/apiTokenAuth';
 
 /**
  * Authenticate a request via cookie session or Bearer token.
@@ -48,30 +48,25 @@ export const authMiddleware: RequestHandler = async (req: Request, res: Response
   try {
     // Opaque sen_sk_ API tokens: scope-based programmatic access. Routed
     // before jwt.verify so the JWT path stays focused on session, mfa_pending,
-    // node_proxy, and pilot_tunnel. Steps 1-3 (prefix, length, checksum)
-    // reject malformed/typoed keys without touching SQLite.
+    // node_proxy, and pilot_tunnel. validateApiToken rejects malformed/typoed
+    // keys (prefix, length, checksum) without touching SQLite.
     if (looksLikeApiToken(token)) {
-      // Uniform 401 message across malformed-checksum, unknown-hash, and
-      // expired/revoked paths so the response body is not a token-existence
-      // oracle. Debug logs still capture the specific reason.
-      if (!verifyApiTokenChecksum(token)) {
-        if (isDebugEnabled()) console.log('[Auth:diag] API token rejected: checksum');
+      // The rate limiter's key generator runs before this middleware and, for
+      // this same bearer, memoizes the validated row on req._apiToken. Reuse it
+      // when present so the token costs one DB lookup per request, not two;
+      // otherwise validate now (e.g. a path the limiter skipped).
+      const validation: ApiTokenValidation = req._apiToken
+        ? { ok: true, token: req._apiToken }
+        : validateApiToken(token);
+      if (!validation.ok) {
+        // Uniform 401 across checksum/unknown/expired/revoked so the response
+        // body is not a token-existence oracle; the debug log keeps the reason.
+        if (isDebugEnabled()) console.log('[Auth:diag] API token rejected:', validation.reason);
         res.status(401).json({ error: 'Invalid or expired token' });
         return;
       }
-      const tokenHash = crypto.createHash('sha256').update(token).digest('hex');
-      const apiToken = DatabaseService.getInstance().getApiTokenByHash(tokenHash);
-      if (!apiToken || apiToken.revoked_at) {
-        if (isDebugEnabled()) console.log('[Auth:diag] API token rejected: not found or revoked');
-        res.status(401).json({ error: 'Invalid or expired token' });
-        return;
-      }
-      if (apiToken.expires_at && apiToken.expires_at < Date.now()) {
-        if (isDebugEnabled()) console.log('[Auth:diag] API token rejected: expired');
-        res.status(401).json({ error: 'Invalid or expired token' });
-        return;
-      }
-      DatabaseService.getInstance().updateApiTokenLastUsed(apiToken.id);
+      const apiToken = validation.token;
+      touchApiTokenLastUsed(apiToken);
       const creator = DatabaseService.getInstance().getUserById(apiToken.user_id);
       req.user = {
         username: creator?.username || `api-token:${apiToken.name}`,

backend/src/middleware/rateLimiters.ts

@@ -1,10 +1,10 @@
 import type { Request } from 'express';
 import rateLimit, { ipKeyGenerator } from 'express-rate-limit';
 import jwt from 'jsonwebtoken';
-import { createHash } from 'crypto';
 import { COOKIE_NAME } from '../helpers/constants';
 import { WEBHOOK_TRIGGER_RE } from '../helpers/routePatterns';
 import { looksLikeApiToken } from '../utils/apiTokenFormat';
+import { validateApiToken } from '../utils/apiTokenAuth';
 
 // ── Rate Limiting ─────────────────────────────────────────────────────────────
 //
@@ -63,32 +63,51 @@ function isNodeProxyRequest(req: Request): boolean {
 }
 
 /**
- * Hybrid rate limit key: JWT username/sub for authenticated requests
- * (per-user budgets), IP otherwise. `jwt.decode()` avoids double-verification;
- * `authMiddleware` handles signature checks downstream.
+ * Hybrid rate limit key: per-token / per-user for authenticated requests, IP
+ * otherwise. Mirrors authMiddleware's bearer-over-cookie precedence (auth.ts
+ * uses `bearerToken || cookieToken`) so the limiter keys off the same credential
+ * auth will use. Checking the cookie first would let a request authenticated by
+ * a Bearer API token be bucketed under an unrelated (or forged) cookie username,
+ * sidestepping the per-token / per-IP keying. `jwt.decode()` avoids
+ * double-verification; `authMiddleware` handles signature checks downstream.
  */
-function rateLimitKeyGenerator(req: Request): string {
-  const cookie = req.cookies?.[COOKIE_NAME];
-  if (cookie) {
-    try {
-      const decoded = jwt.decode(cookie) as { username?: string } | null;
-      if (decoded?.username) return `user:${decoded.username}`;
-    } catch { /* fall through to IP */ }
-  }
+export function rateLimitKeyGenerator(req: Request): string {
   const auth = req.headers.authorization;
   if (auth?.startsWith('Bearer ')) {
     const bearer = auth.slice(7);
-    // Opaque API tokens key by a non-reversible hash slice of the token
-    // itself: each token gets its own rate-limit budget without a DB hit on
-    // the hot path (this runs before authMiddleware).
+    // Opaque API tokens get a per-token rate-limit budget, but ONLY when the
+    // bearer resolves to a real, active token. This runs before authMiddleware,
+    // so keying any token-shaped string by its own hash would let one source
+    // mint a fresh budget per forged value and fragment the limiter; anything
+    // that is not a live token therefore falls through to per-IP keying. The
+    // validated row is memoized on the request so authMiddleware reuses it
+    // without a second lookup (and a request crossing two limiters reuses it
+    // here too). Like the jwt.decode branches, a lookup failure degrades to
+    // per-IP keying rather than throwing out of the key generator.
     if (looksLikeApiToken(bearer)) {
-      const slice = createHash('sha256').update(bearer).digest('hex').slice(0, 16);
-      return `user:sk:${slice}`;
+      if (req._apiToken) return `user:sk:${req._apiToken.token_hash.slice(0, 16)}`;
+      try {
+        const validation = validateApiToken(bearer);
+        if (validation.ok) {
+          // Only ever memoize the row matching this request's bearer;
+          // authMiddleware trusts req._apiToken without re-checking the hash.
+          req._apiToken = validation.token;
+          return `user:sk:${validation.token.token_hash.slice(0, 16)}`;
+        }
+      } catch { /* fall through to IP */ }
+      return ipKeyGenerator(req.ip || 'unknown');
     }
     try {
       const decoded = jwt.decode(bearer) as { username?: string; sub?: string } | null;
       if (decoded?.username) return `user:${decoded.username}`;
       if (decoded?.sub) return `user:${decoded.sub}`;
+    } catch { /* fall through to cookie / IP */ }
+  }
+  const cookie = req.cookies?.[COOKIE_NAME];
+  if (cookie) {
+    try {
+      const decoded = jwt.decode(cookie) as { username?: string } | null;
+      if (decoded?.username) return `user:${decoded.username}`;
     } catch { /* fall through to IP */ }
   }
   return ipKeyGenerator(req.ip || 'unknown');

backend/src/types/express.ts

@@ -1,4 +1,4 @@
-import type { UserRole, ApiTokenScope } from '../services/DatabaseService';
+import type { UserRole, ApiTokenScope, ApiToken } from '../services/DatabaseService';
 import type { LicenseTier, LicenseVariant } from '../services/license-types';
 
 // Extend Express Request type for user and node context.
@@ -10,6 +10,8 @@ declare global {
       user?: { username: string; role: UserRole; userId: number };
       nodeId: number;
       apiTokenScope?: ApiTokenScope;
+      /** Active API token resolved by the rate limiter's key generator (which runs before auth), memoized so `authMiddleware` reuses it without a second DB lookup. Set only for a request bearing a real, non-revoked, non-expired API token. */
+      _apiToken?: ApiToken;
       rawBody?: Buffer;
       /** License tier asserted by the main instance on proxied requests. Only set for trusted node_proxy tokens. */
       proxyTier?: LicenseTier;

backend/src/utils/apiTokenAuth.ts

@@ -0,0 +1,48 @@
+import crypto from 'crypto';
+import { DatabaseService, type ApiToken } from '../services/DatabaseService';
+import { looksLikeApiToken, verifyApiTokenChecksum } from './apiTokenFormat';
+
+/**
+ * Result of validating an opaque `sen_sk_` API token. The failure `reason` is
+ * for diagnostic logging only; callers MUST surface a single uniform 401 so the
+ * response body never becomes a token-existence oracle.
+ */
+export type ApiTokenValidation =
+  | { ok: true; token: ApiToken }
+  | { ok: false; reason: 'not-api-token' | 'checksum' | 'not-found' | 'revoked' | 'expired' };
+
+/**
+ * Validate an API token with no side effects: format, checksum (timing-safe),
+ * hash lookup, revocation, and expiry. The format and checksum checks
+ * short-circuit before the SQLite lookup so malformed or bad-checksum keys never
+ * touch the database. Shared by the HTTP auth middleware, the WebSocket upgrade
+ * handler, and the rate limiter's key generator so all three agree on what
+ * counts as a live token.
+ */
+export function validateApiToken(token: string): ApiTokenValidation {
+  if (!looksLikeApiToken(token)) return { ok: false, reason: 'not-api-token' };
+  if (!verifyApiTokenChecksum(token)) return { ok: false, reason: 'checksum' };
+  const tokenHash = crypto.createHash('sha256').update(token).digest('hex');
+  const apiToken = DatabaseService.getInstance().getApiTokenByHash(tokenHash);
+  if (!apiToken) return { ok: false, reason: 'not-found' };
+  if (apiToken.revoked_at) return { ok: false, reason: 'revoked' };
+  if (apiToken.expires_at && apiToken.expires_at < Date.now()) return { ok: false, reason: 'expired' };
+  return { ok: true, token: apiToken };
+}
+
+/** Minimum interval between `last_used_at` writes for a single token. */
+const LAST_USED_THROTTLE_MS = 60_000;
+
+/**
+ * Record that a token was used, skipping the write when last_used_at is still
+ * within the throttle window. Authenticated API requests fire on every
+ * CI/script call, so an unconditional bump was a synchronous SQLite write per
+ * request; the throttle keeps "last used" accurate to the minute while removing
+ * that write amplification from the hot path. Best-effort: the check reads the
+ * row fetched at request start, so concurrent requests for one token may each
+ * write once.
+ */
+export function touchApiTokenLastUsed(token: ApiToken): void {
+  if (token.last_used_at && Date.now() - token.last_used_at < LAST_USED_THROTTLE_MS) return;
+  DatabaseService.getInstance().updateApiTokenLastUsed(token.id);
+}

backend/src/websocket/upgradeHandler.ts

@@ -2,7 +2,6 @@ import type http from 'http';
 import type { IncomingMessage } from 'http';
 import type { WebSocketServer } from 'ws';
 import jwt from 'jsonwebtoken';
-import crypto from 'crypto';
 import { DatabaseService, type UserRole } from '../services/DatabaseService';
 import { LicenseService } from '../services/LicenseService';
 import { NodeRegistry } from '../services/NodeRegistry';
@@ -15,7 +14,9 @@ import { handleLogsWs } from './logs';
 import { handleHostConsoleWs } from './hostConsole';
 import { handleGenericWs, attachGenericConnectionHandlers } from './generic';
 import { rejectUpgrade as reject } from './reject';
-import { looksLikeApiToken, verifyApiTokenChecksum } from '../utils/apiTokenFormat';
+import { looksLikeApiToken } from '../utils/apiTokenFormat';
+import { validateApiToken, touchApiTokenLastUsed } from '../utils/apiTokenAuth';
+import { isDebugEnabled } from '../utils/debug';
 import { PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from '../services/license-headers';
 import { isLicenseTier, normalizeTier, isLicenseVariant, normalizeVariant } from '../services/license-normalize';
 
@@ -84,13 +85,13 @@ export function attachUpgrade(
       let decoded: { username?: string; scope?: string; role?: string; tv?: number };
       let wsApiTokenScope: string | null = null;
       if (looksLikeApiToken(token)) {
-        if (!verifyApiTokenChecksum(token)) return reject(socket, 401, 'Unauthorized');
-        const tokenHash = crypto.createHash('sha256').update(token).digest('hex');
-        const apiToken = DatabaseService.getInstance().getApiTokenByHash(tokenHash);
-        if (!apiToken || apiToken.revoked_at) return reject(socket, 401, 'Unauthorized');
-        if (apiToken.expires_at && apiToken.expires_at < Date.now()) return reject(socket, 401, 'Unauthorized');
-        DatabaseService.getInstance().updateApiTokenLastUsed(apiToken.id);
-        wsApiTokenScope = apiToken.scope;
+        const validation = validateApiToken(token);
+        if (!validation.ok) {
+          if (isDebugEnabled()) console.log('[Auth:diag] WS API token rejected:', validation.reason);
+          return reject(socket, 401, 'Unauthorized');
+        }
+        touchApiTokenLastUsed(validation.token);
+        wsApiTokenScope = validation.token.scope;
         decoded = { scope: 'api_token' };
       } else {
         const settings = DatabaseService.getInstance().getGlobalSettings();

docs/features/api-tokens.mdx

@@ -360,6 +360,10 @@ Read Only is sufficient for the notification stream.
   <Accordion title="The API Tokens tab is missing from Settings">
     The tab is admin-only. Sign in as an admin user to see it. Tokens are also issued from the hub; if you have selected a remote node in the node switcher, switch back to the local hub and the tab will appear under **Settings → Identity**.
   </Accordion>
+
+  <Accordion title="The token list shows `Couldn't load API tokens`">
+    The list could not be fetched. Use the **Retry** button on the error card to reload. If it persists, confirm you are signed in as an admin and that the hub is reachable, then check the browser console and the hub logs for the underlying error. An empty list with no error card simply means you have not created any tokens yet.
+  </Accordion>
 </AccordionGroup>
 
 ## FAQ

frontend/src/components/ApiTokensSection.tsx

@@ -10,7 +10,7 @@ import { toast } from '@/components/ui/toast-store';
 import { apiFetch } from '@/lib/api';
 import { copyToClipboard } from '@/lib/clipboard';
 import { CapabilityGate } from './CapabilityGate';
-import { Zap, Plus, Copy, Trash2, CheckCircle, RefreshCw, Clock } from 'lucide-react';
+import { Zap, Plus, Copy, Trash2, CheckCircle, RefreshCw, Clock, AlertTriangle } from 'lucide-react';
 import { SettingsPrimaryButton } from './settings/SettingsActions';
 import { SettingsCallout } from './settings/SettingsCallout';
 import { useMastheadStats } from './settings/MastheadStatsContext';
@@ -60,6 +60,7 @@ export function ApiTokensSection() {
     const [showForm, setShowForm] = useState(false);
     const [newToken, setNewToken] = useState<{ id: number; token: string } | null>(null);
     const [revokeTarget, setRevokeTarget] = useState<ApiTokenListItem | null>(null);
+    const [loadError, setLoadError] = useState(false);
 
     const [formName, setFormName] = useState('');
     const [formScope, setFormScope] = useState('read-only');
@@ -71,8 +72,16 @@ export function ApiTokensSection() {
             if (res.ok) {
                 const data: ApiTokenListItem[] = await res.json();
                 setTokens(data.filter(t => !t.revoked_at));
+                setLoadError(false);
+            } else {
+                const err = await res.json().catch(() => ({}));
+                setLoadError(true);
+                toast.error(err?.error || err?.message || 'Failed to load API tokens.');
             }
-        } catch { toast.error('Failed to load API tokens.'); } finally { setLoading(false); }
+        } catch {
+            setLoadError(true);
+            toast.error('Failed to load API tokens.');
+        } finally { setLoading(false); }
     };
 
     // eslint-disable-next-line react-hooks/set-state-in-effect
@@ -224,7 +233,7 @@ export function ApiTokensSection() {
                 )}
 
                 {/* Empty state */}
-                {!loading && tokens.length === 0 && !showForm && (
+                {!loading && !loadError && tokens.length === 0 && !showForm && (
                     <SettingsCallout
                         icon={<Zap className="h-4 w-4" />}
                         title="No API tokens yet"
@@ -232,6 +241,21 @@ export function ApiTokensSection() {
                     />
                 )}
 
+                {/* Load error state */}
+                {!loading && loadError && tokens.length === 0 && !showForm && (
+                    <SettingsCallout
+                        tone="error"
+                        icon={<AlertTriangle className="h-4 w-4" />}
+                        title="Couldn't load API tokens"
+                        subtitle="Check your connection and try again."
+                        action={
+                            <Button variant="outline" size="sm" onClick={() => { setLoading(true); fetchTokens(); }}>
+                                <RefreshCw className="w-4 h-4" strokeWidth={1.5} /> Retry
+                            </Button>
+                        }
+                    />
+                )}
+
                 {/* Token list */}
                 {!loading && tokens.map(token => (
                     <div key={token.id} className="rounded-lg border border-card-border border-t-card-border-top bg-card text-card-foreground shadow-card-bevel transition-colors hover:border-t-card-border-hover p-4 space-y-3">

frontend/src/components/__tests__/ApiTokensSection.test.tsx

@@ -0,0 +1,108 @@
+/**
+ * Coverage for ApiTokensSection load behavior.
+ *
+ * Locks the fix where a non-ok token-list response was swallowed silently: the
+ * section must surface an error toast and an error state with a retry, rather
+ * than presenting the empty "No API tokens yet" state as if no tokens existed.
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { render, screen, waitFor, fireEvent } from '@testing-library/react';
+
+vi.mock('@/lib/api', () => ({
+  apiFetch: vi.fn(),
+}));
+
+vi.mock('@/components/ui/toast-store', () => ({
+  toast: {
+    error: vi.fn(),
+    success: vi.fn(),
+    warning: vi.fn(),
+    info: vi.fn(),
+    loading: vi.fn(),
+    dismiss: vi.fn(),
+  },
+}));
+
+// Render CapabilityGate's children directly: the capability gate is not under test.
+vi.mock('@/context/NodeContext', () => ({
+  useNodes: () => ({
+    hasCapability: () => true,
+    activeNode: { id: 1, name: 'local' },
+    activeNodeMeta: { version: '1.0.0', capabilities: ['api-tokens'], fetchedAt: 0 },
+  }),
+}));
+
+// The masthead stats hook depends on a provider that is not mounted here.
+vi.mock('../settings/MastheadStatsContext', () => ({
+  useMastheadStats: () => {},
+}));
+
+import { apiFetch } from '@/lib/api';
+import { toast } from '@/components/ui/toast-store';
+import { ApiTokensSection } from '../ApiTokensSection';
+
+const mockedFetch = apiFetch as unknown as ReturnType<typeof vi.fn>;
+const mockedToast = toast as unknown as { error: ReturnType<typeof vi.fn> };
+
+function tokenRow(overrides: Record<string, unknown> = {}) {
+  return {
+    id: 1,
+    name: 'ci-pipeline',
+    scope: 'read-only',
+    created_at: 1_700_000_000_000,
+    last_used_at: null,
+    expires_at: null,
+    revoked_at: null,
+    ...overrides,
+  };
+}
+
+describe('ApiTokensSection load behavior', () => {
+  beforeEach(() => {
+    mockedFetch.mockReset();
+    mockedToast.error.mockReset();
+  });
+
+  it('surfaces an error toast and an error state (not the empty state) when the list load fails', async () => {
+    mockedFetch.mockResolvedValue({ ok: false, json: async () => ({ error: 'list blew up' }) });
+
+    render(<ApiTokensSection />);
+
+    await waitFor(() => expect(mockedToast.error).toHaveBeenCalledWith('list blew up'));
+    expect(await screen.findByText("Couldn't load API tokens")).toBeInTheDocument();
+    expect(screen.queryByText('No API tokens yet')).toBeNull();
+  });
+
+  it('shows the empty state and does not toast on a successful empty load', async () => {
+    mockedFetch.mockResolvedValue({ ok: true, json: async () => [] });
+
+    render(<ApiTokensSection />);
+
+    expect(await screen.findByText('No API tokens yet')).toBeInTheDocument();
+    expect(screen.queryByText("Couldn't load API tokens")).toBeNull();
+    expect(mockedToast.error).not.toHaveBeenCalled();
+  });
+
+  it('renders the token list and does not toast on a successful load', async () => {
+    mockedFetch.mockResolvedValue({ ok: true, json: async () => [tokenRow()] });
+
+    render(<ApiTokensSection />);
+
+    expect(await screen.findByText('ci-pipeline')).toBeInTheDocument();
+    expect(mockedToast.error).not.toHaveBeenCalled();
+  });
+
+  it('recovers via Retry: a failed load then a successful one clears the error state', async () => {
+    mockedFetch
+      .mockResolvedValueOnce({ ok: false, json: async () => ({ error: 'transient' }) })
+      .mockResolvedValueOnce({ ok: true, json: async () => [] });
+
+    render(<ApiTokensSection />);
+
+    const retry = await screen.findByRole('button', { name: /retry/i });
+    fireEvent.click(retry);
+
+    expect(await screen.findByText('No API tokens yet')).toBeInTheDocument();
+    expect(screen.queryByText("Couldn't load API tokens")).toBeNull();
+  });
+});