feat: ambient AI visit scribe (record/paste → reviewed SOAP note)

Record a clinician↔patient visit (or paste a transcript) on the patient
sheet; the backend transcribes it (OpenAI Whisper / Gemini), de-identifies
the transcript + context through Veil, and drafts a structured SOAP note
the clinician reviews and edits before saving — the same write-approval
gate as the chat agent.

Backend: POST /api/scribe/{transcribe,draft,save} (routes/scribe.ts,
services/ai/transcribe.ts), veil.redactText() free-text redactor,
appendEncounter service, audio MIME types on attachments. Gated by
patient:write + the clinic AI policy (reception/disabled-AI excluded).
Frontend: ScribeDialog + lib/scribe.ts, "Record visit" on the patient
detail, gated by clinical access + AI availability. New `scribe` locale
namespace across all five languages. Bumps to v0.4.0.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/src/index.ts

@@ -28,6 +28,7 @@ import { patientsRouter } from "./routes/patients.js";
 import { patientsWalletRouter } from "./routes/patients-wallet.js";
 import { portalRouter } from "./routes/portal.js";
 import { prescriptionsRouter } from "./routes/prescriptions.js";
+import { scribeRouter } from "./routes/scribe.js";
 import { settingsRouter } from "./routes/settings.js";
 import { signingRouter } from "./routes/signing.js";
 import { staffRouter } from "./routes/staff.js";
@@ -104,6 +105,7 @@ app.use("/api/notifications", notificationsRouter);
 app.use("/api/settings", settingsRouter);
 app.use("/api/ai", aiRouter);
 app.use("/api/chat", chatRouter);
+app.use("/api/scribe", scribeRouter);
 app.use("/api/integrations", integrationsRouter);
 app.use("/api/portal", portalRouter);
 app.use("/api/auth-helpers", authHelpersRouter);

backend/src/routes/attachments.ts

@@ -41,6 +41,14 @@ const ALLOWED_MIME = new Set([
   "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
   "application/vnd.ms-excel",
   "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  // Ambient visit-scribe recordings (stored as a patient attachment so they're
+  // auditable). Voice Opus/AAC stays well under the 15 MB cap for a long visit.
+  "audio/webm",
+  "audio/ogg",
+  "audio/mp4",
+  "audio/mpeg",
+  "audio/wav",
+  "audio/x-m4a",
 ]);
 
 // Disk storage under UPLOAD_DIR/<orgId>/, keyed by a random id so original

backend/src/routes/scribe.ts

@@ -0,0 +1,240 @@
+import { readFile } from "node:fs/promises";
+
+import { generateText } from "ai";
+import { Router } from "express";
+import { z } from "zod";
+
+import { HttpError } from "../lib/http-error.js";
+import { isReceptionOnly, providerScope } from "../lib/role-scope.js";
+import {
+  requireAuth,
+  requireOrg,
+  requirePermission,
+} from "../middleware/auth.js";
+import { recordActivity } from "../services/activity.js";
+import { getAiSettings } from "../services/ai/config.js";
+import { aiAllowedFor, getPolicy } from "../services/ai/policy.js";
+import { resolveModel } from "../services/ai/provider.js";
+import { transcribeAudio } from "../services/ai/transcribe.js";
+import { createVeil } from "../services/ai/veil.js";
+import { absolutePath, getAttachmentRow } from "../services/attachments.js";
+import { appendEncounter, getPatient } from "../services/patients.js";
+import type { Encounter } from "../types/patient.js";
+
+export const scribeRouter = Router();
+
+// The ambient scribe drafts and saves a clinical encounter note, so it needs
+// full clinical write access — never reception (demographics only).
+scribeRouter.use(
+  requireAuth,
+  requireOrg,
+  requirePermission({ patient: ["write"] }),
+);
+
+// Guard shared by every scribe route: the clinic AI kill-switch must allow this
+// member, and reception (demographics-only) can never draft clinical notes.
+async function ensureScribeAllowed(req: {
+  organizationId?: string;
+  memberRole?: string;
+}): Promise<void> {
+  if (isReceptionOnly(req.memberRole)) {
+    throw new HttpError(403, "The visit scribe is not available for your role.");
+  }
+  const policy = await getPolicy(req.organizationId!);
+  if (!aiAllowedFor(policy, req.memberRole)) {
+    throw new HttpError(403, "The AI assistant is disabled for your account.");
+  }
+}
+
+const transcribeSchema = z.object({
+  attachmentId: z.string().trim().min(1),
+});
+
+// POST /api/scribe/transcribe — turn a stored audio attachment into a raw
+// transcript via the user's speech provider (OpenAI/Gemini). The audio does NOT
+// pass through Veil or the chat loop.
+scribeRouter.post("/transcribe", async (req, res, next) => {
+  try {
+    await ensureScribeAllowed(req);
+    const { attachmentId } = transcribeSchema.parse(req.body);
+    const row = await getAttachmentRow(req.organizationId!, attachmentId);
+    if (!row) throw new HttpError(404, "Recording not found.");
+    if (!row.mimeType.startsWith("audio/")) {
+      throw new HttpError(400, "That attachment is not an audio recording.");
+    }
+
+    const settings = await getAiSettings(req.user!.id);
+    const buffer = await readFile(absolutePath(row.storagePath));
+    const { transcript, provider } = await transcribeAudio(settings, {
+      buffer,
+      mimeType: row.mimeType,
+      filename: row.filename,
+    });
+
+    void recordActivity({
+      orgId: req.organizationId!,
+      actor: { id: req.user!.id, name: req.user!.name },
+      action: `Transcribed a visit recording (${provider})`,
+      entityType: "patient",
+      patientFileNumber: row.fileNumber,
+    });
+
+    res.json({ transcript });
+  } catch (err) {
+    next(err);
+  }
+});
+
+const draftSchema = z.object({
+  fileNumber: z.string().trim().min(1),
+  transcript: z.string().trim().min(1).max(100_000),
+  visitType: z.string().trim().max(120).optional(),
+  date: z.string().trim().max(40).optional(),
+});
+
+// Strip ```json fences and parse; returns null if the text isn't JSON.
+function parseDraftJson(text: string): { type?: string; summary?: string } | null {
+  const cleaned = text
+    .replace(/^\s*```(?:json)?/i, "")
+    .replace(/```\s*$/i, "")
+    .trim();
+  try {
+    const parsed = JSON.parse(cleaned);
+    return typeof parsed === "object" && parsed ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+const DRAFT_SYSTEM = [
+  "You are a clinical scribe. From a visit transcript, write a concise, structured",
+  "encounter note in SOAP format (Subjective, Objective, Assessment, Plan).",
+  "Rules:",
+  "- Use ONLY what the transcript supports. Never invent vitals, doses, or findings.",
+  "- If a SOAP section has nothing to report, write \"Not discussed.\" under it.",
+  "- Identifiers may appear as tokens like [PATIENT_1] or [PROVIDER_1]; keep them",
+  "  verbatim — do not guess real names.",
+  "Respond with a JSON object ONLY (no prose, no code fences):",
+  '{ "type": "<short visit type, e.g. Follow-up / New patient / Telehealth>",',
+  '  "summary": "<the SOAP note as markdown with **Subjective** / **Objective** /',
+  '  **Assessment** / **Plan** headings>" }',
+].join("\n");
+
+// POST /api/scribe/draft — draft an encounter note from a transcript. The
+// transcript + patient context are Veil-redacted before any external call and
+// the output is rehydrated. Nothing is written — the clinician reviews and
+// approves via POST /save (the same write-approval gate as the chat agent).
+scribeRouter.post("/draft", async (req, res, next) => {
+  try {
+    await ensureScribeAllowed(req);
+    const { fileNumber, transcript, visitType, date } = draftSchema.parse(
+      req.body,
+    );
+
+    const patient = await getPatient(
+      req.organizationId!,
+      fileNumber,
+      false,
+      providerScope(req.memberRole, req.user!.id),
+    );
+    if (!patient) throw new HttpError(404, "Patient not found.");
+
+    const settings = await getAiSettings(req.user!.id);
+    const resolved = resolveModel(settings, settings.defaultModel);
+    const veil = createVeil(settings.veilLevel, resolved.isExternal);
+
+    // Seed Veil's token maps with this patient's identifiers, then redact the
+    // free-text transcript against them before it leaves the clinic.
+    const redactedPatient = veil.redactPatient(patient);
+    const redactedTranscript = veil.redactText(transcript);
+
+    const context = [
+      `Patient: ${redactedPatient.name} (MRN ${redactedPatient.fileNumber}), ${patient.age}y ${patient.sex}.`,
+      patient.problems.length
+        ? `Known problems: ${patient.problems.map((p) => p.label).join(", ")}.`
+        : "",
+      patient.medications.length
+        ? `Current medications: ${patient.medications
+            .map((m) => `${m.name} ${m.dose}`)
+            .join(", ")}.`
+        : "",
+      visitType ? `Visit type hint: ${visitType}.` : "",
+      "",
+      "Transcript:",
+      redactedTranscript,
+    ]
+      .filter(Boolean)
+      .join("\n");
+
+    const result = await generateText({
+      model: resolved.model,
+      system: DRAFT_SYSTEM,
+      prompt: context,
+    });
+
+    const parsed = parseDraftJson(result.text);
+    const summary = veil.rehydrate(
+      (parsed?.summary ?? result.text ?? "").trim(),
+    );
+    const type = (parsed?.type ?? visitType ?? "Visit").trim() || "Visit";
+
+    const draft: Encounter = {
+      date: date || new Date().toISOString().slice(0, 10),
+      type,
+      // The responsible clinician is the signed-in user, not the model's guess.
+      provider: req.user!.name ?? "",
+      summary,
+    };
+
+    res.json({
+      draft,
+      veil: {
+        active: veil.active,
+        level: veil.level,
+        classes: veil.usedClasses(),
+        provider: resolved.providerLabel,
+      },
+    });
+  } catch (err) {
+    next(err);
+  }
+});
+
+const saveSchema = z.object({
+  fileNumber: z.string().trim().min(1),
+  encounter: z.object({
+    date: z.string().trim().min(1),
+    type: z.string().trim().min(1),
+    provider: z.string().trim().default(""),
+    summary: z.string().trim().min(1),
+  }),
+});
+
+// POST /api/scribe/save — the approval step: append the reviewed encounter note
+// to the patient record. Re-validates server-side and audits like any add.
+scribeRouter.post("/save", async (req, res, next) => {
+  try {
+    await ensureScribeAllowed(req);
+    const { fileNumber, encounter } = saveSchema.parse(req.body);
+    const updated = await appendEncounter(
+      req.organizationId!,
+      fileNumber,
+      encounter,
+    );
+    if (!updated) throw new HttpError(404, "Patient not found.");
+
+    void recordActivity({
+      orgId: req.organizationId!,
+      actor: { id: req.user!.id, name: req.user!.name },
+      action: `Added a scribe visit note for ${updated.name}`,
+      entityType: "patient",
+      entityId: updated.fileNumber,
+      patientName: updated.name,
+      patientFileNumber: updated.fileNumber,
+    });
+
+    res.status(201).json(updated);
+  } catch (err) {
+    next(err);
+  }
+});

backend/src/services/ai/transcribe.ts

@@ -0,0 +1,132 @@
+import { HttpError } from "../../lib/http-error.js";
+import type { userAiSettings } from "../../db/schema/ai.js";
+import { getApiKey } from "./config.js";
+
+type AiSettingsRow = typeof userAiSettings.$inferSelect;
+
+export type AudioInput = {
+  buffer: Buffer;
+  mimeType: string;
+  filename: string;
+};
+
+// Which transcription backend a user's AI settings can reach. Anthropic has no
+// speech-to-text API, so an Anthropic-only user must paste a transcript instead.
+export type TranscribeProvider = "openai" | "gemini";
+
+export function transcribeProviderFor(
+  settings: AiSettingsRow,
+): TranscribeProvider | null {
+  if (getApiKey(settings, "openai")) return "openai";
+  if (getApiKey(settings, "gemini")) return "gemini";
+  return null;
+}
+
+const OPENAI_MODEL = "whisper-1";
+const GEMINI_MODEL = "gemini-2.5-flash";
+
+const TRANSCRIBE_PROMPT =
+  "Transcribe this clinical visit recording verbatim. Return only the spoken words as plain text, with no commentary, headings, or timestamps.";
+
+async function transcribeWithOpenAI(
+  apiKey: string,
+  audio: AudioInput,
+): Promise<string> {
+  const form = new FormData();
+  form.append(
+    "file",
+    new Blob([new Uint8Array(audio.buffer)], { type: audio.mimeType }),
+    audio.filename,
+  );
+  form.append("model", OPENAI_MODEL);
+  form.append("response_format", "json");
+
+  const res = await fetch("https://api.openai.com/v1/audio/transcriptions", {
+    method: "POST",
+    headers: { Authorization: `Bearer ${apiKey}` },
+    body: form,
+  });
+  if (!res.ok) {
+    const detail = await res.text().catch(() => "");
+    throw new HttpError(
+      502,
+      `Transcription failed (OpenAI ${res.status}). ${detail.slice(0, 300)}`,
+    );
+  }
+  const json = (await res.json()) as { text?: string };
+  return (json.text ?? "").trim();
+}
+
+async function transcribeWithGemini(
+  apiKey: string,
+  audio: AudioInput,
+): Promise<string> {
+  const url = `https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_MODEL}:generateContent?key=${encodeURIComponent(
+    apiKey,
+  )}`;
+  const res = await fetch(url, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      contents: [
+        {
+          parts: [
+            {
+              inline_data: {
+                mime_type: audio.mimeType,
+                data: audio.buffer.toString("base64"),
+              },
+            },
+            { text: TRANSCRIBE_PROMPT },
+          ],
+        },
+      ],
+    }),
+  });
+  if (!res.ok) {
+    const detail = await res.text().catch(() => "");
+    throw new HttpError(
+      502,
+      `Transcription failed (Gemini ${res.status}). ${detail.slice(0, 300)}`,
+    );
+  }
+  const json = (await res.json()) as {
+    candidates?: { content?: { parts?: { text?: string }[] } }[];
+  };
+  const text =
+    json.candidates?.[0]?.content?.parts
+      ?.map((p) => p.text ?? "")
+      .join("")
+      .trim() ?? "";
+  return text;
+}
+
+// Send an audio recording to the user's transcription provider and return the
+// raw transcript. The audio never passes through the chat loop or Veil — Veil
+// cannot redact speech, so the caller must warn the clinician that audio leaves
+// the clinic when an external provider is used (only the DRAFTING step is
+// Veil-protected). Throws a 400 when no speech-capable provider is configured.
+export async function transcribeAudio(
+  settings: AiSettingsRow,
+  audio: AudioInput,
+): Promise<{ transcript: string; provider: TranscribeProvider }> {
+  const provider = transcribeProviderFor(settings);
+  if (!provider) {
+    throw new HttpError(
+      400,
+      "Transcription needs an OpenAI or Gemini API key. Add one in Settings → AI, or paste the visit transcript instead.",
+    );
+  }
+  const apiKey = getApiKey(settings, provider)!;
+  const transcript =
+    provider === "openai"
+      ? await transcribeWithOpenAI(apiKey, audio)
+      : await transcribeWithGemini(apiKey, audio);
+  if (!transcript) {
+    throw new HttpError(
+      502,
+      "The transcription came back empty — try again or paste the transcript.",
+    );
+  }
+  return { transcript, provider };
+}

backend/src/services/ai/veil.ts

@@ -26,6 +26,15 @@ export type Veil = {
   redactPatient: (patient: Patient) => Patient;
   /** Map a possibly-tokenized file number from a tool call back to the real one. */
   resolveFileNumber: (input: string) => string;
+  /**
+   * De-identify free text (e.g. a visit transcript) by swapping any KNOWN
+   * identifiers — the patient name, MRN and provider names already seen via
+   * redactPatient — for their tokens. Seed the token maps by calling
+   * redactPatient(patient) first. Note: this only catches identifiers we know
+   * about; free-text PHI spoken aloud (addresses, relatives' names) is not
+   * covered — see the ambient-scribe consent notice.
+   */
+  redactText: (text: string) => string;
   /** Swap any tokens in model output back to real identifiers. */
   rehydrate: (text: string) => string;
   /** Token classes actually emitted — for the audit log. */
@@ -81,6 +90,21 @@ export function createVeil(level: VeilLevel, active: boolean): Veil {
     return mrnByToken.get(input.trim()) ?? input;
   }
 
+  function redactText(text: string): string {
+    if (!isActive || fromToken.size === 0) return text;
+    let out = text;
+    // Longest real values first so a provider name containing the patient name
+    // (or similar overlap) is replaced whole before its substrings.
+    const byLength = [...fromToken.entries()]
+      .filter(([, real]) => real.trim().length > 0)
+      .sort((a, b) => b[1].length - a[1].length);
+    for (const [token, real] of byLength) {
+      const escaped = real.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+      out = out.replace(new RegExp(escaped, "gi"), token);
+    }
+    return out;
+  }
+
   function rehydrate(text: string): string {
     if (!isActive || fromToken.size === 0) return text;
     let out = text;
@@ -101,6 +125,7 @@ export function createVeil(level: VeilLevel, active: boolean): Veil {
     level,
     redactPatient,
     resolveFileNumber,
+    redactText,
     rehydrate,
     usedClasses,
   };

backend/src/services/patients.ts

@@ -570,6 +570,46 @@ export async function appendLabs(
   return getPatient(orgId, fileNumber);
 }
 
+// Append a single encounter (visit note) without touching the rest of the
+// record — used by the ambient AI scribe, which drafts one note at a time and
+// must not go through updatePatient's wholesale child replacement. Position
+// continues after the current max so the new note sorts last.
+export async function appendEncounter(
+  orgId: string,
+  fileNumber: string,
+  entry: Encounter,
+): Promise<Patient | null> {
+  const inserted = await db.transaction(async (tx) => {
+    const [existing] = await tx
+      .select({ id: patients.id })
+      .from(patients)
+      .where(
+        and(
+          eq(patients.organizationId, orgId),
+          eq(patients.fileNumber, fileNumber),
+        ),
+      );
+    if (!existing) return false;
+
+    const [pos] = await tx
+      .select({ max: sql<number>`coalesce(max(${encounters.position}), -1)` })
+      .from(encounters)
+      .where(eq(encounters.patientId, existing.id));
+    await tx.insert(encounters).values({
+      patientId: existing.id,
+      position: (pos?.max ?? -1) + 1,
+      ...entry,
+    });
+    await tx
+      .update(patients)
+      .set({ updatedAt: new Date() })
+      .where(eq(patients.id, existing.id));
+    return true;
+  });
+  if (!inserted) return null;
+  return getPatient(orgId, fileNumber);
+}
+
 // Remove a single lab result from a patient, identified by its
 // name/value/takenAt (the frontend has no row id). Scoped to the org via the
 // owning patient. Returns the reloaded patient, or null when the chart is gone.