From b29fdff1cbf10a16df8bbd6b3dffe13ba3aface0 Mon Sep 17 00:00:00 2001 From: Khalid Abdi Date: Fri, 3 Jul 2026 18:33:30 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20ambient=20AI=20visit=20scribe=20(record?= =?UTF-8?q?/paste=20=E2=86=92=20reviewed=20SOAP=20note)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record a clinician↔patient visit (or paste a transcript) on the patient sheet; the backend transcribes it (OpenAI Whisper / Gemini), de-identifies the transcript + context through Veil, and drafts a structured SOAP note the clinician reviews and edits before saving — the same write-approval gate as the chat agent. Backend: POST /api/scribe/{transcribe,draft,save} (routes/scribe.ts, services/ai/transcribe.ts), veil.redactText() free-text redactor, appendEncounter service, audio MIME types on attachments. Gated by patient:write + the clinic AI policy (reception/disabled-AI excluded). Frontend: ScribeDialog + lib/scribe.ts, "Record visit" on the patient detail, gated by clinical access + AI availability. New `scribe` locale namespace across all five languages. Bumps to v0.4.0. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 16 + backend/package.json | 2 +- backend/src/index.ts | 2 + backend/src/routes/attachments.ts | 8 + backend/src/routes/scribe.ts | 240 ++++++++++ backend/src/services/ai/transcribe.ts | 132 ++++++ backend/src/services/ai/veil.ts | 25 + backend/src/services/patients.ts | 40 ++ .../patients/patient-detail-sheet.tsx | 17 + .../components/patients/patient-detail.tsx | 18 +- .../components/patients/scribe-dialog.tsx | 436 ++++++++++++++++++ frontend/lib/i18n/locales/ar/translation.json | 44 ++ frontend/lib/i18n/locales/de/translation.json | 44 ++ frontend/lib/i18n/locales/en/translation.json | 44 ++ frontend/lib/i18n/locales/fr/translation.json | 44 ++ frontend/lib/i18n/locales/so/translation.json | 44 ++ frontend/lib/scribe.ts | 51 ++ frontend/package.json | 2 +- package.json | 2 +- 19 files changed, 1207 insertions(+), 4 deletions(-) create mode 100644 backend/src/routes/scribe.ts create mode 100644 backend/src/services/ai/transcribe.ts create mode 100644 frontend/components/patients/scribe-dialog.tsx create mode 100644 frontend/lib/scribe.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 71bdeb4..1780bc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ for how releases are cut and published. ## [Unreleased] +## [0.4.0] — 2026-07-03 + +### Added +- **Ambient AI visit scribe** — a **Record visit** action on the patient sheet turns a + clinician↔patient conversation into a draft **SOAP** encounter note. Record with the + microphone (`MediaRecorder`, stored as an auditable patient attachment) or paste a + transcript; the backend transcribes via the user's **OpenAI (Whisper)** or **Gemini** + key, de-identifies the transcript + patient context through **Veil**, and the model + drafts a structured note that the clinician **reviews and edits before saving** — the + same write-approval gate as the chat agent. New `POST /api/scribe/{transcribe,draft,save}` + (`backend/src/routes/scribe.ts`, `services/ai/transcribe.ts`), a `veil.redactText()` + free-text redactor, and an `appendEncounter` service that adds one note without touching + the rest of the record. Gated by `patient:write` + the clinic AI policy (reception and + disabled-AI accounts don't see it). New `scribe` locale namespace across all five + languages. Drafting also works with local Ollama from a pasted transcript. + ## [0.3.0] — 2026-07-02 ### Added diff --git a/backend/package.json b/backend/package.json index 019ade5..2aeecc3 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "temetro-backend", - "version": "0.3.0", + "version": "0.4.0", "private": true, "type": "module", "description": "temetro backend — Express + Postgres API with Better Auth (email/password, organizations) and org-scoped patient records.", diff --git a/backend/src/index.ts b/backend/src/index.ts index 45248a0..9316c33 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -28,6 +28,7 @@ import { patientsRouter } from "./routes/patients.js"; import { patientsWalletRouter } from "./routes/patients-wallet.js"; import { portalRouter } from "./routes/portal.js"; import { prescriptionsRouter } from "./routes/prescriptions.js"; +import { scribeRouter } from "./routes/scribe.js"; import { settingsRouter } from "./routes/settings.js"; import { signingRouter } from "./routes/signing.js"; import { staffRouter } from "./routes/staff.js"; @@ -104,6 +105,7 @@ app.use("/api/notifications", notificationsRouter); app.use("/api/settings", settingsRouter); app.use("/api/ai", aiRouter); app.use("/api/chat", chatRouter); +app.use("/api/scribe", scribeRouter); app.use("/api/integrations", integrationsRouter); app.use("/api/portal", portalRouter); app.use("/api/auth-helpers", authHelpersRouter); diff --git a/backend/src/routes/attachments.ts b/backend/src/routes/attachments.ts index 4a6f599..016f18a 100644 --- a/backend/src/routes/attachments.ts +++ b/backend/src/routes/attachments.ts @@ -41,6 +41,14 @@ const ALLOWED_MIME = new Set([ "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + // Ambient visit-scribe recordings (stored as a patient attachment so they're + // auditable). Voice Opus/AAC stays well under the 15 MB cap for a long visit. + "audio/webm", + "audio/ogg", + "audio/mp4", + "audio/mpeg", + "audio/wav", + "audio/x-m4a", ]); // Disk storage under UPLOAD_DIR//, keyed by a random id so original diff --git a/backend/src/routes/scribe.ts b/backend/src/routes/scribe.ts new file mode 100644 index 0000000..b76c7da --- /dev/null +++ b/backend/src/routes/scribe.ts @@ -0,0 +1,240 @@ +import { readFile } from "node:fs/promises"; + +import { generateText } from "ai"; +import { Router } from "express"; +import { z } from "zod"; + +import { HttpError } from "../lib/http-error.js"; +import { isReceptionOnly, providerScope } from "../lib/role-scope.js"; +import { + requireAuth, + requireOrg, + requirePermission, +} from "../middleware/auth.js"; +import { recordActivity } from "../services/activity.js"; +import { getAiSettings } from "../services/ai/config.js"; +import { aiAllowedFor, getPolicy } from "../services/ai/policy.js"; +import { resolveModel } from "../services/ai/provider.js"; +import { transcribeAudio } from "../services/ai/transcribe.js"; +import { createVeil } from "../services/ai/veil.js"; +import { absolutePath, getAttachmentRow } from "../services/attachments.js"; +import { appendEncounter, getPatient } from "../services/patients.js"; +import type { Encounter } from "../types/patient.js"; + +export const scribeRouter = Router(); + +// The ambient scribe drafts and saves a clinical encounter note, so it needs +// full clinical write access — never reception (demographics only). +scribeRouter.use( + requireAuth, + requireOrg, + requirePermission({ patient: ["write"] }), +); + +// Guard shared by every scribe route: the clinic AI kill-switch must allow this +// member, and reception (demographics-only) can never draft clinical notes. +async function ensureScribeAllowed(req: { + organizationId?: string; + memberRole?: string; +}): Promise { + if (isReceptionOnly(req.memberRole)) { + throw new HttpError(403, "The visit scribe is not available for your role."); + } + const policy = await getPolicy(req.organizationId!); + if (!aiAllowedFor(policy, req.memberRole)) { + throw new HttpError(403, "The AI assistant is disabled for your account."); + } +} + +const transcribeSchema = z.object({ + attachmentId: z.string().trim().min(1), +}); + +// POST /api/scribe/transcribe — turn a stored audio attachment into a raw +// transcript via the user's speech provider (OpenAI/Gemini). The audio does NOT +// pass through Veil or the chat loop. +scribeRouter.post("/transcribe", async (req, res, next) => { + try { + await ensureScribeAllowed(req); + const { attachmentId } = transcribeSchema.parse(req.body); + const row = await getAttachmentRow(req.organizationId!, attachmentId); + if (!row) throw new HttpError(404, "Recording not found."); + if (!row.mimeType.startsWith("audio/")) { + throw new HttpError(400, "That attachment is not an audio recording."); + } + + const settings = await getAiSettings(req.user!.id); + const buffer = await readFile(absolutePath(row.storagePath)); + const { transcript, provider } = await transcribeAudio(settings, { + buffer, + mimeType: row.mimeType, + filename: row.filename, + }); + + void recordActivity({ + orgId: req.organizationId!, + actor: { id: req.user!.id, name: req.user!.name }, + action: `Transcribed a visit recording (${provider})`, + entityType: "patient", + patientFileNumber: row.fileNumber, + }); + + res.json({ transcript }); + } catch (err) { + next(err); + } +}); + +const draftSchema = z.object({ + fileNumber: z.string().trim().min(1), + transcript: z.string().trim().min(1).max(100_000), + visitType: z.string().trim().max(120).optional(), + date: z.string().trim().max(40).optional(), +}); + +// Strip ```json fences and parse; returns null if the text isn't JSON. +function parseDraftJson(text: string): { type?: string; summary?: string } | null { + const cleaned = text + .replace(/^\s*```(?:json)?/i, "") + .replace(/```\s*$/i, "") + .trim(); + try { + const parsed = JSON.parse(cleaned); + return typeof parsed === "object" && parsed ? parsed : null; + } catch { + return null; + } +} + +const DRAFT_SYSTEM = [ + "You are a clinical scribe. From a visit transcript, write a concise, structured", + "encounter note in SOAP format (Subjective, Objective, Assessment, Plan).", + "Rules:", + "- Use ONLY what the transcript supports. Never invent vitals, doses, or findings.", + "- If a SOAP section has nothing to report, write \"Not discussed.\" under it.", + "- Identifiers may appear as tokens like [PATIENT_1] or [PROVIDER_1]; keep them", + " verbatim — do not guess real names.", + "Respond with a JSON object ONLY (no prose, no code fences):", + '{ "type": "",', + ' "summary": "" }', +].join("\n"); + +// POST /api/scribe/draft — draft an encounter note from a transcript. The +// transcript + patient context are Veil-redacted before any external call and +// the output is rehydrated. Nothing is written — the clinician reviews and +// approves via POST /save (the same write-approval gate as the chat agent). +scribeRouter.post("/draft", async (req, res, next) => { + try { + await ensureScribeAllowed(req); + const { fileNumber, transcript, visitType, date } = draftSchema.parse( + req.body, + ); + + const patient = await getPatient( + req.organizationId!, + fileNumber, + false, + providerScope(req.memberRole, req.user!.id), + ); + if (!patient) throw new HttpError(404, "Patient not found."); + + const settings = await getAiSettings(req.user!.id); + const resolved = resolveModel(settings, settings.defaultModel); + const veil = createVeil(settings.veilLevel, resolved.isExternal); + + // Seed Veil's token maps with this patient's identifiers, then redact the + // free-text transcript against them before it leaves the clinic. + const redactedPatient = veil.redactPatient(patient); + const redactedTranscript = veil.redactText(transcript); + + const context = [ + `Patient: ${redactedPatient.name} (MRN ${redactedPatient.fileNumber}), ${patient.age}y ${patient.sex}.`, + patient.problems.length + ? `Known problems: ${patient.problems.map((p) => p.label).join(", ")}.` + : "", + patient.medications.length + ? `Current medications: ${patient.medications + .map((m) => `${m.name} ${m.dose}`) + .join(", ")}.` + : "", + visitType ? `Visit type hint: ${visitType}.` : "", + "", + "Transcript:", + redactedTranscript, + ] + .filter(Boolean) + .join("\n"); + + const result = await generateText({ + model: resolved.model, + system: DRAFT_SYSTEM, + prompt: context, + }); + + const parsed = parseDraftJson(result.text); + const summary = veil.rehydrate( + (parsed?.summary ?? result.text ?? "").trim(), + ); + const type = (parsed?.type ?? visitType ?? "Visit").trim() || "Visit"; + + const draft: Encounter = { + date: date || new Date().toISOString().slice(0, 10), + type, + // The responsible clinician is the signed-in user, not the model's guess. + provider: req.user!.name ?? "", + summary, + }; + + res.json({ + draft, + veil: { + active: veil.active, + level: veil.level, + classes: veil.usedClasses(), + provider: resolved.providerLabel, + }, + }); + } catch (err) { + next(err); + } +}); + +const saveSchema = z.object({ + fileNumber: z.string().trim().min(1), + encounter: z.object({ + date: z.string().trim().min(1), + type: z.string().trim().min(1), + provider: z.string().trim().default(""), + summary: z.string().trim().min(1), + }), +}); + +// POST /api/scribe/save — the approval step: append the reviewed encounter note +// to the patient record. Re-validates server-side and audits like any add. +scribeRouter.post("/save", async (req, res, next) => { + try { + await ensureScribeAllowed(req); + const { fileNumber, encounter } = saveSchema.parse(req.body); + const updated = await appendEncounter( + req.organizationId!, + fileNumber, + encounter, + ); + if (!updated) throw new HttpError(404, "Patient not found."); + + void recordActivity({ + orgId: req.organizationId!, + actor: { id: req.user!.id, name: req.user!.name }, + action: `Added a scribe visit note for ${updated.name}`, + entityType: "patient", + entityId: updated.fileNumber, + patientName: updated.name, + patientFileNumber: updated.fileNumber, + }); + + res.status(201).json(updated); + } catch (err) { + next(err); + } +}); diff --git a/backend/src/services/ai/transcribe.ts b/backend/src/services/ai/transcribe.ts new file mode 100644 index 0000000..c7ee40f --- /dev/null +++ b/backend/src/services/ai/transcribe.ts @@ -0,0 +1,132 @@ +import { HttpError } from "../../lib/http-error.js"; +import type { userAiSettings } from "../../db/schema/ai.js"; +import { getApiKey } from "./config.js"; + +type AiSettingsRow = typeof userAiSettings.$inferSelect; + +export type AudioInput = { + buffer: Buffer; + mimeType: string; + filename: string; +}; + +// Which transcription backend a user's AI settings can reach. Anthropic has no +// speech-to-text API, so an Anthropic-only user must paste a transcript instead. +export type TranscribeProvider = "openai" | "gemini"; + +export function transcribeProviderFor( + settings: AiSettingsRow, +): TranscribeProvider | null { + if (getApiKey(settings, "openai")) return "openai"; + if (getApiKey(settings, "gemini")) return "gemini"; + return null; +} + +const OPENAI_MODEL = "whisper-1"; +const GEMINI_MODEL = "gemini-2.5-flash"; + +const TRANSCRIBE_PROMPT = + "Transcribe this clinical visit recording verbatim. Return only the spoken words as plain text, with no commentary, headings, or timestamps."; + +async function transcribeWithOpenAI( + apiKey: string, + audio: AudioInput, +): Promise { + const form = new FormData(); + form.append( + "file", + new Blob([new Uint8Array(audio.buffer)], { type: audio.mimeType }), + audio.filename, + ); + form.append("model", OPENAI_MODEL); + form.append("response_format", "json"); + + const res = await fetch("https://api.openai.com/v1/audio/transcriptions", { + method: "POST", + headers: { Authorization: `Bearer ${apiKey}` }, + body: form, + }); + if (!res.ok) { + const detail = await res.text().catch(() => ""); + throw new HttpError( + 502, + `Transcription failed (OpenAI ${res.status}). ${detail.slice(0, 300)}`, + ); + } + const json = (await res.json()) as { text?: string }; + return (json.text ?? "").trim(); +} + +async function transcribeWithGemini( + apiKey: string, + audio: AudioInput, +): Promise { + const url = `https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_MODEL}:generateContent?key=${encodeURIComponent( + apiKey, + )}`; + const res = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + contents: [ + { + parts: [ + { + inline_data: { + mime_type: audio.mimeType, + data: audio.buffer.toString("base64"), + }, + }, + { text: TRANSCRIBE_PROMPT }, + ], + }, + ], + }), + }); + if (!res.ok) { + const detail = await res.text().catch(() => ""); + throw new HttpError( + 502, + `Transcription failed (Gemini ${res.status}). ${detail.slice(0, 300)}`, + ); + } + const json = (await res.json()) as { + candidates?: { content?: { parts?: { text?: string }[] } }[]; + }; + const text = + json.candidates?.[0]?.content?.parts + ?.map((p) => p.text ?? "") + .join("") + .trim() ?? ""; + return text; +} + +// Send an audio recording to the user's transcription provider and return the +// raw transcript. The audio never passes through the chat loop or Veil — Veil +// cannot redact speech, so the caller must warn the clinician that audio leaves +// the clinic when an external provider is used (only the DRAFTING step is +// Veil-protected). Throws a 400 when no speech-capable provider is configured. +export async function transcribeAudio( + settings: AiSettingsRow, + audio: AudioInput, +): Promise<{ transcript: string; provider: TranscribeProvider }> { + const provider = transcribeProviderFor(settings); + if (!provider) { + throw new HttpError( + 400, + "Transcription needs an OpenAI or Gemini API key. Add one in Settings → AI, or paste the visit transcript instead.", + ); + } + const apiKey = getApiKey(settings, provider)!; + const transcript = + provider === "openai" + ? await transcribeWithOpenAI(apiKey, audio) + : await transcribeWithGemini(apiKey, audio); + if (!transcript) { + throw new HttpError( + 502, + "The transcription came back empty — try again or paste the transcript.", + ); + } + return { transcript, provider }; +} diff --git a/backend/src/services/ai/veil.ts b/backend/src/services/ai/veil.ts index 389aef8..1f2a1fe 100644 --- a/backend/src/services/ai/veil.ts +++ b/backend/src/services/ai/veil.ts @@ -26,6 +26,15 @@ export type Veil = { redactPatient: (patient: Patient) => Patient; /** Map a possibly-tokenized file number from a tool call back to the real one. */ resolveFileNumber: (input: string) => string; + /** + * De-identify free text (e.g. a visit transcript) by swapping any KNOWN + * identifiers — the patient name, MRN and provider names already seen via + * redactPatient — for their tokens. Seed the token maps by calling + * redactPatient(patient) first. Note: this only catches identifiers we know + * about; free-text PHI spoken aloud (addresses, relatives' names) is not + * covered — see the ambient-scribe consent notice. + */ + redactText: (text: string) => string; /** Swap any tokens in model output back to real identifiers. */ rehydrate: (text: string) => string; /** Token classes actually emitted — for the audit log. */ @@ -81,6 +90,21 @@ export function createVeil(level: VeilLevel, active: boolean): Veil { return mrnByToken.get(input.trim()) ?? input; } + function redactText(text: string): string { + if (!isActive || fromToken.size === 0) return text; + let out = text; + // Longest real values first so a provider name containing the patient name + // (or similar overlap) is replaced whole before its substrings. + const byLength = [...fromToken.entries()] + .filter(([, real]) => real.trim().length > 0) + .sort((a, b) => b[1].length - a[1].length); + for (const [token, real] of byLength) { + const escaped = real.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + out = out.replace(new RegExp(escaped, "gi"), token); + } + return out; + } + function rehydrate(text: string): string { if (!isActive || fromToken.size === 0) return text; let out = text; @@ -101,6 +125,7 @@ export function createVeil(level: VeilLevel, active: boolean): Veil { level, redactPatient, resolveFileNumber, + redactText, rehydrate, usedClasses, }; diff --git a/backend/src/services/patients.ts b/backend/src/services/patients.ts index 948bcb8..45f1cfe 100644 --- a/backend/src/services/patients.ts +++ b/backend/src/services/patients.ts @@ -570,6 +570,46 @@ export async function appendLabs( return getPatient(orgId, fileNumber); } +// Append a single encounter (visit note) without touching the rest of the +// record — used by the ambient AI scribe, which drafts one note at a time and +// must not go through updatePatient's wholesale child replacement. Position +// continues after the current max so the new note sorts last. +export async function appendEncounter( + orgId: string, + fileNumber: string, + entry: Encounter, +): Promise { + const inserted = await db.transaction(async (tx) => { + const [existing] = await tx + .select({ id: patients.id }) + .from(patients) + .where( + and( + eq(patients.organizationId, orgId), + eq(patients.fileNumber, fileNumber), + ), + ); + if (!existing) return false; + + const [pos] = await tx + .select({ max: sql`coalesce(max(${encounters.position}), -1)` }) + .from(encounters) + .where(eq(encounters.patientId, existing.id)); + await tx.insert(encounters).values({ + patientId: existing.id, + position: (pos?.max ?? -1) + 1, + ...entry, + }); + await tx + .update(patients) + .set({ updatedAt: new Date() }) + .where(eq(patients.id, existing.id)); + return true; + }); + if (!inserted) return null; + return getPatient(orgId, fileNumber); +} + // Remove a single lab result from a patient, identified by its // name/value/takenAt (the frontend has no row id). Scoped to the org via the // owning patient. Returns the reloaded patient, or null when the chart is gone. diff --git a/frontend/components/patients/patient-detail-sheet.tsx b/frontend/components/patients/patient-detail-sheet.tsx index 95fadbc..eb7a65a 100644 --- a/frontend/components/patients/patient-detail-sheet.tsx +++ b/frontend/components/patients/patient-detail-sheet.tsx @@ -7,6 +7,7 @@ import { AiBadge } from "@/components/ai-badge"; import { PatientFormDialog } from "@/components/chat/patient-form-dialog"; import { RecordGraph } from "@/components/graph/record-graph"; import { PatientDetail } from "@/components/patients/patient-detail"; +import { ScribeDialog } from "@/components/patients/scribe-dialog"; import { TransferPatientDialog } from "@/components/patients/transfer-patient-dialog"; import { ConfirmDialog } from "@/components/ui/confirm-dialog"; import { @@ -28,6 +29,7 @@ import { type Appointment, listAppointments } from "@/lib/appointments"; import { type Invoice, listInvoices } from "@/lib/invoices"; import { deletePatient, getPatient, type Patient } from "@/lib/patients"; import { listPrescriptions, type Prescription } from "@/lib/prescriptions"; +import { useAiAccess } from "@/lib/ai-policy"; import { hasClinicalAccess, useActiveRole } from "@/lib/roles"; import { notify } from "@/lib/toast"; @@ -76,9 +78,14 @@ export function PatientDetailSheet({ // Deleting a chart is destructive — only offer it once we know the role is // a full clinician (patient:delete), never optimistically. const canDelete = role != null && hasClinicalAccess(role); + // The ambient scribe writes a clinical note, so it needs full clinical write + // access AND the clinic's AI must be enabled for this member. + const { allowed: aiAllowed } = useAiAccess(); + const canScribe = role != null && hasClinicalAccess(role) && aiAllowed; const [patient, setPatient] = useState(null); const [status, setStatus] = useState("loading"); const [editOpen, setEditOpen] = useState(false); + const [scribeOpen, setScribeOpen] = useState(false); const [transferOpen, setTransferOpen] = useState(false); const [confirmOpen, setConfirmOpen] = useState(false); // Graph popped out of the sheet into its own dialog (the sheet closes first). @@ -171,6 +178,7 @@ export function PatientDetailSheet({ setEditKey((k) => k + 1); setEditOpen(true); }} + onScribe={canScribe ? () => setScribeOpen(true) : undefined} onOpenGraph={() => { onOpenChange(false); setGraphOpen(true); @@ -197,6 +205,15 @@ export function PatientDetailSheet({ /> )} + {patient && ( + setPatient(updated)} + open={scribeOpen} + patient={patient} + /> + )} + {patient && ( void; + // Opens the ambient AI visit scribe (record/transcribe → draft note). + onScribe?: () => void; onTransfer?: () => void; onDelete?: () => void; // Pops the record graph out into its own dialog (closing this sheet). @@ -287,6 +297,12 @@ export function PatientDetail({ {t("patients.transfer.action")} )} + {onScribe && ( + + )} {onEdit && (