From 6e5598262e722683c10d8d8a2e33cfac4e158a1e Mon Sep 17 00:00:00 2001 From: Khalid Abdi Date: Mon, 15 Jun 2026 18:27:43 +0300 Subject: [PATCH] feat(chat): file attachments as chips, CoT closed, AI add-to-inventory Chat fixes (#1/#6/#7): - chat-input passes raw File[] up instead of inlining file text - chat-panel sends files as FileUIPart and renders them with the ai-elements Attachments component (no more raw text dumps) - backend extracts text-like file content for the model in routes/chat - Chain-of-Thought now defaults to collapsed - file upload works regardless of whether the input has text AI add-to-inventory (#2): - new proposeInventory tool (validates items, streams an approval card) - system prompt distinguishes stocking inventory vs. billing a patient - ActionPreviewCard commits inventory via POST /api/inventory Co-Authored-By: Claude Opus 4.8 --- backend/src/routes/chat.ts | 53 ++++++++++- backend/src/services/ai/tools.ts | 72 ++++++++++++++ .../components/chat/action-preview-card.tsx | 21 +++++ frontend/components/chat/chat-input.tsx | 34 ++----- frontend/components/chat/chat-panel.tsx | 93 +++++++++++++++---- frontend/lib/ai-chat.ts | 3 +- frontend/lib/i18n/locales/en/translation.json | 10 +- 7 files changed, 235 insertions(+), 51 deletions(-) diff --git a/backend/src/routes/chat.ts b/backend/src/routes/chat.ts index 1966b08..fa03e67 100644 --- a/backend/src/routes/chat.ts +++ b/backend/src/routes/chat.ts @@ -33,6 +33,50 @@ export const chatRouter = Router(); chatRouter.use(requireAuth, requireOrg, requirePermission({ patient: ["read"] })); +// Text-like uploads (CSV/JSON/TXT/…) the model should read as parseable text +// rather than an opaque data URL. Images/PDFs are left as file parts so +// vision-capable providers read them directly. +const TEXT_LIKE_MEDIA = /^(text\/|application\/(json|xml|csv|x-ndjson))/i; +const TEXT_LIKE_EXT = /\.(csv|tsv|json|txt|md|xml|ndjson|tab)$/i; + +function decodeDataUrl(url: string): string { + const comma = url.indexOf(","); + if (comma === -1) return ""; + const meta = url.slice(0, comma); + const data = url.slice(comma + 1); + return meta.includes("base64") + ? Buffer.from(data, "base64").toString("utf8") + : decodeURIComponent(data); +} + +// Replace text-like file parts with a text part carrying the file's content +// (capped) so the agent can parse uploads (e.g. a medications list to add to +// inventory, or a database export to import). Display/storage are unaffected — +// this only shapes what the model sees. +function inlineTextFiles(messages: UIMessage[]): UIMessage[] { + return messages.map((message) => { + if (!Array.isArray(message.parts)) return message; + const parts = message.parts.flatMap((part) => { + if ( + part.type === "file" && + typeof part.url === "string" && + (TEXT_LIKE_MEDIA.test(part.mediaType ?? "") || + TEXT_LIKE_EXT.test(part.filename ?? "")) + ) { + const content = decodeDataUrl(part.url).slice(0, 200_000); + return [ + { + type: "text" as const, + text: `--- File: ${part.filename ?? "file"} ---\n${content}`, + }, + ]; + } + return [part]; + }); + return { ...message, parts } as UIMessage; + }); +} + function systemPrompt(veilActive: boolean, providerLabel: string): string { return [ "You are temetro, a clinical assistant that helps clinicians retrieve,", @@ -55,10 +99,15 @@ function systemPrompt(veilActive: boolean, providerLabel: string): string { " asks to add/book/create one. They show an approval card; the record is only", " written after the clinician clicks Add. NEVER say you added/booked/created", " something — say you've drafted it for their approval.", + "- proposeInventory: when the clinician wants to ADD STOCK to the clinic's", + " inventory — e.g. they upload a list of medications/supplies with quantities", + " (and optionally prices) to stock. Parse it into items {name, form, strength,", + " unit, stockQuantity, reorderThreshold, expiresAt} and call proposeInventory.", "- proposeInvoice: when the clinician wants to bill someone — e.g. they upload", " a list of purchased medications/items. Parse it into line items", " {description, quantity, unitPrice} (use the prices in the document) and call", - " proposeInvoice with the patient/client name.", + " proposeInvoice with the patient/client name. (Stocking inventory vs. billing a", + " patient are different — pick proposeInventory for the former.)", "- previewImport: when the clinician wants to import/migrate an existing", " patient database file, or add a single patient. Parse the uploaded content", " into our patient shape and call previewImport.", @@ -113,7 +162,7 @@ chatRouter.post("/", async (req, res, next) => { }, }; - const modelMessages = await convertToModelMessages(messages); + const modelMessages = await convertToModelMessages(inlineTextFiles(messages)); const system = systemPrompt(veil.active, resolved.providerLabel); const stream = createUIMessageStream({ diff --git a/backend/src/services/ai/tools.ts b/backend/src/services/ai/tools.ts index 8029f85..a7155b3 100644 --- a/backend/src/services/ai/tools.ts +++ b/backend/src/services/ai/tools.ts @@ -8,6 +8,7 @@ import { db } from "../../db/index.js"; import { organization } from "../../db/schema/auth.js"; import { appointmentInputSchema } from "../../lib/appointment-validation.js"; import { initialsFromName } from "../../lib/initials.js"; +import { inventoryInputSchema } from "../../lib/inventory-validation.js"; import { invoiceInputSchema } from "../../lib/invoice-validation.js"; import { patientInputSchema } from "../../lib/patient-validation.js"; import { prescriptionInputSchema } from "../../lib/prescription-validation.js"; @@ -474,6 +475,77 @@ export function createChatTools(ctx: ToolContext) { }, }), + proposeInventory: tool({ + description: + "Propose adding one or more items to the clinic's inventory (medications/supplies) for the clinician to approve — e.g. parse an uploaded stock or purchase list into stock items. Does NOT save; it shows an approval card the clinician confirms. Each item needs a name; form/strength/unit/stockQuantity/reorderThreshold/location/expiresAt (YYYY-MM-DD)/notes are optional. Use this for STOCKING inventory; use proposeInvoice instead when billing a patient for purchased items.", + inputSchema: z.object({ + items: z + .array( + z.object({ + name: z.string().describe("Item / medication name"), + form: z + .string() + .optional() + .describe("Dosage form, e.g. Tablet, Capsule, Syrup"), + strength: z.string().optional().describe("e.g. 500mg"), + unit: z + .string() + .optional() + .describe("Dispensing unit, e.g. box, bottle"), + stockQuantity: z + .number() + .optional() + .describe("Units currently in stock"), + reorderThreshold: z + .number() + .optional() + .describe("Low-stock reorder level"), + location: z.string().optional().describe("Storage location"), + expiresAt: z + .string() + .nullish() + .describe("Expiry date, YYYY-MM-DD"), + notes: z.string().nullish(), + }), + ) + .describe("Inventory items to add (prices/quantities from the document)"), + }), + execute: async ({ items }) => { + step(`Drafting ${items.length} inventory item(s)`); + // Inventory is non-PHI — no Veil resolution needed. + const validated: unknown[] = []; + const issues: string[] = []; + items.forEach((item, index) => { + const parsed = inventoryInputSchema.safeParse(item); + if (parsed.success) { + validated.push(parsed.data); + } else { + issues.push( + ...parsed.error.issues.map( + (i) => + `item ${index + 1} ${i.path.join(".") || "(root)"}: ${i.message}`, + ), + ); + } + }); + writer.write({ + type: "data-actionPreview", + data: { + token: `inventory-${stepSeq}`, + kind: "inventory" as const, + record: { items: validated.length ? validated : items }, + issues, + }, + }); + return { + ok: issues.length === 0, + count: validated.length, + issues, + note: "Preview only — awaiting clinician approval before any write.", + }; + }, + }), + proposeInvoice: tool({ description: "Propose a new invoice for the clinician to approve — e.g. parse an uploaded list of purchased medications into billable line items. Does NOT save; it shows an approval card the clinician confirms. Provide the patient/client name (a file number if known) and line items {description, quantity, unitPrice}; prices come from the uploaded document.", diff --git a/frontend/components/chat/action-preview-card.tsx b/frontend/components/chat/action-preview-card.tsx index 7d8d656..100ca68 100644 --- a/frontend/components/chat/action-preview-card.tsx +++ b/frontend/components/chat/action-preview-card.tsx @@ -2,6 +2,7 @@ import { AlertTriangle, + Boxes, CalendarPlus, Check, ClipboardList, @@ -22,6 +23,7 @@ import { type InvoiceInput, type InvoiceLineItem, } from "@/lib/invoices"; +import { type InventoryInput, createInventory } from "@/lib/inventory"; import { type PrescriptionInput, createPrescription } from "@/lib/prescriptions"; import { type TaskInput, createTask } from "@/lib/tasks"; import { notify } from "@/lib/toast"; @@ -33,6 +35,7 @@ export const ACTION_ICONS = { task: ClipboardList, prescription: Pill, invoice: Receipt, + inventory: Boxes, } as const; const ICONS = ACTION_ICONS; @@ -61,6 +64,18 @@ export function summarize(data: ActionPreviewData): string[] { `${items.length} item${items.length === 1 ? "" : "s"} · ${formatMoney(total)}`, ].filter(Boolean); } + if (data.kind === "inventory") { + const items = (r.items as InventoryInput[] | undefined) ?? []; + return [ + `${items.length} item${items.length === 1 ? "" : "s"}`, + items + .map((it) => + [it.name, it.strength].filter(Boolean).join(" ") + + (it.stockQuantity ? ` ×${it.stockQuantity}` : ""), + ) + .join(", "), + ].filter(Boolean); + } // prescription return [ [r.medication, r.dose].filter(Boolean).join(" "), @@ -81,6 +96,12 @@ export async function commitAction(data: ActionPreviewData): Promise { await createTask(data.record as TaskInput); } else if (data.kind === "invoice") { await createInvoice({ ...(data.record as InvoiceInput), source: "ai" }); + } else if (data.kind === "inventory") { + const { items = [] } = data.record as { items?: InventoryInput[] }; + // Commit each proposed stock item via the RBAC-gated create endpoint. + for (const item of items) { + await createInventory(item); + } } else { await createPrescription({ ...(data.record as PrescriptionInput), diff --git a/frontend/components/chat/chat-input.tsx b/frontend/components/chat/chat-input.tsx index e3b7153..e225777 100644 --- a/frontend/components/chat/chat-input.tsx +++ b/frontend/components/chat/chat-input.tsx @@ -17,7 +17,7 @@ import type { Effort } from "@/lib/ai-models"; import { cn } from "@/lib/utils"; type ChatInputProps = { - onSubmit: (text: string) => void; + onSubmit: (text: string, files: File[]) => void; status: ChatStatus; onStop?: () => void; model: string; @@ -55,36 +55,16 @@ export function ChatInput({ const canSend = (value.trim().length > 0 || files.length > 0) && !isGenerating; - const submit = useCallback(async () => { + const submit = useCallback(() => { const trimmed = value.trim(); // Allow submitting while generating — the panel queues it (Claude-style). if (!trimmed && files.length === 0) { return; } - const parts: string[] = []; - if (trimmed) { - parts.push(trimmed); - } - // Include the text of attached files so the agent can parse them (e.g. a - // database export to import). Read text-like files; cap each so a huge file - // can't blow the context. Binary files are referenced by name only. - for (const file of files) { - const textLike = - /\.(csv|tsv|json|txt|md|xml|ndjson|tab)$/i.test(file.name) || - file.type.startsWith("text/") || - file.type === "application/json"; - if (textLike) { - try { - const content = (await file.text()).slice(0, 200_000); - parts.push(`--- File: ${file.name} ---\n${content}`); - } catch { - parts.push(`[Attached: ${file.name}]`); - } - } else { - parts.push(`[Attached: ${file.name}]`); - } - } - onSubmit(parts.join("\n\n")); + // Hand the raw files to the panel; it sends them as proper attachment parts + // (rendered as chips, not raw inlined text) and the backend extracts any + // text-like content for the model. + onSubmit(trimmed, files); setValue(""); setFiles([]); }, [value, files, onSubmit]); @@ -234,7 +214,7 @@ export function ChatInput({ onSubmit(`/patient ${fileNumber}`)} + onCreated={(fileNumber) => onSubmit(`/patient ${fileNumber}`, [])} onOpenChange={setAddOpen} open={addOpen} /> diff --git a/frontend/components/chat/chat-panel.tsx b/frontend/components/chat/chat-panel.tsx index 6e0393e..d3b2e69 100644 --- a/frontend/components/chat/chat-panel.tsx +++ b/frontend/components/chat/chat-panel.tsx @@ -1,13 +1,23 @@ "use client"; import { useChat } from "@ai-sdk/react"; -import { DefaultChatTransport, type ToolUIPart } from "ai"; +import { + DefaultChatTransport, + type FileUIPart, + type ToolUIPart, +} from "ai"; import { AlertTriangle, Brain, ChevronDown, ShieldCheck, X } from "lucide-react"; import { nanoid } from "nanoid"; import { useSearchParams } from "next/navigation"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; +import { + Attachment, + AttachmentInfo, + AttachmentPreview, + Attachments, +} from "@/components/ai-elements/attachments"; import { ChainOfThought, ChainOfThoughtContent, @@ -83,6 +93,23 @@ import { getPatient } from "@/lib/patients"; // pulls records instantly without the LLM (also works offline). const PATIENT_COMMAND = /^\/(?:patient\s+)?(\d+)$/i; +// Read a File into a FileUIPart (data URL). The backend extracts text-like +// content for the model; images/PDFs are read directly by vision providers. +function fileToPart(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => + resolve({ + type: "file", + mediaType: file.type || "application/octet-stream", + filename: file.name, + url: reader.result as string, + }); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(file); + }); +} + export function ChatPanel() { const { t } = useTranslation(); const [model, setModel] = useState(DEFAULT_MODEL_ID); @@ -92,11 +119,14 @@ export function ChatPanel() { // per session before the first such send — inline (no modal). `pendingConsent` // holds the message text waiting on that one-time approval. const [consented, setConsented] = useState(false); - const [pendingConsent, setPendingConsent] = useState(null); + const [pendingConsent, setPendingConsent] = useState<{ + text: string; + files: File[]; + } | null>(null); // Claude-style message queue: messages submitted while the assistant is busy // (or waiting on the Veil gate) wait here and auto-send when it goes idle. - const [queued, setQueued] = useState([]); + const [queued, setQueued] = useState<{ text: string; files: File[] }[]>([]); // Persisted conversation: a client-owned thread id (a fresh one per new chat), // saved to the server after each exchange so history survives reloads. @@ -151,9 +181,10 @@ export function ChatPanel() { // Run the LLM agent for a message (after any Veil gate) on a given model. const runAgentWith = useCallback( - (text: string, modelId: string) => { + async (text: string, modelId: string, files: File[] = []) => { + const fileParts = await Promise.all(files.map(fileToPart)); sendMessage( - { text }, + { text, files: fileParts }, { body: { model: modelId, effort, threadId: threadIdRef.current } }, ); }, @@ -161,13 +192,13 @@ export function ChatPanel() { ); const send = useCallback( - async (text: string) => { + async (text: string, files: File[] = []) => { const trimmed = text.trim(); - if (!trimmed) return; + if (!trimmed && files.length === 0) return; // Busy or awaiting the Veil gate → queue and auto-send when idle. if (status === "submitted" || status === "streaming" || pendingConsent) { - setQueued((q) => [...q, trimmed]); + setQueued((q) => [...q, { text: trimmed, files }]); return; } @@ -206,10 +237,10 @@ export function ChatPanel() { // Cloud model → inline Veil consent once before sending externally. if (isCloudModel && !consented) { - setPendingConsent(trimmed); + setPendingConsent({ text: trimmed, files }); return; } - runAgentWith(trimmed, model); + void runAgentWith(trimmed, model, files); }, [ consented, @@ -228,22 +259,22 @@ export function ChatPanel() { if (status !== "ready" || pendingConsent || queued.length === 0) return; const [next, ...rest] = queued; setQueued(rest); - if (next) void send(next); + if (next) void send(next.text, next.files); }, [status, pendingConsent, queued, send]); // Veil gate actions. const confirmConsent = useCallback(() => { setConsented(true); - const text = pendingConsent; + const pending = pendingConsent; setPendingConsent(null); - if (text) runAgentWith(text, model); + if (pending) void runAgentWith(pending.text, model, pending.files); }, [pendingConsent, runAgentWith, model]); const useLocalInstead = useCallback(() => { setModel("ollama"); - const text = pendingConsent; + const pending = pendingConsent; setPendingConsent(null); - if (text) runAgentWith(text, "ollama"); + if (pending) void runAgentWith(pending.text, "ollama", pending.files); }, [pendingConsent, runAgentWith]); const cancelConsent = useCallback(() => setPendingConsent(null), []); @@ -381,10 +412,13 @@ export function ChatPanel() { {queued.map((q, i) => ( - +
- {q} + + {q.text || + t("chat.queue.attachmentsOnly", { count: q.files.length })} + p.type === "data-actionPreview", ); + // Attachments the clinician uploaded — rendered once as a chip group. + const fileParts = message.parts.filter((p) => p.type === "file"); + const firstFileIdx = message.parts.findIndex((p) => p.type === "file"); return ( {steps.length > 0 ? ( {t("chat.steps")} @@ -478,6 +515,26 @@ export function ChatPanel() { {part.text} ); } + if (part.type === "file") { + // Render the whole message's files as one chip group, once. + if (i !== firstFileIdx) return null; + return ( + + {fileParts.map((fp, fi) => ( + + + + + ))} + + ); + } if (part.type === "data-patientCard") { return (