From a6a856107ef332d7d90ca2a5d9cf539329d4d511 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:47:02 +0100 Subject: [PATCH] fix(assistant): backport invocation and evidence identity repair Release v6.4 reproduces lost concurrent invocation IDs and duplicate approval cards. Backport the four runtime/test files from reviewed main fix 33b852f66b and portable synthetic browser fixture 2d36bb7d4d, without private replay attestations or unrelated main state. All 167 affected units and three browser widths pass; independent release-parent runtime substitutions fail. Record fresh scoped browser evidence and contract obligations; this does not qualify providers or release promotion. Change-source: pulse-maintainer --- .../PATROL_ASSISTANT_CUSTOMER_JOURNEY.md | 23 ++ .../v6/internal/subsystems/ai-runtime.md | 6 + frontend-modern/browser-verification.json | 30 ++- .../assistant-identity/README.md | 37 +++ .../assistant-identity/fixture.tsx | 72 ++++++ .../assistant-identity/index.html | 10 + .../src/components/AI/Chat/ChatMessages.tsx | 122 +++++---- .../AI/Chat/__tests__/ChatMessages.test.tsx | 73 ++++++ .../AI/Chat/__tests__/useChat.test.ts | 90 +++++++ .../src/components/AI/Chat/hooks/useChat.ts | 15 +- scripts/check-assistant-identity.mjs | 232 ++++++++++++++++++ 11 files changed, 626 insertions(+), 84 deletions(-) create mode 100644 docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md create mode 100644 frontend-modern/qualification/assistant-identity/README.md create mode 100644 frontend-modern/qualification/assistant-identity/fixture.tsx create mode 100644 frontend-modern/qualification/assistant-identity/index.html create mode 100644 scripts/check-assistant-identity.mjs diff --git a/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md b/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md new file mode 100644 index 000000000..062d7be2f --- /dev/null +++ b/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md @@ -0,0 +1,23 @@ + +## Release-line identity regression verification — 6 September 2026 + +Scoped backport of main fix `33b852f66b` onto `a5caca0d63`, with portable +synthetic fixture from `2d36bb7d4d`. No private capture or main browser receipt +is used. `scripts/check-assistant-identity.mjs` exercises this checkout's real +reducer and renderer at 1440, 900 and 390 pixels; browser service workers are +blocked and API/external HTTP requests are aborted and fail the check. + +All 167 affected unit tests pass. The browser check passes at all three widths: +concurrent same-name IDs stay distinct, completion retains the sibling approval, +and removal of a workflow row preserves shared input/output evidence. Each +original runtime file substituted independently from the release parent fails +the browser check; restored fixed files pass. Desktop approval and narrow +expanded-evidence screenshots were visually inspected. Transient running rows +remain visible in the synthetic streaming approval screenshot; this proof does +not assert complete workflow-status correctness. + +Evidence is retained with lane run `20260906T114016Z-release-line`, under +`evidence/final/receipt.json` (source and screenshot hashes), sensitivity logs, +and `evidence/units.log`. Reproduce using the fixture README. This is component +browser regression evidence only, not production routing/build, provider +reasoning, authorised action execution, installed receipt or release readiness. diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index c74636a29..daad986e1 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -2349,6 +2349,12 @@ deriving an older display status from `workflowStatusHistory`. normalized tool name when an older server omits IDs, upsert one pending tool row and collapse stale duplicate pending rows instead of replaying several near-identical steps in the transcript. + A supplied invocation ID is authoritative. Distinct IDs must not merge by + tool name during start, progress, cancellation, completion or approval + cleanup. Completing one invocation must leave its sibling approvals intact. + Message rendering keys rows by message ID and reads immutable transcript + values. It must not deeply reconcile shared tool objects: removing a status + row must never mutate evidence retained in another event or in toolCalls. `frontend-modern/src/api/aiChat.ts` owns the Assistant predicate: token content and hidden reasoning may continue to opt out of those checkpoints so answer streaming remains fast, while session, workflow, model-switch, tool, diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json index 2fbe08cc7..f305d1974 100644 --- a/frontend-modern/browser-verification.json +++ b/frontend-modern/browser-verification.json @@ -1,34 +1,40 @@ { "version": 1, - "base_sha": "89c8e61463d134408caef430b580b53e154e17f4", - "verified_at": "2026-09-06T00:31:33.609713Z", + "base_sha": "a5caca0d6311f1de18a9e55e26bb4e947895b319", + "verified_at": "2026-09-06T11:46:12.688Z", "result": "passed", "changed_paths": [ - "frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts" + "frontend-modern/src/components/AI/Chat/ChatMessages.tsx", + "frontend-modern/src/components/AI/Chat/hooks/useChat.ts" ], "content_sha256": { - "frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts": "1eadf6df30b4f1130f868b8f139561ea60fdb87e1823deeeaaf2f8fbc699a00c" + "frontend-modern/src/components/AI/Chat/ChatMessages.tsx": "9672f7608d1e3a531c73cba20fd4a78752316783212afd0c292ddfd11d2bf371", + "frontend-modern/src/components/AI/Chat/hooks/useChat.ts": "0b56b7a56e35d51ca96f0e126dd493b3164aa9e0ad4d8ae24bcf3af7a574b97c" }, "routes": [ - "/qualification (isolated caller/card fixture, not application routing)" + "/qualification/assistant-identity/ (loopback synthetic component fixture, not production routing)" ], "viewports": [ { "width": 1440, - "height": 900 + "height": 1000 + }, + { + "width": 900, + "height": 1000 }, { "width": 390, - "height": 900 + "height": 1000 } ], "states": [ - "Real Chromium with current useAlertDestinationsTabState, useNotificationDeliveryHealth and AlertDeliveryHealthCard; scripted API promises and queue actions. No installed backend or provider receipt.", - "Older healthy/newer degraded, older degraded/newer healthy, older error/newer healthy, older healthy/newer error. Warning presence and rendered state remain owned by newer request. Desktop and narrow screenshots inspected; this is not a full-shell or accessibility audit." + "Real current-checkout useChat and ChatMessages; concurrent invocation identity, sibling approval retention, settled results and shared evidence after workflow row removal. No backend, provider or real actions.", + "Desktop approval and narrow expanded-evidence screenshots visually inspected. Transient running rows in streaming approval screenshot are not evidence of complete workflow-status correctness." ], "interactions": [ - "pulse-heavy-run -- node scripts/check-delivery-health-ordering.mjs: 12 cases passed.", - "Configuration Retry overlaps pending mount health. Resolve newer then older and compare rendered main text and alert presence.", - "Retry retained deliveries and Dismiss retained failures each start post-action refresh while another read is pending. Older completion leaves loading true; latest healthy response clears warning. Confirmations and API responses are scripted." + "pulse-heavy-run -- node scripts/check-assistant-identity.mjs: all three widths pass; service workers blocked; API/external HTTP aborted and fail the check.", + "Independent release-parent useChat substitution fails lost invocation assertion; independent ChatMessages substitution fails with three approval cards instead of two; fixed files restored and final run passes.", + "Expand both retained input/output evidence rows; verify their text and absence of document overflow. 167 affected unit tests pass. Source/screenshot hashes: lane 20260906T114016Z-release-line evidence/final/receipt.json." ] } diff --git a/frontend-modern/qualification/assistant-identity/README.md b/frontend-modern/qualification/assistant-identity/README.md new file mode 100644 index 000000000..4e61a6840 --- /dev/null +++ b/frontend-modern/qualification/assistant-identity/README.md @@ -0,0 +1,37 @@ +# Assistant identity browser regression + +From the repository root (Node 24): + +```sh +npm ci --ignore-scripts +npm ci --ignore-scripts --prefix frontend-modern +npx playwright install chromium +pulse-heavy-run -- node scripts/check-assistant-identity.mjs +``` + +This standalone Vite fixture imports the current checkout's real `useChat` and +`ChatMessages`, including `MessageItem`, approval cards and tool detail rows. +It is not a production entry point and needs no backend, credentials, provider, +customer captures or live infrastructure. The only replaced API is the chat +stream callback; all browser API/external HTTP requests are blocked and fail the +check. Approval completion is synthetic, not evidence of an authorised action. + +Checks at 1440, 900 and 390 pixels cover concurrent same-name invocation IDs, +progress/completion isolation, retaining a sibling approval after completion, +settled rendered results, and shared input/output evidence after removal of a +workflow row. Both evidence rows are expanded and document overflow is checked. +The shared-object journey deliberately retains references, matching the original +regression rather than concealing it with cloned fixtures. + +Output defaults to `tmp/assistant-identity`; set `ASSISTANT_IDENTITY_OUTPUT` to +an absolute path to retain a candidate-specific run. A successful run writes +screenshots and `receipt.json` with browser version and SHA-256 source hashes. +Run on the actual backport checkout: a main receipt does not qualify release +content. This is a component/reducer browser proof, not the release admission +attestation, a production-build check, SSE transport qualification, provider +reasoning proof, or installed end-to-end action verification. + +Regression sensitivity can be checked by temporarily substituting either +`ChatMessages.tsx` or `hooks/useChat.ts` from the parent of fix `33b852f66b`: +each must fail independently. Restore files afterwards; never admit those +experimental substitutions as candidate content. diff --git a/frontend-modern/qualification/assistant-identity/fixture.tsx b/frontend-modern/qualification/assistant-identity/fixture.tsx new file mode 100644 index 000000000..155274cc1 --- /dev/null +++ b/frontend-modern/qualification/assistant-identity/fixture.tsx @@ -0,0 +1,72 @@ +// Synthetic component journey: real reducer and transcript, no application server. +import { createSignal } from 'solid-js'; +import { render } from 'solid-js/web'; +import { AIChatAPI, type StreamEvent } from '../../src/api/aiChat'; +import { useChat } from '../../src/components/AI/Chat/hooks/useChat'; +import { ChatMessages } from '../../src/components/AI/Chat/ChatMessages'; +import type { ChatMessage } from '../../src/components/AI/Chat/types'; +import '../../src/index.css'; + +let dispatch: (event: StreamEvent) => void; +AIChatAPI.chat = async (_prompt, _session, _model, onEvent) => { + dispatch = onEvent; + await new Promise(() => {}); // Keep stream open until the fixture is disposed. +}; +const noOp = () => {}; +function Fixture() { + const chat = useChat({ sessionId: 'synthetic-identity' }); + const [override, setOverride] = createSignal(null); + const a = { name: 'pulse_query', input: 'client', output: 'client evidence', success: true }; + const b = { name: 'pulse_alerts', input: 'alerts', output: 'alert evidence', success: true }; + const pendingA = { id: 'a', name: a.name, input: a.input }; + const pendingB = { id: 'b', name: b.name, input: b.input }; + const workflow = { + type: 'workflow_status' as const, + workflowStatus: { phase: 'provider_start', message: 'Starting' }, + }; + const message = (extra: Partial): ChatMessage => ({ + id: 'shared', + role: 'assistant', + content: '', + timestamp: new Date('2026-01-01T00:00:00Z'), + ...extra, + }); + Object.assign(window, { + identityFixture: { + start: () => { + void chat.sendMessage('Synthetic identity check'); + }, + fire: (event: StreamEvent) => dispatch(event), + snapshot: () => chat.messages().find((m) => m.role === 'assistant'), + evidence: () => [a, b], + shared: (stage: number) => + setOverride([ + message({ + toolCalls: stage === 0 ? [] : stage === 1 ? [a] : [a, b], + pendingTools: stage === 0 ? [pendingA, pendingB] : stage === 1 ? [pendingB] : [], + streamEvents: [ + ...(stage < 3 ? [workflow] : []), + stage === 0 + ? { type: 'pending_tool', toolId: 'a', pendingTool: pendingA } + : { type: 'tool', toolId: 'a', tool: a }, + stage < 2 + ? { type: 'pending_tool', toolId: 'b', pendingTool: pendingB } + : { type: 'tool', toolId: 'b', tool: b }, + ], + }), + ]), + }, + }); + return ( +
+ chat.updateApproval(id, approval.toolId, { removed: true })} + onSkip={(id, toolId) => chat.updateApproval(id, toolId, { removed: true })} + onAnswerQuestion={noOp} + onSkipQuestion={noOp} + /> +
+ ); +} +render(() => , document.getElementById('root')!); diff --git a/frontend-modern/qualification/assistant-identity/index.html b/frontend-modern/qualification/assistant-identity/index.html new file mode 100644 index 000000000..302d435a8 --- /dev/null +++ b/frontend-modern/qualification/assistant-identity/index.html @@ -0,0 +1,10 @@ + + + + + + +
+ + + diff --git a/frontend-modern/src/components/AI/Chat/ChatMessages.tsx b/frontend-modern/src/components/AI/Chat/ChatMessages.tsx index e6f3adf91..b5be6a9a8 100644 --- a/frontend-modern/src/components/AI/Chat/ChatMessages.tsx +++ b/frontend-modern/src/components/AI/Chat/ChatMessages.tsx @@ -1,5 +1,4 @@ import { Component, Show, For, createEffect, createMemo, createSignal } from 'solid-js'; -import { createStore, reconcile } from 'solid-js/store'; import ArrowDownIcon from 'lucide-solid/icons/arrow-down'; import { Button } from '@/components/shared/Button'; import { MessageItem } from './MessageItem'; @@ -66,22 +65,13 @@ export const ChatMessages: Component = (props) => { let containerRef: HTMLDivElement | undefined; const [isPinnedToBottom, setIsPinnedToBottom] = createSignal(true); - // useChat hands us a fresh, immutably-rebuilt message array on every stream - // event (each content chunk, workflow-status change, tool update spreads a new - // message object). Rendering that array directly through , which keys by - // object reference, tears down and recreates the whole MessageItem on every - // event — the visible flashing / rows popping in and out and the transcript - // jumping up and down during a turn. - // - // Reconcile the incoming array into a keyed store mirror so each message keeps - // a stable identity across updates (matched by id). MessageItem already reads - // every field through `() => props.message.x` accessors, so once it stops - // re-mounting, only the genuinely changed text/rows update in place. This keeps - // the streaming transcript stable the way OpenCode's timeline is. - const [mirroredMessages, setMirroredMessages] = createStore([]); - createEffect(() => { - setMirroredMessages(reconcile(props.messages, { key: 'id', merge: false })); - }); + // Key only the message rows. Deep store reconciliation mutates objects shared + // by toolCalls and streamEvents when a status row is removed, corrupting the + // source transcript. Read each immutable message through its stable ID instead. + const messagesById = createMemo( + () => new Map(props.messages.map((message) => [message.id, message])), + ); + const messageIds = createMemo(() => [...messagesById().keys()]); // Regenerate re-runs the LAST turn via session undo, so only the final // assistant answer qualifies, and only once it has settled with nothing @@ -342,54 +332,58 @@ export const ChatMessages: Component = (props) => { {/* Messages */} - - {(message) => { - const queuedMeta = createMemo(() => queuedFollowUpMetaByMessageId().get(message.id)); + + {(messageId) => { + const queuedMeta = createMemo(() => queuedFollowUpMetaByMessageId().get(messageId)); return ( - props.onApprove(message.id, approval)} - onSkip={(toolId) => props.onSkip(message.id, toolId)} - onAnswerQuestion={(question, answers) => - props.onAnswerQuestion(message.id, question, answers) - } - onSkipQuestion={(questionId) => props.onSkipQuestion(message.id, questionId)} - onRetry={props.onRetry} - onRegenerate={ - message.id === regenerableMessageId() - ? () => props.onRegenerate?.(message.id) - : undefined - } - onEditPrompt={ - message.id === editablePromptMessageId() - ? () => props.onEditPrompt?.(message.id) - : undefined - } - onChangeModel={props.onChangeModel} - getModelRouteLabel={props.getModelRouteLabel} - modelRouteAlternative={props.getModelRouteAlternative?.(message)} - onUseModelRoute={props.onUseModelRoute} - queuedPosition={queuedMeta()?.position} - queuedCount={queuedMeta()?.count} - queuedPaused={queuedMeta()?.paused} - queuedSteering={queuedMeta()?.steering} - onEditQueued={ - queuedMeta() && !queuedMeta()?.steering && props.onEditQueuedFollowUp - ? () => { - const meta = queuedMeta(); - if (meta) props.onEditQueuedFollowUp?.(meta.id); - } - : undefined - } - onCancelQueued={ - queuedMeta() && !queuedMeta()?.steering && props.onCancelQueuedFollowUp - ? () => { - const meta = queuedMeta(); - if (meta) props.onCancelQueuedFollowUp?.(meta.id); - } - : undefined - } - /> + + {(message) => ( + props.onApprove(messageId, approval)} + onSkip={(toolId) => props.onSkip(messageId, toolId)} + onAnswerQuestion={(question, answers) => + props.onAnswerQuestion(messageId, question, answers) + } + onSkipQuestion={(questionId) => props.onSkipQuestion(messageId, questionId)} + onRetry={props.onRetry} + onRegenerate={ + messageId === regenerableMessageId() + ? () => props.onRegenerate?.(messageId) + : undefined + } + onEditPrompt={ + messageId === editablePromptMessageId() + ? () => props.onEditPrompt?.(messageId) + : undefined + } + onChangeModel={props.onChangeModel} + getModelRouteLabel={props.getModelRouteLabel} + modelRouteAlternative={props.getModelRouteAlternative?.(message())} + onUseModelRoute={props.onUseModelRoute} + queuedPosition={queuedMeta()?.position} + queuedCount={queuedMeta()?.count} + queuedPaused={queuedMeta()?.paused} + queuedSteering={queuedMeta()?.steering} + onEditQueued={ + queuedMeta() && !queuedMeta()?.steering && props.onEditQueuedFollowUp + ? () => { + const meta = queuedMeta(); + if (meta) props.onEditQueuedFollowUp?.(meta.id); + } + : undefined + } + onCancelQueued={ + queuedMeta() && !queuedMeta()?.steering && props.onCancelQueuedFollowUp + ? () => { + const meta = queuedMeta(); + if (meta) props.onCancelQueuedFollowUp?.(meta.id); + } + : undefined + } + /> + )} + ); }} diff --git a/frontend-modern/src/components/AI/Chat/__tests__/ChatMessages.test.tsx b/frontend-modern/src/components/AI/Chat/__tests__/ChatMessages.test.tsx index 25d40557f..09518c33c 100644 --- a/frontend-modern/src/components/AI/Chat/__tests__/ChatMessages.test.tsx +++ b/frontend-modern/src/components/AI/Chat/__tests__/ChatMessages.test.tsx @@ -115,6 +115,79 @@ function setScrollMetrics( } describe('ChatMessages', () => { + it('does not mutate shared tool evidence when completed status rows are removed', () => { + const a = { name: 'pulse_query', input: 'client', output: 'client evidence', success: true }; + const b = { name: 'pulse_alerts', input: 'alerts', output: 'alert evidence', success: true }; + const pendingA = { id: 'a', name: a.name, input: a.input }; + const pendingB = { id: 'b', name: b.name, input: b.input }; + const [messages, setMessages] = createSignal([ + makeMessage({ + role: 'assistant', + toolCalls: [], + streamEvents: [ + { + type: 'workflow_status', + workflowStatus: { phase: 'provider_start', message: 'Starting' }, + }, + { type: 'pending_tool', toolId: 'a', pendingTool: pendingA }, + { type: 'pending_tool', toolId: 'b', pendingTool: pendingB }, + ], + pendingTools: [pendingA, pendingB], + }), + ]); + render(() => ); + setMessages([ + makeMessage({ + role: 'assistant', + toolCalls: [a], + streamEvents: [ + { + type: 'workflow_status', + workflowStatus: { phase: 'provider_start', message: 'Starting' }, + }, + { type: 'tool', toolId: 'a', tool: a }, + { type: 'pending_tool', toolId: 'b', pendingTool: pendingB }, + ], + pendingTools: [pendingB], + }), + ]); + setMessages([ + makeMessage({ + role: 'assistant', + toolCalls: [a, b], + streamEvents: [ + { + type: 'workflow_status', + workflowStatus: { phase: 'provider_start', message: 'Starting' }, + }, + { type: 'tool', toolId: 'a', tool: a }, + { type: 'tool', toolId: 'b', tool: b }, + ], + pendingTools: [], + }), + ]); + setMessages([ + makeMessage({ + role: 'assistant', + toolCalls: [a, b], + streamEvents: [ + { type: 'tool', toolId: 'a', tool: a }, + { type: 'tool', toolId: 'b', tool: b }, + ], + pendingTools: [], + }), + ]); + expect(a).toEqual({ + name: 'pulse_query', + input: 'client', + output: 'client evidence', + success: true, + }); + expect( + capturedMessageItemProps[0].message.streamEvents?.map((event) => event.tool?.output), + ).toEqual(['client evidence', 'alert evidence']); + }); + describe('empty transcript', () => { it('keeps the transcript blank when there are no messages or resume actions', () => { const { container } = render(() => ); diff --git a/frontend-modern/src/components/AI/Chat/__tests__/useChat.test.ts b/frontend-modern/src/components/AI/Chat/__tests__/useChat.test.ts index 13e1ec523..74f4cf2fb 100644 --- a/frontend-modern/src/components/AI/Chat/__tests__/useChat.test.ts +++ b/frontend-modern/src/components/AI/Chat/__tests__/useChat.test.ts @@ -2219,6 +2219,96 @@ describe('useChat', () => { dispose(); }); + it('preserves distinct invocation IDs across concurrent same-name tool updates', async () => { + const { getFireEvent } = setupWithEventCapture(); + const { value: chat, dispose } = withRoot(() => useChat({ sessionId: 's' })); + await chat.sendMessage('hi'); + const fire = getFireEvent(); + const inputA = '{"action":"search","query":"client"}'; + const inputB = '{"action":"search","query":"Tower"}'; + const assistant = () => chat.messages().find((m) => m.role === 'assistant')!; + fire({ type: 'tool_start', data: { id: 'a', name: 'pulse_query', input: inputA } }); + fire({ type: 'tool_start', data: { id: 'b', name: 'pulse_query', input: inputB } }); + expect(assistant().pendingTools?.map((tool) => [tool.id, tool.input])).toEqual([ + ['a', inputA], + ['b', inputB], + ]); + fire({ + type: 'tool_progress', + data: { id: 'a', name: 'pulse_query', message: 'Reading client' }, + }); + fire({ type: 'tool_start', data: { id: 'a', name: 'pulse_query', input: inputA } }); + expect(assistant().pendingTools?.map((tool) => tool.id)).toEqual(['a', 'b']); + expect(assistant().pendingTools?.[1].progress).toBeUndefined(); + fire({ + type: 'tool_end', + data: { id: 'a', name: 'pulse_query', output: 'client evidence', success: true }, + }); + expect( + assistant() + .streamEvents?.filter((event) => event.type === 'tool' || event.type === 'pending_tool') + .map((event) => [ + event.type, + event.toolId, + event.tool?.input || event.pendingTool?.input, + ]), + ).toEqual([ + ['tool', 'a', inputA], + ['pending_tool', 'b', inputB], + ]); + fire({ + type: 'tool_end', + data: { id: 'b', name: 'pulse_query', output: 'host unavailable', success: false }, + }); + expect(assistant().pendingTools).toEqual([]); + expect(assistant().toolCalls?.map((tool) => [tool.input, tool.output, tool.success])).toEqual( + [ + [inputA, 'client evidence', true], + [inputB, 'host unavailable', false], + ], + ); + dispose(); + }); + + it('does not cancel or approve a sibling invocation with the same name', async () => { + const { getFireEvent } = setupWithEventCapture(); + const { value: chat, dispose } = withRoot(() => useChat({ sessionId: 's' })); + await chat.sendMessage('hi'); + const fire = getFireEvent(); + const assistant = () => chat.messages().find((m) => m.role === 'assistant')!; + for (const id of ['a', 'b']) { + fire({ + type: 'tool_start', + data: { id, name: 'pulse_control', input: JSON.stringify({ resource_id: id }) }, + }); + fire({ + type: 'approval_needed', + data: { + tool_id: id, + tool_name: 'pulse_control', + approval_id: `approval-${id}`, + command: id, + }, + }); + } + fire({ type: 'tool_end', data: { id: 'a', name: 'pulse_control', output: 'completed a' } }); + expect(assistant().pendingApprovals?.map((approval) => approval.toolId)).toEqual(['b']); + expect( + assistant() + .streamEvents?.filter((event) => event.type === 'approval') + .map((event) => event.approval?.toolId), + ).toEqual(['b']); + fire({ type: 'tool_start', data: { id: 'c', name: 'pulse_control', input: '{}' } }); + fire({ type: 'tool_cancel', data: { id: 'c', name: 'pulse_control', reason: 'Skipped' } }); + expect(assistant().pendingTools?.map((tool) => tool.id)).toEqual(['b']); + expect( + assistant() + .streamEvents?.filter((event) => event.type === 'tool_cancel') + .map((event) => event.toolId), + ).toEqual(['c']); + dispose(); + }); + it('processes tool_start events', async () => { const { getFireEvent } = setupWithEventCapture(); const { value: chat, dispose } = withRoot(() => useChat({ sessionId: 's' })); diff --git a/frontend-modern/src/components/AI/Chat/hooks/useChat.ts b/frontend-modern/src/components/AI/Chat/hooks/useChat.ts index f49d8f4b3..eb96406a2 100644 --- a/frontend-modern/src/components/AI/Chat/hooks/useChat.ts +++ b/frontend-modern/src/components/AI/Chat/hooks/useChat.ts @@ -821,8 +821,8 @@ export function useChat(options: UseChatOptions = {}) { const matchesTool = (tool?: PendingTool, toolId?: string) => { if (!tool) return false; - if (data.id && toolId === data.id) return true; - if (data.id && tool.id === data.id) return true; + // A supplied invocation ID must never fall back to a sibling tool name. + if (data.id) return toolId === data.id || tool.id === data.id; return normalizedName !== '' && normalizeChatToolName(tool.name) === normalizedName; }; @@ -903,8 +903,8 @@ export function useChat(options: UseChatOptions = {}) { const matchesTool = (tool?: PendingTool, toolId?: string) => { if (!tool) return false; - if (data.id && toolId === data.id) return true; - if (data.id && tool.id === data.id) return true; + // A supplied invocation ID must never fall back to a sibling tool name. + if (data.id) return toolId === data.id || tool.id === data.id; return normalizedName !== '' && normalizeChatToolName(tool.name) === normalizedName; }; @@ -1519,8 +1519,8 @@ export function useChat(options: UseChatOptions = {}) { const normalizedName = normalizeChatToolName(data.name || ''); const matchesTool = (tool?: PendingTool, toolId?: string) => { if (!tool) return false; - if (data.id && toolId === data.id) return true; - if (data.id && tool.id === data.id) return true; + // A supplied invocation ID must never fall back to a sibling tool name. + if (data.id) return toolId === data.id || tool.id === data.id; return normalizedName !== '' && normalizeChatToolName(tool.name) === normalizedName; }; const pendingTools = msg.pendingTools || []; @@ -1588,8 +1588,7 @@ export function useChat(options: UseChatOptions = {}) { data.input && data.input.trim() ? data.input : resolvedPendingTool?.input || '{}'; const completedRawInput = data.raw_input ?? resolvedPendingTool?.rawInput; const matchesCompletedTool = (toolId?: string, toolName?: string) => { - if (data.id && toolId === data.id) return true; - if (completedToolId && toolId === completedToolId) return true; + if (completedToolId) return toolId === completedToolId; return ( normalizedEndName !== '' && normalizeChatToolName(toolName || '') === normalizedEndName diff --git a/scripts/check-assistant-identity.mjs b/scripts/check-assistant-identity.mjs new file mode 100644 index 000000000..68bd976b5 --- /dev/null +++ b/scripts/check-assistant-identity.mjs @@ -0,0 +1,232 @@ +// Run with pulse-heavy-run -- node scripts/check-assistant-identity.mjs +import { chromium, expect } from "../node_modules/@playwright/test/index.mjs"; +import { createServer } from "../frontend-modern/node_modules/vite/dist/node/index.js"; +import solid from "../frontend-modern/node_modules/vite-plugin-solid/dist/esm/index.mjs"; +import { resolve } from "node:path"; +import { mkdir, readFile, writeFile, readdir, rm } from "node:fs/promises"; +import { createHash } from "node:crypto"; +const root = resolve(import.meta.dirname, "../frontend-modern"); +const output = resolve( + process.env.ASSISTANT_IDENTITY_OUTPUT || `${root}/../tmp/assistant-identity`, +); +await mkdir(output, { recursive: true }); +await rm(`${output}/receipt.json`, { force: true }); +process.chdir(root); +const server = await createServer({ + configFile: false, + root, + optimizeDeps: { esbuildOptions: { target: "esnext" } }, + plugins: [solid()], + resolve: { alias: { "@": `${root}/src` } }, + server: { host: "127.0.0.1", port: 0 }, +}); +let browser; +const results = []; +try { + await server.listen(); + const base = server.resolvedUrls.local[0]; + browser = await chromium.launch({ headless: true }); + for (const width of [1440, 900, 390]) { + const context = await browser.newContext({ + viewport: { width, height: 1000 }, + serviceWorkers: "block", + }); + const page = await context.newPage(); + const errors = [], + blocked = []; + page.on("pageerror", (e) => errors.push(e.message)); + await context.route("**/*", (route) => { + const url = new URL(route.request().url()); + if ( + url.origin === new URL(base).origin && + !url.pathname.startsWith("/api/") + ) + return route.continue(); + blocked.push(url.pathname); + return route.abort(); + }); + await page.goto(`${base}qualification/assistant-identity/`); + await page.waitForFunction(() => !!window.identityFixture); + const call = (method, arg) => + page.evaluate( + ([method, arg]) => window.identityFixture[method](arg), + [method, arg], + ); + const fire = (type, data) => call("fire", { type, data }); + await call("start"); + for (const id of ["a", "b"]) + await fire("tool_start", { + id, + name: "pulse_query", + input: JSON.stringify({ action: "search", query: `fixture-${id}` }), + }); + expect((await call("snapshot")).pendingTools.map((t) => t.id)).toEqual([ + "a", + "b", + ]); + await fire("tool_progress", { + id: "a", + name: "pulse_query", + message: "Reading A", + }); + await fire("tool_end", { + id: "a", + name: "pulse_query", + output: "identity evidence A", + success: true, + }); + expect((await call("snapshot")).pendingTools.map((t) => t.id)).toEqual([ + "b", + ]); + await fire("tool_end", { + id: "b", + name: "pulse_query", + output: "identity evidence B", + success: false, + }); + expect((await call("snapshot")).toolCalls.map((t) => t.output)).toEqual([ + "identity evidence A", + "identity evidence B", + ]); + for (const id of ["c", "d"]) { + await fire("tool_start", { + id, + name: "pulse_control", + input: JSON.stringify({ resource_id: id }), + }); + await fire("approval_needed", { + tool_id: id, + tool_name: "pulse_control", + approval_id: `approval-${id}`, + command: `synthetic-${id}`, + }); + } + await expect( + page.getByText("Approval Required", { exact: true }), + ).toHaveCount(2); + await fire("tool_end", { + id: "c", + name: "pulse_control", + output: "synthetic c completed", + success: true, + }); + await expect( + page.getByText("Approval Required", { exact: true }), + ).toHaveCount(1); + expect( + (await call("snapshot")).pendingApprovals.map((a) => a.toolId), + ).toEqual(["d"]); + await expect(page.getByText("synthetic-d", { exact: true })).toBeVisible(); + await page.screenshot({ + path: `${output}/approvals-${width}.png`, + fullPage: true, + }); + await fire("done", {}); + while (await page.locator('[aria-expanded="false"]').count()) + await page.locator('[aria-expanded="false"]').first().click(); + await expect( + page.getByText("identity evidence A", { exact: true }).last(), + ).toBeVisible(); + await expect( + page.getByText("identity evidence B", { exact: true }).last(), + ).toBeVisible(); + for (let stage = 0; stage < 4; stage++) { + await call("shared", stage); + // Allow Solid's render effects to run between immutable snapshots. + await page.evaluate(() => new Promise(requestAnimationFrame)); + } + expect(await call("evidence")).toEqual([ + { + name: "pulse_query", + input: "client", + output: "client evidence", + success: true, + }, + { + name: "pulse_alerts", + input: "alerts", + output: "alert evidence", + success: true, + }, + ]); + const expand = page.locator('[role="button"][aria-expanded="false"]'); + while (await expand.count()) await expand.first().click(); + await expect( + page.getByText("client evidence", { exact: true }).last(), + ).toBeVisible(); + await expect( + page.getByText("alert evidence", { exact: true }).last(), + ).toBeVisible(); + await expect( + page.getByText("client", { exact: true }).last(), + ).toBeVisible(); + await expect( + page.getByText("alerts", { exact: true }).last(), + ).toBeVisible(); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= innerWidth, + ), + ).toBe(true); + await page.screenshot({ + path: `${output}/expanded-${width}.png`, + fullPage: true, + }); + expect(errors).toEqual([]); + expect(blocked).toEqual([]); + results.push({ width, passed: true }); + await context.close(); + } + const hashes = {}; + for (const file of await readdir(output)) { + if (file.endsWith(".png")) + hashes[`artifacts/${file}`] = createHash("sha256") + .update(await readFile(`${output}/${file}`)) + .digest("hex"); + } + hashes["../package-lock.json"] = createHash("sha256") + .update(await readFile(`${root}/../package-lock.json`)) + .digest("hex"); + async function hashTree(dir) { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = `${dir}/${entry.name}`; + if (entry.isDirectory()) await hashTree(path); + else + hashes[path.slice(root.length + 1)] = createHash("sha256") + .update(await readFile(path)) + .digest("hex"); + } + } + await hashTree(`${root}/src`); + await hashTree(`${root}/qualification/assistant-identity`); + for (const path of [ + "package-lock.json", + "tailwind.config.js", + "postcss.config.js", + ]) + hashes[path] = createHash("sha256") + .update(await readFile(`${root}/${path}`)) + .digest("hex"); + hashes["../scripts/check-assistant-identity.mjs"] = createHash("sha256") + .update(await readFile(import.meta.filename)) + .digest("hex"); + await writeFile( + `${output}/receipt.json`, + JSON.stringify( + { + recordedAt: new Date().toISOString(), + scope: + "Synthetic reducer and renderer only; no provider, backend or real actions qualified", + browser: browser.version(), + results, + hashes, + }, + null, + 2, + ), + ); + console.log(JSON.stringify({ output, results })); +} finally { + await browser?.close(); + await server.close(); +}