diff --git a/frontend-modern/src/api/__tests__/aiChat.test.ts b/frontend-modern/src/api/__tests__/aiChat.test.ts index 01381f393..f9d98ed4e 100644 --- a/frontend-modern/src/api/__tests__/aiChat.test.ts +++ b/frontend-modern/src/api/__tests__/aiChat.test.ts @@ -247,6 +247,55 @@ describe('AIChatAPI', () => { }); }); + it('runs the reasoning-leak dev stream fixture without opening a provider request', async () => { + const onEvent = vi.fn(); + + await AIChatAPI.chat( + '/fixture reasoning-leak', + undefined, + 'openrouter:deepseek/deepseek-chat', + onEvent, + ); + + expect(apiFetchMock).not.toHaveBeenCalled(); + expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual([ + 'session', + 'workflow_state', + 'content', + 'content', + 'tool_start', + 'tool_end', + 'content', + 'done', + ]); + expect(onEvent.mock.calls[2][0]).toMatchObject({ + type: 'content', + data: { + text: expect.stringContaining('Thinking'), + }, + }); + expect(onEvent.mock.calls[3][0]).toMatchObject({ + type: 'content', + data: { + text: expect.stringContaining('pulse_read'), + }, + }); + expect(onEvent.mock.calls[4][0]).toMatchObject({ + type: 'tool_start', + data: { + id: 'fixture-tool-reasoning-leak', + name: 'pulse_read', + input: expect.stringContaining('ls /dev | wc -l'), + }, + }); + expect(onEvent.mock.calls[6][0]).toMatchObject({ + type: 'content', + data: { + text: 'There are 4,358 entries under `/dev`.', + }, + }); + }); + it('runs the workflow-burst dev stream fixture without opening a provider request', async () => { const onEvent = vi.fn(); @@ -557,6 +606,7 @@ describe('AIChatAPI', () => { expect(AI_CHAT_DEV_STREAM_FIXTURE_NAMES).toContain('provider-retry'); expect(AI_CHAT_DEV_STREAM_FIXTURE_NAMES).toContain('send-hold'); expect(AI_CHAT_DEV_STREAM_FIXTURE_NAMES).toContain('command-tool'); + expect(AI_CHAT_DEV_STREAM_FIXTURE_NAMES).toContain('reasoning-leak'); expect(AI_CHAT_DEV_STREAM_FIXTURE_NAMES).not.toContain('/fixture provider-retry'); expect(AI_CHAT_DEV_STREAM_FIXTURE_ALIAS_NAMES).toEqual(['burst-tool', 'queued-follow-up']); }); diff --git a/frontend-modern/src/api/aiChatDevStreamFixture.ts b/frontend-modern/src/api/aiChatDevStreamFixture.ts index 18acebd4e..188088776 100644 --- a/frontend-modern/src/api/aiChatDevStreamFixture.ts +++ b/frontend-modern/src/api/aiChatDevStreamFixture.ts @@ -5,6 +5,7 @@ export const AI_CHAT_DEV_STREAM_FIXTURE_PROMPTS = [ '/fixture assistant-stream', '/fixture send-hold', '/fixture tool-burst', + '/fixture reasoning-leak', '/fixture workflow-burst', '/fixture context-group', '/fixture status-boundary', @@ -372,6 +373,67 @@ const buildToolBurstFixtureEvents = (model?: string): AIChatStreamEvent[] => [ }, ]; +const buildReasoningLeakFixtureEvents = (model?: string): AIChatStreamEvent[] => [ + { + type: 'session', + data: { id: 'dev-fixture-reasoning-leak' }, + }, + { + type: 'workflow_state', + data: { + phase: 'request_start', + message: 'Preparing Pulse context.', + }, + }, + { + type: 'content', + data: { + text: 'Thinking\nWe need to inspect the prompt and count device nodes before answering.', + }, + }, + { + type: 'content', + data: { + text: '\npulse_read(target_host="current_resource", command="ls /dev | wc -l")', + }, + }, + { + type: 'tool_start', + data: { + id: 'fixture-tool-reasoning-leak', + name: 'pulse_read', + input: '{"action":"exec","target_host":"current_resource","command":"ls /dev | wc -l"}', + raw_input: 'pulse_read(target_host="current_resource", command="ls /dev | wc -l")', + }, + }, + { + type: 'tool_end', + data: { + id: 'fixture-tool-reasoning-leak', + name: 'pulse_read', + input: '{"action":"exec","target_host":"current_resource","command":"ls /dev | wc -l"}', + raw_input: 'pulse_read(target_host="current_resource", command="ls /dev | wc -l")', + output: '4358', + success: true, + }, + }, + { + type: 'content', + data: { + text: 'There are 4,358 entries under `/dev`.', + }, + }, + { + type: 'done', + data: { + session_id: 'dev-fixture-reasoning-leak', + model: assistantFixtureModel(model), + input_tokens: 88, + output_tokens: 24, + }, + }, +]; + const buildWorkflowBurstFixtureEvents = (model?: string): AIChatStreamEvent[] => [ { type: 'session', @@ -1009,6 +1071,9 @@ const buildFixtureEvents = (prompt: string, model?: string): AIChatStreamEvent[] if (normalized === '/fixture tool-burst') { return buildToolBurstFixtureEvents(model); } + if (normalized === '/fixture reasoning-leak') { + return buildReasoningLeakFixtureEvents(model); + } if (normalized === '/fixture workflow-burst') { return buildWorkflowBurstFixtureEvents(model); } diff --git a/frontend-modern/src/components/AI/Chat/__tests__/assistantOutputHygiene.test.ts b/frontend-modern/src/components/AI/Chat/__tests__/assistantOutputHygiene.test.ts index 2690b149d..135fc6d39 100644 --- a/frontend-modern/src/components/AI/Chat/__tests__/assistantOutputHygiene.test.ts +++ b/frontend-modern/src/components/AI/Chat/__tests__/assistantOutputHygiene.test.ts @@ -45,6 +45,17 @@ describe('stripAssistantOutputArtifacts', () => { expect(result.stripped).toBe(true); }); + it('suppresses content-channel reasoning preludes before raw function-call leaks', () => { + const result = stripAssistantOutputArtifacts( + 'Thinking\nWe need to interpret the user question and count the device nodes.\npulse_read(target_host="current_resource", command="ls /dev | wc -l")', + ); + + expect(result).toEqual({ + text: '', + stripped: true, + }); + }); + it('leaves ordinary prose and unrelated function calls alone', () => { expect(stripAssistantOutputArtifacts('Call helper(target="x") in the example.')).toEqual({ text: 'Call helper(target="x") in the example.', @@ -115,6 +126,59 @@ describe('stripAssistantOutputArtifacts', () => { expect(flushPendingAssistantOutputText(state)).toBe(''); }); + it('holds content-channel reasoning preludes so they never flash before tool leaks', () => { + const state = createAssistantOutputArtifactStreamState(); + + expect( + appendVisibleTextBeforeAssistantOutputArtifacts( + state, + 'Thinking\nWe need to count device nodes before answering.', + ), + ).toEqual({ + text: '', + stripped: false, + }); + expect( + appendVisibleTextBeforeAssistantOutputArtifacts( + state, + '\npulse_read(target_host="current_resource", command="ls /dev | wc -l")', + ), + ).toEqual({ + text: '', + stripped: true, + }); + expect(flushPendingAssistantOutputText(state)).toBe(''); + }); + + it('drops held content-channel reasoning if no answer text follows before stream end', () => { + const state = createAssistantOutputArtifactStreamState(); + + expect( + appendVisibleTextBeforeAssistantOutputArtifacts( + state, + 'Thinking\nWe need to inspect the prompt before answering.', + ), + ).toEqual({ + text: '', + stripped: false, + }); + expect(flushPendingAssistantOutputText(state)).toBe(''); + }); + + it('releases normal prose that only starts with a similar word', () => { + const state = createAssistantOutputArtifactStreamState(); + + expect(appendVisibleTextBeforeAssistantOutputArtifacts(state, 'Think')).toEqual({ + text: '', + stripped: false, + }); + expect(appendVisibleTextBeforeAssistantOutputArtifacts(state, ' about the result this way.')).toEqual({ + text: 'Think about the result this way.', + stripped: false, + }); + expect(flushPendingAssistantOutputText(state)).toBe(''); + }); + it('releases a held prefix when the next delta proves it is normal prose', () => { const state = createAssistantOutputArtifactStreamState(); diff --git a/frontend-modern/src/components/AI/Chat/__tests__/useChat.test.ts b/frontend-modern/src/components/AI/Chat/__tests__/useChat.test.ts index a3c2a7bdf..700c86337 100644 --- a/frontend-modern/src/components/AI/Chat/__tests__/useChat.test.ts +++ b/frontend-modern/src/components/AI/Chat/__tests__/useChat.test.ts @@ -1109,6 +1109,30 @@ describe('useChat', () => { dispose(); }); + it('keeps content-channel reasoning and raw tool-call leaks out of assistant content', async () => { + const { getFireEvent } = setupWithEventCapture(); + const { value: chat, dispose } = withRoot(() => useChat({ sessionId: 's' })); + + await chat.sendMessage('how many devices?'); + const fire = getFireEvent(); + + fire({ + type: 'content', + data: 'Thinking\nWe need to inspect the prompt and count device nodes first.', + }); + fire({ + type: 'content', + data: '\npulse_read(target_host="current_resource", command="ls /dev | wc -l")', + }); + + const assistant = chat.messages().find((m) => m.role === 'assistant')!; + expect(assistant.content).toBe(''); + expect(assistant.streamEvents?.filter((event) => event.type === 'content') ?? []).toEqual( + [], + ); + dispose(); + }); + it('processes thinking events — merges consecutive thinking', async () => { const { getFireEvent } = setupWithEventCapture(); const { value: chat, dispose } = withRoot(() => useChat({ sessionId: 's' })); diff --git a/frontend-modern/src/components/AI/Chat/assistantOutputHygiene.ts b/frontend-modern/src/components/AI/Chat/assistantOutputHygiene.ts index 4c9bc7ad2..bf042c6df 100644 --- a/frontend-modern/src/components/AI/Chat/assistantOutputHygiene.ts +++ b/frontend-modern/src/components/AI/Chat/assistantOutputHygiene.ts @@ -30,6 +30,10 @@ const functionToolCallLeakRe = /(?:^|[^a-zA-Z0-9_])([a-zA-Z_][a-zA-Z0-9_]*)[ \t\ const minimaxToolCallLeakRe = /^minimax:tool_call\b/m; const visibleInternalToolIdentifierRe = /`?\b((?:pulse|patrol)_[a-zA-Z0-9_]+|run_command)\b`?(?:[ \t]+(tool|command|query|call))?/g; +const internalReasoningHeadingRe = + /^\s*(?:#{1,6}[ \t]*)?(thinking|thoughts?|reasoning|analysis)[ \t]*:?[ \t]*(?:\r?\n|$)/i; +const internalReasoningCueRe = + /\b(?:we need to|i need to|let me|let's|need to|the user (?:asks|asked|is asking)|user question|before answering|before responding|tool call|call the|use the|inspect the prompt)\b/i; const VISIBLE_INTERNAL_TOOL_IDENTIFIER_LABELS: Record = { patrol_collect: 'collection', @@ -65,7 +69,7 @@ export function stripAssistantOutputArtifacts(content: string): { } const visiblePrefix = content.slice(0, idx).trim(); return { - text: isCompactedToolPrelude(visiblePrefix) + text: shouldSuppressBeforeAssistantOutputArtifact(visiblePrefix) ? '' : normalizeAssistantVisibleInternalIdentifiers(visiblePrefix), stripped: true, @@ -93,7 +97,7 @@ export function appendVisibleTextBeforeAssistantOutputArtifacts( const candidate = existingRawText + text; const idx = assistantOutputArtifactIndex(candidate); if (idx < 0) { - if (!existingRawText && isCompactedToolPrelude(text)) { + if (!existingRawText && shouldHoldPotentialAssistantOutputPrelude(text)) { state.pendingText = text; return { text: '', stripped: false }; } @@ -107,7 +111,7 @@ export function appendVisibleTextBeforeAssistantOutputArtifacts( } const safeText = candidate.slice(0, idx).trimEnd(); - if (isCompactedToolPrelude(safeText)) { + if (shouldSuppressBeforeAssistantOutputArtifact(safeText)) { const previousVisibleText = state.visibleText; state.visibleText = ''; state.rawVisibleText = ''; @@ -142,7 +146,7 @@ export function flushPendingAssistantOutputText(state: AssistantOutputArtifactSt } const text = state.pendingText; state.pendingText = ''; - if (isCompactedToolPrelude(text)) { + if (shouldSuppressBeforeAssistantOutputArtifact(text)) { return ''; } const normalizedText = normalizeAssistantVisibleInternalIdentifiers(text); @@ -273,3 +277,36 @@ function isCompactedToolPrelude(content: string): boolean { if (whitespace === 0) return true; return letters >= 48 && whitespace <= 1; } + +function shouldSuppressBeforeAssistantOutputArtifact(content: string): boolean { + return isCompactedToolPrelude(content) || isContentChannelReasoningPrelude(content); +} + +function shouldHoldPotentialAssistantOutputPrelude(content: string): boolean { + return isCompactedToolPrelude(content) || isPotentialContentChannelReasoningPrelude(content); +} + +function isContentChannelReasoningPrelude(content: string): boolean { + const trimmed = content.trim(); + if (!trimmed) return false; + + const match = trimmed.match(internalReasoningHeadingRe); + if (!match) return false; + + const body = trimmed.slice(match[0].length).trim(); + if (!body) return true; + if (internalReasoningCueRe.test(body)) return true; + + const words = body.split(/\s+/).filter(Boolean); + return words.length >= 8; +} + +function isPotentialContentChannelReasoningPrelude(content: string): boolean { + const trimmed = content.trimStart(); + if (!trimmed) return false; + + const lower = trimmed.toLowerCase(); + if (lower.length < 'thinking'.length && 'thinking'.startsWith(lower)) return true; + + return internalReasoningHeadingRe.test(trimmed); +}