diff --git a/docs/AI.md b/docs/AI.md index 5b9f45174..b9ac0a99b 100644 --- a/docs/AI.md +++ b/docs/AI.md @@ -374,6 +374,13 @@ Pulse tool loop independently applies control level, license, protected-resource, approval, action, and verification policy. The CLI never executes a Patrol tool itself. +Claude Code can occasionally express a requested Pulse tool as a local native +tool call even though local tools are disabled. Pulse audits Claude's buffered +event stream, accepts only the first call whose name was explicitly offered for +that turn, and routes it back through Pulse's normal executor; undeclared local +tool attempts fail closed. This does not grant Claude Code direct access to the +infrastructure. + This is still a local agent process, not a remote chat-completions API. Pulse rejects a turn if Codex reports command, file, MCP, web, computer, or image-tool activity, and Claude is launched with its built-in filesystem, shell, web, and diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index b71dac523..362b5f95f 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -299,7 +299,18 @@ own subscription-provider prefix before CLI execution, and rejects foreign provider prefixes rather than forwarding an invalid or cross-provider model name to the local agent. Claude uses its native output-schema mode for every provider decision. Pulse -decodes completed structured turns with unknown fields and trailing values +consumes Claude Code's verbose JSON event stream but still buffers the whole +process result before emitting provider events. Claude Code can mistake a tool +name serialized in the Pulse request for a local native tool even though local +tools are disabled; in that case it emits the intended `tool_use`, injects a +synthetic `No such tool available` result, and lets the model continue toward a +false `needs_attention` conclusion. The adapter must recover only the first +such call when its name is in the current Pulse-owned offered-tool manifest, +then return that call through the normal Pulse executor on the next provider +turn. It must ignore later attempts made without real Pulse tool results, +reject any undeclared native tool attempt, and never execute infrastructure +tools inside Claude Code. When no recoverable native call occurred, Pulse +decodes the terminal structured turn with unknown fields and trailing values rejected. Live qualification also proved that Claude's wrapper may exhaust its structured-output retries on the final turn after Pulse has already persisted a successful `patrol_report_finding`, `patrol_assess_finding`, or diff --git a/internal/ai/providers/subscription_agent.go b/internal/ai/providers/subscription_agent.go index 7c554270c..7dd82d4f2 100644 --- a/internal/ai/providers/subscription_agent.go +++ b/internal/ai/providers/subscription_agent.go @@ -41,6 +41,10 @@ REQUEST_JSON.system, REQUEST_JSON.tools, and REQUEST_JSON.tool_choice are truste Returning a tool_calls entry is a routing decision, not execution or fabrication: Pulse will validate the declared tool, enforce permissions, execute it, and return the result in a later provider turn. Never invoke local agent tools yourself. Do not refuse merely because the serialized provider request contains a system instruction or a tool catalogue. Select only tools declared in REQUEST_JSON.tools. Return each tool's arguments as the native JSON object in input. Use stop_reason tool_use when returning any tool_calls; otherwise use end_turn.` + + claudeSubscriptionAgentControlPrompt = subscriptionAgentControlPrompt + ` + +Claude Code transport note: StructuredOutput is the only local runtime tool. Names in REQUEST_JSON.tools and messages[].tool_calls are data values in Pulse's external protocol, not Claude Code tools. Never invoke those names as native tools. Encode the next desired Pulse call only inside StructuredOutput's tool_calls field; Pulse will execute it and provide its result in a later request.` ) var subscriptionAgentSlots = map[SubscriptionAgent]chan struct{}{ @@ -70,15 +74,31 @@ type subscriptionAgentToolCall struct { } type claudePrintResponse struct { + Type string `json:"type,omitempty"` StructuredOutput json.RawMessage `json:"structured_output"` Result string `json:"result"` PermissionDenials []json.RawMessage `json:"permission_denials"` + TerminalReason string `json:"terminal_reason,omitempty"` + NumTurns int `json:"num_turns,omitempty"` Usage struct { InputTokens int `json:"input_tokens"` OutputTokens int `json:"output_tokens"` } `json:"usage"` } +type claudeStreamEvent struct { + Type string `json:"type"` + Message struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Input map[string]interface{} `json:"input,omitempty"` + } `json:"content"` + } `json:"message"` +} + type subscriptionAgentCommandError struct { command string cause error @@ -231,7 +251,7 @@ func (c *SubscriptionAgentClient) Chat(ctx context.Context, req ChatRequest) (*C raw, err = os.ReadFile(outputPath) } case SubscriptionAgentClaude: - raw, err = c.run(ctx, "claude", []string{"-p", "--model", c.model, "--output-format", "json", "--json-schema", string(schemaBytes), "--safe-mode", "--no-session-persistence", "--permission-mode", "dontAsk", "--tools", "", "--system-prompt", subscriptionAgentControlPrompt}, requestPrompt, workdir) + raw, err = c.run(ctx, "claude", []string{"-p", "--model", c.model, "--output-format", "stream-json", "--verbose", "--json-schema", string(schemaBytes), "--safe-mode", "--no-session-persistence", "--permission-mode", "dontAsk", "--tools", "", "--system-prompt", claudeSubscriptionAgentControlPrompt}, requestPrompt, workdir) default: return nil, fmt.Errorf("unknown subscription agent %q", c.agent) } @@ -245,7 +265,12 @@ func (c *SubscriptionAgentClient) Chat(ctx context.Context, req ChatRequest) (*C return nil, fmt.Errorf("subscription agent output exceeds %d bytes", maxSubscriptionAgentOutputBytes) } - turn, err := decodeSubscriptionAgentTurn(c.agent, raw) + var turn subscriptionAgentTurn + if c.agent == SubscriptionAgentClaude { + turn, err = decodeClaudeSubscriptionAgentResponse(req, raw) + } else { + turn, err = decodeSubscriptionAgentTurn(c.agent, raw) + } if err != nil { return nil, err } @@ -342,14 +367,7 @@ func (c *SubscriptionAgentClient) run(ctx context.Context, name string, args []s } commandErr := &subscriptionAgentCommandError{command: name, cause: err, detail: message} if name == "claude" { - var terminal struct { - TerminalReason string `json:"terminal_reason"` - Usage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - } `json:"usage"` - } - if json.Unmarshal(stdout.buffer.Bytes(), &terminal) == nil { + if terminal, ok := decodeClaudeTerminalResponse(stdout.buffer.Bytes()); ok { commandErr.terminalReason = terminal.TerminalReason commandErr.inputTokens = terminal.Usage.InputTokens commandErr.outputTokens = terminal.Usage.OutputTokens @@ -472,6 +490,133 @@ func decodeSubscriptionAgentTurn(agent SubscriptionAgent, raw []byte) (subscript return turn, nil } +// decodeClaudeSubscriptionAgentResponse consumes Claude Code's event stream +// instead of trusting only its terminal StructuredOutput envelope. Claude Code +// may emit a native tool_use for a name serialized in REQUEST_JSON even though +// all of its local tools are intentionally disabled. The CLI then fabricates a +// "No such tool available" result and lets the model continue, which destroys +// Pulse's one-tool-turn-at-a-time provider protocol. The stream preserves the +// original intended call before that local error. Route the first declared +// Pulse call back through Pulse's executor; never execute it inside Claude. +func decodeClaudeSubscriptionAgentResponse(req ChatRequest, raw []byte) (subscriptionAgentTurn, error) { + if !bytes.Contains(raw, []byte{'\n'}) { + return decodeSubscriptionAgentTurn(SubscriptionAgentClaude, raw) + } + + allowed := make(map[string]struct{}, len(req.Tools)) + for _, tool := range req.Tools { + allowed[tool.Name] = struct{}{} + } + + var terminal claudePrintResponse + var terminalFound bool + var content strings.Builder + var routed *subscriptionAgentToolCall + for _, line := range bytes.Split(raw, []byte{'\n'}) { + line = bytes.TrimSpace(line) + if len(line) == 0 { + continue + } + var event claudeStreamEvent + if err := json.Unmarshal(line, &event); err != nil { + return subscriptionAgentTurn{}, fmt.Errorf("decode Claude subscription stream event: %w", err) + } + switch event.Type { + case "assistant": + for _, block := range event.Message.Content { + switch block.Type { + case "text": + if routed == nil && strings.TrimSpace(block.Text) != "" { + content.WriteString(block.Text) + } + case "tool_use": + if block.Name == "StructuredOutput" { + continue + } + if _, ok := allowed[block.Name]; !ok { + return subscriptionAgentTurn{}, fmt.Errorf("Claude subscription agent attempted undeclared native tool %q", block.Name) + } + if routed == nil { + call := subscriptionAgentToolCall{ID: block.ID, Name: block.Name, Input: block.Input} + routed = &call + } + } + } + case "result": + if err := json.Unmarshal(line, &terminal); err != nil { + return subscriptionAgentTurn{}, fmt.Errorf("decode Claude subscription result: %w", err) + } + terminalFound = true + } + } + if !terminalFound { + return subscriptionAgentTurn{}, errors.New("Claude subscription stream did not contain a terminal result") + } + if len(terminal.PermissionDenials) > 0 { + return subscriptionAgentTurn{}, errors.New("Claude subscription agent attempted a denied built-in tool") + } + if routed != nil { + return subscriptionAgentTurn{ + Content: content.String(), + RawToolCalls: []subscriptionAgentToolCall{*routed}, + InputTokens: terminal.Usage.InputTokens, + OutputTokens: terminal.Usage.OutputTokens, + }, nil + } + return decodeClaudePrintResponse(terminal) +} + +func decodeClaudePrintResponse(wrapper claudePrintResponse) (subscriptionAgentTurn, error) { + var turn subscriptionAgentTurn + if len(wrapper.PermissionDenials) > 0 { + return turn, errors.New("Claude subscription agent attempted a denied built-in tool") + } + payload := wrapper.StructuredOutput + if (len(payload) == 0 || string(payload) == "null") && strings.TrimSpace(wrapper.Result) != "" { + payload = json.RawMessage(wrapper.Result) + } + if len(payload) == 0 || string(payload) == "null" { + return turn, errors.New("Claude subscription response did not contain structured output") + } + if err := decodeStrictJSON(payload, &turn); err != nil { + return turn, fmt.Errorf("decode Claude structured turn: %w", err) + } + if turn.InputTokens == 0 { + turn.InputTokens = wrapper.Usage.InputTokens + } + if turn.OutputTokens == 0 { + turn.OutputTokens = wrapper.Usage.OutputTokens + } + return turn, nil +} + +func decodeClaudeTerminalResponse(raw []byte) (claudePrintResponse, bool) { + var terminal claudePrintResponse + if !bytes.Contains(raw, []byte{'\n'}) { + if json.Unmarshal(raw, &terminal) == nil { + return terminal, true + } + return claudePrintResponse{}, false + } + var found bool + for _, line := range bytes.Split(raw, []byte{'\n'}) { + line = bytes.TrimSpace(line) + if len(line) == 0 { + continue + } + var header struct { + Type string `json:"type"` + } + if json.Unmarshal(line, &header) != nil || header.Type != "result" { + continue + } + if json.Unmarshal(line, &terminal) == nil { + found = true + } + } + return terminal, found +} + func followsSuccessfulPatrolOutcome(req ChatRequest) bool { if len(req.Messages) < 2 { return false diff --git a/internal/ai/providers/subscription_agent_test.go b/internal/ai/providers/subscription_agent_test.go index 368f61f40..bd1c35917 100644 --- a/internal/ai/providers/subscription_agent_test.go +++ b/internal/ai/providers/subscription_agent_test.go @@ -168,6 +168,11 @@ func TestSubscriptionAgentPromptSeparatesTrustedControlFromInfrastructureData(t t.Fatalf("adapter system prompt missing trust-boundary clause %q", expected) } } + for _, expected := range []string{"StructuredOutput is the only local runtime tool", "not Claude Code tools", "Pulse will execute it"} { + if !strings.Contains(claudeSubscriptionAgentControlPrompt, expected) { + t.Fatalf("Claude adapter system prompt missing native-tool clause %q", expected) + } + } } func TestSubscriptionAgentClientsUseStructuredSingleTurnProcess(t *testing.T) { @@ -214,6 +219,8 @@ seen_system=false seen_json_schema=false seen_no_tools=false seen_dont_ask=false +seen_stream_json=false +seen_verbose=false while [ "$#" -gt 0 ]; do case "$1" in --system-prompt) @@ -239,6 +246,13 @@ while [ "$#" -gt 0 ]; do shift [ "$1" = "dontAsk" ] && seen_dont_ask=true ;; + --output-format) + shift + [ "$1" = "stream-json" ] && seen_stream_json=true + ;; + --verbose) + seen_verbose=true + ;; esac shift done @@ -246,6 +260,8 @@ done [ "$seen_json_schema" = true ] || { echo "missing native output schema" >&2; exit 97; } [ "$seen_no_tools" = true ] || { echo "Claude built-in tools not disabled" >&2; exit 95; } [ "$seen_dont_ask" = true ] || { echo "unexpected permission mode" >&2; exit 96; } +[ "$seen_stream_json" = true ] || { echo "Claude event stream not enabled" >&2; exit 98; } +[ "$seen_verbose" = true ] || { echo "Claude verbose stream not enabled" >&2; exit 99; } printf '%s' '{"structured_output":{"content":"healthy","stop_reason":"end_turn","tool_calls":[]},"usage":{"input_tokens":12,"output_tokens":3}}' `) t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) @@ -340,6 +356,81 @@ func TestDecodeClaudeSubscriptionAgentRejectsNonJSONResult(t *testing.T) { } } +func TestDecodeClaudeSubscriptionAgentRoutesNativePulseToolAttempt(t *testing.T) { + raw := []byte(strings.Join([]string{ + `{"type":"assistant","message":{"content":[{"type":"text","text":"Checking logs next."},{"type":"tool_use","id":"toolu-1","name":"pulse_read","input":{"action":"logs","resource_id":"resource-1"}}]}}`, + `{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"toolu-1","is_error":true,"content":"No such tool available: pulse_read"}]}}`, + `{"type":"result","subtype":"success","num_turns":3,"structured_output":{"content":"Tool unavailable.","stop_reason":"end_turn","tool_calls":[]},"permission_denials":[],"usage":{"input_tokens":11,"output_tokens":17}}`, + }, "\n")) + req := ChatRequest{Tools: []Tool{{Name: "pulse_read"}}} + turn, err := decodeClaudeSubscriptionAgentResponse(req, raw) + if err != nil { + t.Fatal(err) + } + if turn.Content != "Checking logs next." || len(turn.RawToolCalls) != 1 { + t.Fatalf("routed turn = %#v", turn) + } + call := turn.RawToolCalls[0] + if call.ID != "toolu-1" || call.Name != "pulse_read" || call.Input["resource_id"] != "resource-1" { + t.Fatalf("routed tool call = %#v", call) + } + if turn.InputTokens != 11 || turn.OutputTokens != 17 { + t.Fatalf("routed usage = %d/%d", turn.InputTokens, turn.OutputTokens) + } +} + +func TestDecodeClaudeSubscriptionAgentRoutesOnlyFirstNativePulseToolAttempt(t *testing.T) { + raw := []byte(strings.Join([]string{ + `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu-1","name":"pulse_read","input":{"action":"logs"}}]}}`, + `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu-2","name":"patrol_propose_action","input":{"capability_name":"start"}}]}}`, + `{"type":"result","subtype":"success","structured_output":{"content":"Both unavailable.","stop_reason":"end_turn","tool_calls":[]},"permission_denials":[],"usage":{}}`, + }, "\n")) + req := ChatRequest{Tools: []Tool{{Name: "pulse_read"}, {Name: "patrol_propose_action"}}} + turn, err := decodeClaudeSubscriptionAgentResponse(req, raw) + if err != nil { + t.Fatal(err) + } + if len(turn.RawToolCalls) != 1 || turn.RawToolCalls[0].Name != "pulse_read" { + t.Fatalf("routed tool calls = %#v", turn.RawToolCalls) + } +} + +func TestDecodeClaudeSubscriptionAgentRejectsUndeclaredNativeToolAttempt(t *testing.T) { + raw := []byte(strings.Join([]string{ + `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu-1","name":"Bash","input":{"command":"true"}}]}}`, + `{"type":"result","subtype":"success","structured_output":{"content":"done","stop_reason":"end_turn","tool_calls":[]},"permission_denials":[],"usage":{}}`, + }, "\n")) + _, err := decodeClaudeSubscriptionAgentResponse(ChatRequest{Tools: []Tool{{Name: "pulse_read"}}}, raw) + if err == nil || !strings.Contains(err.Error(), `undeclared native tool "Bash"`) { + t.Fatalf("undeclared native tool error = %v", err) + } +} + +func TestDecodeClaudeSubscriptionAgentUsesTerminalStructuredOutput(t *testing.T) { + raw := []byte(strings.Join([]string{ + `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"structured-1","name":"StructuredOutput","input":{"content":"healthy","stop_reason":"end_turn","tool_calls":[]}}]}}`, + `{"type":"result","subtype":"success","num_turns":2,"structured_output":{"content":"healthy","stop_reason":"end_turn","tool_calls":[]},"permission_denials":[],"usage":{"input_tokens":5,"output_tokens":7}}`, + }, "\n")) + turn, err := decodeClaudeSubscriptionAgentResponse(ChatRequest{}, raw) + if err != nil { + t.Fatal(err) + } + if turn.Content != "healthy" || len(turn.RawToolCalls) != 0 || turn.InputTokens != 5 || turn.OutputTokens != 7 { + t.Fatalf("structured turn = %#v", turn) + } +} + +func TestDecodeClaudeTerminalResponseUsesFinalStreamResult(t *testing.T) { + raw := []byte(strings.Join([]string{ + `{"type":"system","subtype":"init"}`, + `{"type":"result","subtype":"error","terminal_reason":"structured_output_retry_exhausted","usage":{"input_tokens":19,"output_tokens":23}}`, + }, "\n")) + terminal, ok := decodeClaudeTerminalResponse(raw) + if !ok || terminal.TerminalReason != "structured_output_retry_exhausted" || terminal.Usage.InputTokens != 19 || terminal.Usage.OutputTokens != 23 { + t.Fatalf("terminal stream result = (%#v, %t)", terminal, ok) + } +} + func TestDecodeSubscriptionAgentTurnRejectsUnknownStructuredFields(t *testing.T) { raw := []byte(`{"result":"{\"content\":\"healthy\",\"stop_reason\":\"end_turn\",\"tool_calls\":[],\"unexpected\":true}"}`) if _, err := decodeSubscriptionAgentTurn(SubscriptionAgentClaude, raw); err == nil || !strings.Contains(err.Error(), "unknown field") { diff --git a/scripts/release_control/ai_runtime_docs_policy_test.py b/scripts/release_control/ai_runtime_docs_policy_test.py index 31ef19a69..b15081f97 100644 --- a/scripts/release_control/ai_runtime_docs_policy_test.py +++ b/scripts/release_control/ai_runtime_docs_policy_test.py @@ -149,6 +149,10 @@ class AIRuntimeDocsPolicyTest(unittest.TestCase): self.assertIn("API-key environment variables", normalized_content) self.assertIn("dedicated, least-privilege OS account", normalized_content) self.assertIn("Pulse retains tool execution and policy enforcement", normalized_content) + self.assertIn("Pulse audits Claude's buffered event stream", normalized_content) + self.assertIn("accepts only the first call whose name was explicitly offered", normalized_content) + self.assertIn("undeclared local tool attempts fail closed", normalized_content) + self.assertIn("does not grant Claude Code direct access to the infrastructure", normalized_content) self.assertIn("not represented as a zero-dollar API price", normalized_content) self.assertIn("per-run metered-API budget as not applicable", normalized_content) self.assertIn("inference_route=coding_plan_allowance", content)