Sanitize LLM control tokens from OpenAI-compatible responses

Some local models (llama.cpp, LM Studio) output internal control tokens
like <|channel|>, <|constrain|>, <|message|> instead of using proper
function calling. These tokens leak into the UI creating a poor UX.

This adds sanitization to strip these control tokens from both streaming
and non-streaming responses before they reach the user.
This commit is contained in:
rcourtman
2026-02-03 13:12:17 +00:00
parent 71f80c8a99
commit e5eb15918e
+22 -5
View File
@@ -8,12 +8,23 @@ import (
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"github.com/rs/zerolog/log"
)
// controlTokenPattern matches common LLM control tokens that shouldn't appear in output
// These are internal tokens used by some models (llama.cpp, etc.) for chat templating
var controlTokenPattern = regexp.MustCompile(`<\|[a-zA-Z_]+\|>`)
// sanitizeContent removes LLM control tokens from content that shouldn't be shown to users
func sanitizeContent(content string) string {
// Remove control tokens like <|channel|>, <|constrain|>, <|message|>, etc.
return controlTokenPattern.ReplaceAllString(content, "")
}
const (
openaiAPIURL = "https://api.openai.com/v1/chat/completions"
openaiMaxRetries = 3
@@ -425,6 +436,9 @@ func (c *OpenAIClient) Chat(ctx context.Context, req ChatRequest) (*ChatResponse
contentToUse = choice.Message.ReasoningContent
}
// Sanitize content to remove any leaked control tokens from local models
contentToUse = sanitizeContent(contentToUse)
result := &ChatResponse{
Content: contentToUse,
ReasoningContent: choice.Message.ReasoningContent, // DeepSeek thinking mode
@@ -774,12 +788,15 @@ func (c *OpenAIClient) ChatStream(ctx context.Context, req ChatRequest, callback
delta := choice.Delta
// Regular content
// Regular content - sanitize to remove any leaked control tokens
if delta.Content != "" {
callback(StreamEvent{
Type: "content",
Data: ContentEvent{Text: delta.Content},
})
sanitized := sanitizeContent(delta.Content)
if sanitized != "" {
callback(StreamEvent{
Type: "content",
Data: ContentEvent{Text: sanitized},
})
}
}
// Reasoning content (DeepSeek)