mirror of
https://github.com/abhinavxd/libredesk.git
synced 2026-09-11 13:28:57 +00:00
Abhinav Raut
a4be8ecb95
Correctness fixes from the internal/ai code review: - gate reindex commits on the snippet row still existing, so a delete racing reconcile can't re-insert its embeddings forever - cap one logical provider request at 90s across all retries, so a hanging provider can't stall the reply box for minutes - return proper error envelopes from the copilot message store instead of raw sqlx errors that surfaced as "Error interface conversion failed" - reject blank AI replies in the generate-reply and copilot handlers - build GET tool URLs with url.Parse so a # in the URL doesn't swallow the query params - let a blank api_key clear the stored key (masked keeps it), restoring a way to disable AI Cleanups and hardening: - share one SSRF transport between tool and provider clients instead of building a new transport per AI call - copy the provider config struct instead of listing every field - backfill temperature 0.7 for configured providers upgrading from releases that hardcoded it - only pass user/assistant roles from copilot history to the provider - reject enc:-prefixed secrets that would be stored raw and fail decrypt - rune-safe truncation of test errors, one-line doc comments
76 lines
1.7 KiB
Go
76 lines
1.7 KiB
Go
package ai
|
|
|
|
import (
|
|
"strings"
|
|
"sync"
|
|
"unicode/utf8"
|
|
|
|
"github.com/pkoukk/tiktoken-go"
|
|
tiktokenloader "github.com/pkoukk/tiktoken-go-loader"
|
|
"github.com/zerodha/logf"
|
|
)
|
|
|
|
// All OpenAI embedding models (text-embedding-3-*, ada-002) tokenize with cl100k_base.
|
|
const embeddingEncoding = "cl100k_base"
|
|
|
|
var (
|
|
encoderOnce sync.Once
|
|
encoder *tiktoken.Tiktoken
|
|
)
|
|
|
|
// initEncoder loads tiktoken's BPE vocab from the binary (no network fetch); on failure encoder stays nil and callers fall back to a rune estimate.
|
|
func initEncoder(lo *logf.Logger) {
|
|
encoderOnce.Do(func() {
|
|
tiktoken.SetBpeLoader(tiktokenloader.NewOfflineLoader())
|
|
enc, err := tiktoken.GetEncoding(embeddingEncoding)
|
|
if err != nil {
|
|
if lo != nil {
|
|
lo.Error("could not load tiktoken encoding, falling back to rune-based token estimates", "error", err)
|
|
}
|
|
return
|
|
}
|
|
encoder = enc
|
|
})
|
|
}
|
|
|
|
// newTokenCounter returns the chunker's token-counting function.
|
|
func newTokenCounter(lo *logf.Logger) func(string) int {
|
|
initEncoder(lo)
|
|
return countTokens
|
|
}
|
|
|
|
func countTokens(s string) int {
|
|
if encoder == nil {
|
|
return len([]rune(s)) * 2 / 5
|
|
}
|
|
return len(encoder.Encode(s, nil, nil))
|
|
}
|
|
|
|
// capToTokens truncates s to at most maxTokens, byte-capping on a rune boundary when the encoder is unavailable.
|
|
func capToTokens(s string, maxTokens int) string {
|
|
if maxTokens <= 0 {
|
|
return ""
|
|
}
|
|
if encoder == nil {
|
|
if len(s) <= maxTokens {
|
|
return s
|
|
}
|
|
return trimToRuneBoundary(s, maxTokens)
|
|
}
|
|
toks := encoder.Encode(s, nil, nil)
|
|
if len(toks) <= maxTokens {
|
|
return s
|
|
}
|
|
return strings.ToValidUTF8(encoder.Decode(toks[:maxTokens]), "")
|
|
}
|
|
|
|
func trimToRuneBoundary(s string, n int) string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
for n > 0 && !utf8.RuneStart(s[n]) {
|
|
n--
|
|
}
|
|
return s[:n]
|
|
}
|