diff --git a/cmd/chat.go b/cmd/chat.go index 37c44fa4..00940e56 100644 --- a/cmd/chat.go +++ b/cmd/chat.go @@ -1210,7 +1210,7 @@ func validateAttributeValue(key string, value any, app *App) any { const maxValueLength = 1000 if len(strValue) > maxValueLength { app.lo.Warn("form field value exceeds maximum length", "key", key, "length", len(strValue), "max", maxValueLength) - return strValue[:maxValueLength] + return stringutil.TruncateUTF8(strValue, maxValueLength) } return strValue } diff --git a/cmd/chat_test.go b/cmd/chat_test.go index 7bcb1f81..1fae9f8d 100644 --- a/cmd/chat_test.go +++ b/cmd/chat_test.go @@ -2,11 +2,30 @@ package main import ( "math" + "strings" "testing" + "unicode/utf8" "github.com/abhinavxd/libredesk/internal/inbox/channel/livechat" + "github.com/zerodha/logf" ) +func TestValidateAttributeValuePreservesUTF8(t *testing.T) { + lo := logf.New(logf.Opts{}) + app := &App{lo: &lo} + for _, char := range []string{"a", "é", "界", "😀"} { + for padding := range len(char) { + prefix := strings.Repeat("a", padding) + value := prefix + strings.Repeat(char, 1001) + want := prefix + strings.Repeat(char, (1000-padding)/len(char)) + got := validateAttributeValue("description", value, app).(string) + if !utf8.ValidString(got) || got != want { + t.Fatalf("padding %d: got %q, want %q", padding, got, want) + } + } + } +} + func TestIsFormFieldValuePresent(t *testing.T) { tests := []struct { name string diff --git a/cmd/csat.go b/cmd/csat.go index 707bcc49..0171a0cf 100644 --- a/cmd/csat.go +++ b/cmd/csat.go @@ -5,6 +5,7 @@ import ( "strconv" "github.com/abhinavxd/libredesk/internal/envelope" + "github.com/abhinavxd/libredesk/internal/stringutil" "github.com/valyala/fasthttp" "github.com/zerodha/fastglue" ) @@ -202,13 +203,9 @@ func validateCSATForm(r *fastglue.Request) (int, string, json.RawMessage, string if len(meta) >= maxCsatMetaKeys { return } - if len(k) > maxCsatMetaKeyLength { - k = k[:maxCsatMetaKeyLength] - } + k = stringutil.TruncateUTF8(k, maxCsatMetaKeyLength) v := string(value) - if len(v) > maxCsatMetaValLength { - v = v[:maxCsatMetaValLength] - } + v = stringutil.TruncateUTF8(v, maxCsatMetaValLength) meta[k] = v }) diff --git a/cmd/widget_ws.go b/cmd/widget_ws.go index e279f707..843dca49 100644 --- a/cmd/widget_ws.go +++ b/cmd/widget_ws.go @@ -12,6 +12,7 @@ import ( "github.com/abhinavxd/libredesk/internal/httputil" "github.com/abhinavxd/libredesk/internal/inbox/channel/livechat" + "github.com/abhinavxd/libredesk/internal/stringutil" "github.com/fasthttp/websocket" "github.com/google/uuid" "github.com/zerodha/fastglue" @@ -331,12 +332,8 @@ func handleWidgetPageVisit(app *App, data json.RawMessage, contactID int) { return } - if len(visit.URL) > 2048 { - visit.URL = visit.URL[:2048] - } - if len(visit.Title) > 256 { - visit.Title = visit.Title[:256] - } + visit.URL = stringutil.TruncateUTF8(visit.URL, 2048 /* maxBytes */) + visit.Title = stringutil.TruncateUTF8(visit.Title, 256 /* maxBytes */) parsedURL, err := url.Parse(visit.URL) if err != nil || (parsedURL.Scheme != "http" && parsedURL.Scheme != "https") { diff --git a/internal/conversation/continuity.go b/internal/conversation/continuity.go index eb4837a5..62ce3057 100644 --- a/internal/conversation/continuity.go +++ b/internal/conversation/continuity.go @@ -145,7 +145,7 @@ func (m *Manager) sendContinuityEmail(conv models.ContinuityConversation, maxMes emailSubject = m.formatRefMarker(conv.ReferenceNumber) if text := strings.TrimSpace(unreadMessages[0].TextContent); text != "" { if len(text) > 100 { - text = text[:100] + "..." + text = stringutil.TruncateUTF8(text, 100 /* maxBytes */) + "..." } emailSubject = fmt.Sprintf("%s - %s", text, emailSubject) } diff --git a/internal/stringutil/stringutil.go b/internal/stringutil/stringutil.go index dc9a918b..5efb192e 100644 --- a/internal/stringutil/stringutil.go +++ b/internal/stringutil/stringutil.go @@ -10,6 +10,7 @@ import ( "strings" "time" "unicode" + "unicode/utf8" "github.com/inbucket/html2text" "github.com/yuin/goldmark" @@ -90,6 +91,19 @@ func SanitizeUTF8(s string) string { return strings.ToValidUTF8(s, "�") } +func TruncateUTF8(s string, maxBytes int) string { + if maxBytes <= 0 { + return "" + } + if len(s) <= maxBytes { + return s + } + for maxBytes > 0 && !utf8.RuneStart(s[maxBytes]) { + maxBytes-- + } + return s[:maxBytes] +} + // GenerateSlug generates a URL-friendly slug from a title; a script with no ASCII form falls back to a random slug. func GenerateSlug(title string) string { slug := strings.ToLower(strings.TrimSpace(foldAccents(title))) diff --git a/internal/stringutil/stringutil_test.go b/internal/stringutil/stringutil_test.go index e7164bd9..b79744ea 100644 --- a/internal/stringutil/stringutil_test.go +++ b/internal/stringutil/stringutil_test.go @@ -1,10 +1,39 @@ package stringutil import ( + "strings" "testing" "time" + "unicode/utf8" ) +func TestTruncateUTF8(t *testing.T) { + texts := []string{ + "", + strings.Repeat("a", 1100), + strings.Repeat("é", 550), + strings.Repeat("界", 400), + strings.Repeat("😀", 300), + strings.Repeat("aé界😀", 120), + "这是客服的测试回复:如果你在邮箱里看到这条,说明离线邮件已经打通。直接回复这封邮件,内容会回到这个会话里。", + } + for _, text := range texts { + for limit := -1; limit <= len(text)+1; limit++ { + got := TruncateUTF8(text, limit) + var want strings.Builder + for _, r := range text { + if want.Len()+utf8.RuneLen(r) > limit { + break + } + want.WriteRune(r) + } + if !utf8.ValidString(got) || got != want.String() { + t.Fatalf("limit %d: got %q, want %q", limit, got, want.String()) + } + } + } +} + func TestWhatsAppPhoneForDialCode(t *testing.T) { tests := []struct { name string