fix Unicode text truncation

cmd/chat.go

@@ -1210,7 +1210,7 @@ func validateAttributeValue(key string, value any, app *App) any {
 		const maxValueLength = 1000
 		if len(strValue) > maxValueLength {
 			app.lo.Warn("form field value exceeds maximum length", "key", key, "length", len(strValue), "max", maxValueLength)
-			return strValue[:maxValueLength]
+			return stringutil.TruncateUTF8(strValue, maxValueLength)
 		}
 		return strValue
 	}

cmd/chat_test.go

@@ -2,11 +2,30 @@ package main
 
 import (
 	"math"
+	"strings"
 	"testing"
+	"unicode/utf8"
 
 	"github.com/abhinavxd/libredesk/internal/inbox/channel/livechat"
+	"github.com/zerodha/logf"
 )
 
+func TestValidateAttributeValuePreservesUTF8(t *testing.T) {
+	lo := logf.New(logf.Opts{})
+	app := &App{lo: &lo}
+	for _, char := range []string{"a", "é", "界", "😀"} {
+		for padding := range len(char) {
+			prefix := strings.Repeat("a", padding)
+			value := prefix + strings.Repeat(char, 1001)
+			want := prefix + strings.Repeat(char, (1000-padding)/len(char))
+			got := validateAttributeValue("description", value, app).(string)
+			if !utf8.ValidString(got) || got != want {
+				t.Fatalf("padding %d: got %q, want %q", padding, got, want)
+			}
+		}
+	}
+}
+
 func TestIsFormFieldValuePresent(t *testing.T) {
 	tests := []struct {
 		name  string

cmd/csat.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 
 	"github.com/abhinavxd/libredesk/internal/envelope"
+	"github.com/abhinavxd/libredesk/internal/stringutil"
 	"github.com/valyala/fasthttp"
 	"github.com/zerodha/fastglue"
 )
@@ -202,13 +203,9 @@ func validateCSATForm(r *fastglue.Request) (int, string, json.RawMessage, string
 		if len(meta) >= maxCsatMetaKeys {
 			return
 		}
-		if len(k) > maxCsatMetaKeyLength {
-			k = k[:maxCsatMetaKeyLength]
-		}
+		k = stringutil.TruncateUTF8(k, maxCsatMetaKeyLength)
 		v := string(value)
-		if len(v) > maxCsatMetaValLength {
-			v = v[:maxCsatMetaValLength]
-		}
+		v = stringutil.TruncateUTF8(v, maxCsatMetaValLength)
 		meta[k] = v
 	})
 

cmd/widget_ws.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/abhinavxd/libredesk/internal/httputil"
 	"github.com/abhinavxd/libredesk/internal/inbox/channel/livechat"
+	"github.com/abhinavxd/libredesk/internal/stringutil"
 	"github.com/fasthttp/websocket"
 	"github.com/google/uuid"
 	"github.com/zerodha/fastglue"
@@ -331,12 +332,8 @@ func handleWidgetPageVisit(app *App, data json.RawMessage, contactID int) {
 		return
 	}
 
-	if len(visit.URL) > 2048 {
-		visit.URL = visit.URL[:2048]
-	}
-	if len(visit.Title) > 256 {
-		visit.Title = visit.Title[:256]
-	}
+	visit.URL = stringutil.TruncateUTF8(visit.URL, 2048 /* maxBytes */)
+	visit.Title = stringutil.TruncateUTF8(visit.Title, 256 /* maxBytes */)
 
 	parsedURL, err := url.Parse(visit.URL)
 	if err != nil || (parsedURL.Scheme != "http" && parsedURL.Scheme != "https") {

internal/conversation/continuity.go

@@ -145,7 +145,7 @@ func (m *Manager) sendContinuityEmail(conv models.ContinuityConversation, maxMes
 		emailSubject = m.formatRefMarker(conv.ReferenceNumber)
 		if text := strings.TrimSpace(unreadMessages[0].TextContent); text != "" {
 			if len(text) > 100 {
-				text = text[:100] + "..."
+				text = stringutil.TruncateUTF8(text, 100 /* maxBytes */) + "..."
 			}
 			emailSubject = fmt.Sprintf("%s - %s", text, emailSubject)
 		}

internal/stringutil/stringutil.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 	"unicode"
+	"unicode/utf8"
 
 	"github.com/inbucket/html2text"
 	"github.com/yuin/goldmark"
@@ -90,6 +91,19 @@ func SanitizeUTF8(s string) string {
 	return strings.ToValidUTF8(s, "�")
 }
 
+func TruncateUTF8(s string, maxBytes int) string {
+	if maxBytes <= 0 {
+		return ""
+	}
+	if len(s) <= maxBytes {
+		return s
+	}
+	for maxBytes > 0 && !utf8.RuneStart(s[maxBytes]) {
+		maxBytes--
+	}
+	return s[:maxBytes]
+}
+
 // GenerateSlug generates a URL-friendly slug from a title; a script with no ASCII form falls back to a random slug.
 func GenerateSlug(title string) string {
 	slug := strings.ToLower(strings.TrimSpace(foldAccents(title)))

internal/stringutil/stringutil_test.go

@@ -1,10 +1,39 @@
 package stringutil
 
 import (
+	"strings"
 	"testing"
 	"time"
+	"unicode/utf8"
 )
 
+func TestTruncateUTF8(t *testing.T) {
+	texts := []string{
+		"",
+		strings.Repeat("a", 1100),
+		strings.Repeat("é", 550),
+		strings.Repeat("界", 400),
+		strings.Repeat("😀", 300),
+		strings.Repeat("aé界😀", 120),
+		"这是客服的测试回复：如果你在邮箱里看到这条，说明离线邮件已经打通。直接回复这封邮件，内容会回到这个会话里。",
+	}
+	for _, text := range texts {
+		for limit := -1; limit <= len(text)+1; limit++ {
+			got := TruncateUTF8(text, limit)
+			var want strings.Builder
+			for _, r := range text {
+				if want.Len()+utf8.RuneLen(r) > limit {
+					break
+				}
+				want.WriteRune(r)
+			}
+			if !utf8.ValidString(got) || got != want.String() {
+				t.Fatalf("limit %d: got %q, want %q", limit, got, want.String())
+			}
+		}
+	}
+}
+
 func TestWhatsAppPhoneForDialCode(t *testing.T) {
 	tests := []struct {
 		name        string