add help article citations to AI replies

Link cited public help articles in chat replies and emails. Batch reference lookups and keep source metadata out of customer replies.
This commit is contained in:
Abhinav Raut
2026-10-10 20:57:12 +05:30
parent 5b3fbcb381
commit 170fcd026d
28 changed files with 840 additions and 57 deletions
+1
View File
@@ -289,6 +289,7 @@ func main() {
waClient := initWhatsAppClient()
waTemplates := initWhatsAppTemplates(db, i18n, waClient, inbox)
conversation.SetWhatsAppTemplateStore(waTemplates)
conversation.SetArticleReferenceStore(helpCenter)
go automation.Run(ctx, automationWorkers)
go autoassigner.Run(ctx, autoAssignInterval)
+2
View File
@@ -71,6 +71,7 @@ func handleGetMessages(r *fastglue.Request) error {
}
rootURL, _ := app.setting.GetAppRootURL()
app.conversation.RenderMessagesArticleReferences(messages)
for i := range messages {
total = messages[i].Total
app.conversation.SignAttachmentURLs(messages[i].Attachments)
@@ -138,6 +139,7 @@ func handleGetMessage(r *fastglue.Request) error {
}
rootURL, _ := app.setting.GetAppRootURL()
app.conversation.RenderArticleReferences(&message)
app.conversation.SignAttachmentURLs(message.Attachments)
resolveQuotedCIDs(app, &message)
resolveAttachmentCIDs(&message, rootURL)
+1 -1
View File
@@ -47,7 +47,7 @@ var migList = []migFunc{
{"v2.5.0", migrations.V2_5_0},
{"v2.6.0", migrations.V2_6_0},
{"v2.8.0", migrations.V2_8_0},
{"v2.9.0-rc.8", migrations.V2_9_0},
{"v2.9.0-rc.8", migrations.V2_9_0_RC8},
{"v2.9.0-rc.9", migrations.V2_9_0_RC9},
}
@@ -134,6 +134,17 @@
</FormItem>
</FormField>
<FormField v-slot="{ componentField, handleChange, meta }" name="citations_enabled">
<FormItem>
<SwitchField
:title="t('admin.ai.assistant.includeArticleReferences')"
:description="t('admin.ai.assistant.includeArticleReferencesHint')"
:checked="componentField.modelValue"
@update:checked="(value) => handleChange(value, meta.validated)"
/>
</FormItem>
</FormField>
<FormField v-slot="{ componentField, handleChange }" name="handoff_enabled">
<FormItem>
<SwitchField
@@ -317,6 +328,7 @@ const form = useForm({
.max(20, { message: t('admin.ai.assistant.maxTurnsHint') }),
fallback_team_id: z.string().optional(),
handoff_enabled: z.boolean().optional(),
citations_enabled: z.boolean().optional(),
languages: z.array(z.string()).optional(),
instructions: z.string().optional(),
guardrails: z.string().optional(),
@@ -332,6 +344,7 @@ const form = useForm({
max_turns: 6,
fallback_team_id: 'none',
handoff_enabled: true,
citations_enabled: true,
languages: [],
instructions: '',
guardrails: '',
@@ -360,6 +373,7 @@ watch(
max_turns: values.max_turns ?? 6,
fallback_team_id: values.fallback_team_id ? String(values.fallback_team_id) : 'none',
handoff_enabled: values.handoff_enabled ?? true,
citations_enabled: values.citations_enabled ?? !props.isEditing,
languages: [...(values.languages || [])],
instructions: values.instructions || '',
guardrails: values.guardrails || '',
@@ -404,6 +418,7 @@ const onSubmit = form.handleSubmit(async (values) => {
? Number(values.fallback_team_id)
: null,
handoff_enabled: !!values.handoff_enabled,
citations_enabled: !!values.citations_enabled,
languages: values.languages || [],
instructions: values.instructions || '',
guardrails: values.guardrails || '',
@@ -67,6 +67,25 @@
</span>
</div>
</div>
<div v-if="previewReferences.length" class="space-y-2">
<div class="text-sm font-medium text-foreground">
{{ t('globals.terms.articleReference', 2) }}
</div>
<ul class="space-y-1">
<li v-for="source in previewReferences" :key="source.id">
<Button
as="a"
variant="link"
:href="source.url"
target="_blank"
rel="noopener noreferrer"
class="h-auto px-0 whitespace-normal text-left"
>
{{ source.title }}
</Button>
</li>
</ul>
</div>
<div v-if="previewSources.length" class="space-y-2">
<div class="text-sm font-medium text-foreground">
{{ t('admin.ai.assistant.preview.sources') }}
@@ -189,6 +208,9 @@ const previewMessage = ref('')
const previewReply = ref('')
const previewSources = ref([])
const previewLoading = ref(false)
const previewReferences = computed(() =>
previewSources.value.filter((source) => source.url)
)
const fmtNumber = (value) => Number(value ?? 0).toLocaleString()
const fmtPercent = (value) => `${Number(value ?? 0).toFixed(1)}%`
+3
View File
@@ -71,6 +71,8 @@
"admin.ai.assistant.guardrails": "Guardrails",
"admin.ai.assistant.guardrailsHint": "Rules the assistant must never break.",
"admin.ai.assistant.guardrailsPlaceholder": "Never issue a refund without a human agent's approval. Never share other customers' order details.",
"admin.ai.assistant.includeArticleReferences": "Include article references",
"admin.ai.assistant.includeArticleReferencesHint": "Link to public help center articles used in chat and email replies. Private knowledge snippets are never listed.",
"admin.ai.assistant.instructions": "Instructions",
"admin.ai.assistant.instructionsHint": "Custom system prompt that guides how this assistant responds.",
"admin.ai.assistant.instructionsPlaceholder": "Always confirm the order ID before looking anything up. Keep replies under 3 sentences.",
@@ -1079,6 +1081,7 @@
"globals.terms.appliesTo": "Applies to",
"globals.terms.arguments": "Arguments",
"globals.terms.article": "Article | Articles",
"globals.terms.articleReference": "Article reference | Article references",
"globals.terms.ascending": "Ascending",
"globals.terms.assignedTeam": "Assigned team",
"globals.terms.assignee": "Assignee",
+2 -2
View File
@@ -280,7 +280,7 @@ func (m *Manager) CreateAssistant(a models.Assistant) (models.Assistant, error)
}
var id int
if err := tx.Stmtx(m.q.InsertAssistant).QueryRow(userID, a.Description, a.Instructions, a.Guardrails, a.Tone, a.ResponseLength, a.MaxTurns, a.FallbackTeamID, a.Enabled, a.Expectation, a.HandoffEnabled, a.Languages).Scan(&id); err != nil {
if err := tx.Stmtx(m.q.InsertAssistant).QueryRow(userID, a.Description, a.Instructions, a.Guardrails, a.Tone, a.ResponseLength, a.MaxTurns, a.FallbackTeamID, a.Enabled, a.Expectation, a.HandoffEnabled, a.Languages, a.CitationsEnabled).Scan(&id); err != nil {
m.lo.Error("error creating assistant", "error", err)
return models.Assistant{}, envelope.NewError(envelope.GeneralError, m.i18n.T("globals.messages.somethingWentWrong"), nil)
}
@@ -317,7 +317,7 @@ func (m *Manager) UpdateAssistant(id int, a models.Assistant) (models.Assistant,
}
defer tx.Rollback()
if _, err := tx.Stmtx(m.q.UpdateAssistant).Exec(id, a.Description, a.Instructions, a.Guardrails, a.Tone, a.ResponseLength, a.MaxTurns, a.FallbackTeamID, a.Enabled, a.Expectation, a.HandoffEnabled, a.Languages); err != nil {
if _, err := tx.Stmtx(m.q.UpdateAssistant).Exec(id, a.Description, a.Instructions, a.Guardrails, a.Tone, a.ResponseLength, a.MaxTurns, a.FallbackTeamID, a.Enabled, a.Expectation, a.HandoffEnabled, a.Languages, a.CitationsEnabled); err != nil {
m.lo.Error("error updating assistant", "error", err)
return models.Assistant{}, envelope.NewError(envelope.GeneralError, m.i18n.T("globals.messages.somethingWentWrong"), nil)
}
+70
View File
@@ -0,0 +1,70 @@
package aiagent
import (
"encoding/json"
"strings"
aimodels "github.com/abhinavxd/libredesk/internal/ai/models"
)
func splitArticleSources(answer string, hits []aimodels.SearchResult, enabled bool) (string, []int) {
allowed := map[int]bool{}
if enabled {
for _, hit := range hits {
if hit.SourceType == aimodels.SourceHelpArticle && hit.SourceID > 0 {
allowed[hit.SourceID] = true
}
}
}
var articleIDs []int
var body strings.Builder
for {
before, payload, found := strings.Cut(answer, sourcesMarker)
if !found {
body.WriteString(answer)
break
}
body.WriteString(before)
payload = strings.TrimLeft(payload, " \t\r\n")
decoder := json.NewDecoder(strings.NewReader(payload))
var ids []int
err := decoder.Decode(&ids)
end := int(decoder.InputOffset())
if end == 0 {
end = len(payload)
if strings.HasPrefix(payload, "[") {
if close := strings.IndexByte(payload, ']'); close >= 0 {
end = close + 1
}
} else if newline := strings.IndexByte(payload, '\n'); newline >= 0 {
end = newline
}
for _, marker := range []string{confirmMarker, suggestionsMarker, sourcesMarker} {
if next := strings.Index(payload, marker); next >= 0 {
end = min(end, next)
}
}
}
if err == nil {
for _, id := range ids {
if allowed[id] {
articleIDs = append(articleIDs, id)
delete(allowed, id)
}
}
}
answer = strings.TrimLeft(payload[end:], " \t\r")
if strings.HasSuffix(strings.TrimRight(before, " \t\r"), "\n") {
answer = strings.TrimPrefix(answer, "\n")
}
}
return strings.TrimSpace(body.String()), articleIDs
}
func replyMeta(suggestions []string, articleIDs []int) map[string]any {
meta := suggestedRepliesMeta(suggestions)
if len(articleIDs) > 0 {
meta["ai_article_ids"] = articleIDs
}
return meta
}
+20 -18
View File
@@ -22,24 +22,25 @@ var (
// Assistant is one AI assistant: a persona plus the ai_assistant user that carries its identity.
type Assistant struct {
ID int `db:"id" json:"id"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
UserID int `db:"user_id" json:"user_id"`
Name string `db:"name" json:"name"`
AvatarURL null.String `db:"avatar_url" json:"avatar_url"`
Description string `db:"description" json:"description"`
Instructions string `db:"instructions" json:"instructions"`
Guardrails string `db:"guardrails" json:"guardrails"`
Expectation string `db:"expectation" json:"expectation"`
Tone string `db:"tone" json:"tone"`
ResponseLength string `db:"response_length" json:"response_length"`
MaxTurns int `db:"max_turns" json:"max_turns"`
FallbackTeamID null.Int `db:"fallback_team_id" json:"fallback_team_id"`
HandoffEnabled bool `db:"handoff_enabled" json:"handoff_enabled"`
Languages pq.StringArray `db:"languages" json:"languages"`
Enabled bool `db:"enabled" json:"enabled"`
ToolIDs []int `db:"-" json:"tool_ids"`
ID int `db:"id" json:"id"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
UserID int `db:"user_id" json:"user_id"`
Name string `db:"name" json:"name"`
AvatarURL null.String `db:"avatar_url" json:"avatar_url"`
Description string `db:"description" json:"description"`
Instructions string `db:"instructions" json:"instructions"`
Guardrails string `db:"guardrails" json:"guardrails"`
Expectation string `db:"expectation" json:"expectation"`
Tone string `db:"tone" json:"tone"`
ResponseLength string `db:"response_length" json:"response_length"`
MaxTurns int `db:"max_turns" json:"max_turns"`
FallbackTeamID null.Int `db:"fallback_team_id" json:"fallback_team_id"`
CitationsEnabled bool `db:"citations_enabled" json:"citations_enabled"`
HandoffEnabled bool `db:"handoff_enabled" json:"handoff_enabled"`
Languages pq.StringArray `db:"languages" json:"languages"`
Enabled bool `db:"enabled" json:"enabled"`
ToolIDs []int `db:"-" json:"tool_ids"`
// RemoveAvatar, when set on a save request, clears the assistant's current avatar.
RemoveAvatar bool `db:"-" json:"remove_avatar"`
@@ -51,6 +52,7 @@ type PreviewSource struct {
Type string `json:"type"`
Title string `json:"title"`
Score float64 `json:"score"`
URL string `json:"url,omitempty"`
}
// RecentConversation is a summary row of a contact's past conversation, fed to the assistant as context.
+3
View File
@@ -167,6 +167,9 @@ func buildSystemPrompt(a models.Assistant) string {
b.WriteString("\n\nInstructions from the workspace admin (follow these):\n")
b.WriteString(instr)
}
if a.CitationsEnabled {
b.WriteString("\n\nWhen your answer uses help center articles from this turn's search results, add a separate line in this exact form: [[sources]] [12,34]. Include only the article_id values of articles you actually used, once each. Never cite snippets or articles from earlier turns, and never invent an article ID or URL. Put this line before [[confirm]] or [[suggestions]] when present. Omit it when no help center article supports the answer, including greetings, clarifying questions, refusals and handoffs. The customer never sees this line. Article links are added by the application.")
}
if guard := strings.TrimSpace(a.Guardrails); guard != "" {
b.WriteString("\n\nGuardrails (never violate these):\n")
b.WriteString(guard)
+6 -6
View File
@@ -1,20 +1,20 @@
-- name: get-assistants
SELECT a.id, a.created_at, a.updated_at, a.user_id, u.first_name AS name, u.avatar_url,
a.description, a.instructions, a.guardrails, a.expectation, a.tone, a.response_length, a.max_turns, a.fallback_team_id, a.handoff_enabled, a.languages, a.enabled
a.description, a.instructions, a.guardrails, a.expectation, a.tone, a.response_length, a.max_turns, a.fallback_team_id, a.handoff_enabled, a.citations_enabled, a.languages, a.enabled
FROM ai_assistants a
JOIN users u ON u.id = a.user_id AND u.deleted_at IS NULL
ORDER BY a.updated_at DESC;
-- name: get-assistant
SELECT a.id, a.created_at, a.updated_at, a.user_id, u.first_name AS name, u.avatar_url,
a.description, a.instructions, a.guardrails, a.expectation, a.tone, a.response_length, a.max_turns, a.fallback_team_id, a.handoff_enabled, a.languages, a.enabled
a.description, a.instructions, a.guardrails, a.expectation, a.tone, a.response_length, a.max_turns, a.fallback_team_id, a.handoff_enabled, a.citations_enabled, a.languages, a.enabled
FROM ai_assistants a
JOIN users u ON u.id = a.user_id AND u.deleted_at IS NULL
WHERE a.id = $1;
-- name: get-assistant-by-user-id
SELECT a.id, a.created_at, a.updated_at, a.user_id, u.first_name AS name, u.avatar_url,
a.description, a.instructions, a.guardrails, a.expectation, a.tone, a.response_length, a.max_turns, a.fallback_team_id, a.handoff_enabled, a.languages, a.enabled
a.description, a.instructions, a.guardrails, a.expectation, a.tone, a.response_length, a.max_turns, a.fallback_team_id, a.handoff_enabled, a.citations_enabled, a.languages, a.enabled
FROM ai_assistants a
JOIN users u ON u.id = a.user_id AND u.deleted_at IS NULL
WHERE a.user_id = $1;
@@ -29,13 +29,13 @@ VALUES ('ai_assistant', $1, '', true)
RETURNING id;
-- name: insert-assistant
INSERT INTO ai_assistants (user_id, description, instructions, guardrails, tone, response_length, max_turns, fallback_team_id, enabled, expectation, handoff_enabled, languages)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
INSERT INTO ai_assistants (user_id, description, instructions, guardrails, tone, response_length, max_turns, fallback_team_id, enabled, expectation, handoff_enabled, languages, citations_enabled)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
RETURNING id;
-- name: update-assistant
UPDATE ai_assistants
SET description = $2, instructions = $3, guardrails = $4, tone = $5, response_length = $6, max_turns = $7, fallback_team_id = $8, enabled = $9, expectation = $10, handoff_enabled = $11, languages = $12, updated_at = now()
SET description = $2, instructions = $3, guardrails = $4, tone = $5, response_length = $6, max_turns = $7, fallback_team_id = $8, enabled = $9, expectation = $10, handoff_enabled = $11, languages = $12, citations_enabled = $13, updated_at = now()
WHERE id = $1;
-- name: get-assistant-expectation-by-user-id
+8 -4
View File
@@ -70,9 +70,9 @@ type runOutcome struct {
}
type searchKnowledgeTool struct {
m *Manager
// collect, when set, receives the results each search actually used (preview source attribution).
collect func([]aimodels.SearchResult)
m *Manager
collect func([]aimodels.SearchResult)
citationsEnabled bool
}
func (t *searchKnowledgeTool) Name() string { return "search_knowledge_base" }
@@ -115,7 +115,11 @@ func (t *searchKnowledgeTool) Execute(ctx context.Context, args string) (string,
if t.collect != nil {
used = append(used, r)
}
fmt.Fprintf(&b, "<<result %d>>\n%s\n<<end result %d>>\n\n", i+1, neutralizeMarkers(r.ChunkText), i+1)
fmt.Fprintf(&b, "<<result %d>>\n", i+1)
if t.citationsEnabled && r.SourceType == aimodels.SourceHelpArticle {
fmt.Fprintf(&b, "article_id: %d\n", r.SourceID)
}
fmt.Fprintf(&b, "%s\n<<end result %d>>\n\n", neutralizeMarkers(r.ChunkText), i+1)
}
if t.collect != nil {
t.collect(used)
+22 -4
View File
@@ -35,6 +35,7 @@ const (
// can be split off and sent as a separate message.
confirmMarker = "[[confirm]]"
suggestionsMarker = "[[suggestions]]"
sourcesMarker = "[[sources]]"
// typingRefreshInterval must stay under the widget's 5s typing expiry (TYPING_RECEIVE_TIMEOUT).
typingRefreshInterval = 3 * time.Second
@@ -285,8 +286,9 @@ func (m *Manager) handle(ctx context.Context, convID int) {
m.lo.Debug("ai agent verification state", "conversation_uuid", conv.UUID, "channel", conv.InboxChannel, "contact_type", conv.Contact.Type, "has_email", conv.Contact.Email.String != "", "verified", runVerified)
outcome := &runOutcome{}
var hits []aimodels.SearchResult
tools := []ai.Tool{
&searchKnowledgeTool{m: m},
&searchKnowledgeTool{m: m, citationsEnabled: assistant.CitationsEnabled, collect: func(rs []aimodels.SearchResult) { hits = append(hits, rs...) }},
&resolveTool{m: m, conv: conv, outcome: outcome},
}
if assistant.HandoffEnabled {
@@ -351,6 +353,7 @@ func (m *Manager) handle(ctx context.Context, convID int) {
return
}
// The model's text answer is the reply to the customer. Handoff and resolve are separate tool actions.
answer, articleIDs := splitArticleSources(answer, hits, assistant.CitationsEnabled)
answer, confirm := splitConfirmation(strings.TrimSpace(answer))
answer, answerSuggestions := splitSuggestions(answer)
confirm, confirmSuggestions := splitSuggestions(confirm)
@@ -361,7 +364,7 @@ func (m *Manager) handle(ctx context.Context, convID int) {
}
if answer != "" {
m.lo.Debug("ai agent replying", "conversation_uuid", conv.UUID, "reply_len", len(answer), "resolved", outcome.resolved)
if err := m.postReply(conv, assistant, answer, suggestedRepliesMeta(answerSuggestions)); err != nil {
if err := m.postReply(conv, assistant, answer, replyMeta(answerSuggestions, articleIDs)); err != nil {
m.handoff(conv, assistant, m.i18n.T("ai.agent.handoffError"))
return
}
@@ -545,7 +548,7 @@ func (m *Manager) PreviewReply(ctx context.Context, assistantID int, message str
}
history := []aimodels.ChatMessage{{Role: aimodels.RoleUser, Content: message}}
var hits []aimodels.SearchResult
tools := []ai.Tool{&searchKnowledgeTool{m: m, collect: func(rs []aimodels.SearchResult) { hits = append(hits, rs...) }}}
tools := []ai.Tool{&searchKnowledgeTool{m: m, citationsEnabled: a.CitationsEnabled, collect: func(rs []aimodels.SearchResult) { hits = append(hits, rs...) }}}
runCtx, cancel := context.WithTimeout(ctx, livechatRunTimeout)
defer cancel()
// Preview is search-only: no custom tools (empty allowed set), no built-in, no side effects.
@@ -553,11 +556,26 @@ func (m *Manager) PreviewReply(ctx context.Context, assistantID int, message str
if err != nil {
return "", nil, err
}
answer, articleIDs := splitArticleSources(answer, hits, a.CitationsEnabled)
main, confirm := splitConfirmation(strings.TrimSpace(answer))
if confirm != "" {
main += "\n\n" + confirm
}
return main, m.previewSources(hits), nil
sources := m.previewSources(hits)
references, err := m.convo.GetArticleReferences(articleIDs)
if err != nil {
m.lo.Error("error loading preview article references", "error", err)
}
urlByID := make(map[int]string, len(references))
for _, reference := range references {
urlByID[reference.ID] = reference.URL
}
for i := range sources {
if sources[i].Type == aimodels.SourceHelpArticle {
sources[i].URL = urlByID[sources[i].ID]
}
}
return main, sources, nil
}
// previewSources dedupes search hits by source, keeping each one's best score. Snippets and help
+58
View File
@@ -2,7 +2,11 @@ package aiagent
import (
"reflect"
"strings"
"testing"
aimodels "github.com/abhinavxd/libredesk/internal/ai/models"
"github.com/abhinavxd/libredesk/internal/aiagent/models"
)
func TestSplitSuggestions(t *testing.T) {
@@ -22,3 +26,57 @@ func TestSplitSuggestionsRejectsInvalidPayload(t *testing.T) {
t.Fatalf("unexpected result: %q %#v", body, suggestions)
}
}
func TestSplitArticleSources(t *testing.T) {
hits := []aimodels.SearchResult{
{SourceID: 12, SourceType: aimodels.SourceHelpArticle},
{SourceID: 34, SourceType: aimodels.SourceHelpArticle},
{SourceID: 34, SourceType: aimodels.SourceHelpArticle},
{SourceID: 56, SourceType: aimodels.SourceSnippet},
}
tests := []struct {
name string
answer string
enabled bool
body string
ids []int
}{
{"used articles", "Answer.\n[[sources]] [34,12]", true, "Answer.", []int{34, 12}},
{"duplicates and unknown IDs", "Answer.\n[[sources]] [12,12,0,-1,999,56,34]", true, "Answer.", []int{12, 34}},
{"multiple searches and lines", "Answer.\n[[sources]] [12]\n[[sources]] [12,34]", true, "Answer.", []int{12, 34}},
{"disabled", "Answer.\n[[sources]] [12]", false, "Answer.", nil},
{"invalid payload", "Answer.\n[[sources]] [12,", true, "Answer.", nil},
{"noninteger IDs", "Answer.\n[[sources]] [12,1.5]", true, "Answer.", nil},
{"empty list", "Answer.\n[[sources]] []", true, "Answer.", nil},
{"null", "Answer.\n[[sources]] null", true, "Answer.", nil},
{"no marker", "Hello!", true, "Hello!", nil},
{"inline marker", "Answer. [[sources]] [12]", true, "Answer.", []int{12}},
{"next line array", "Answer.\n[[sources]]\n[12,34]", true, "Answer.", []int{12, 34}},
{"multiline array", "Answer.\n[[sources]] [\n12,\n34\n]\nFollowup.", true, "Answer.\nFollowup.", []int{12, 34}},
{"same line confirmation", "Answer.\n[[sources]] [12] [[confirm]] Did that help?", true, "Answer.\n[[confirm]] Did that help?", []int{12}},
{"same line suggestions", "Answer.\n[[sources]] [12] [[suggestions]] [\"Yes\",\"No\"]", true, "Answer.\n[[suggestions]] [\"Yes\",\"No\"]", []int{12}},
{"invalid array with confirmation", "Answer.\n[[sources]] [12, [[confirm]] Did that help?", true, "Answer.\n[[confirm]] Did that help?", nil},
{"invalid payload with suggestions", "Answer.\n[[sources]] garbage [[suggestions]] [\"Yes\",\"No\"]", true, "Answer.\n[[suggestions]] [\"Yes\",\"No\"]", nil},
{"noninteger multiline array", "Answer.\n[[sources]]\n[12,1.5]\n[[confirm]] Did that help?", true, "Answer.\n[[confirm]] Did that help?", nil},
{"multiple markers on one line", "Answer.\n[[sources]] [12] [[sources]] [12,34] [[confirm]] Did that help?", true, "Answer.\n[[confirm]] Did that help?", []int{12, 34}},
{"disabled multiline", "Answer.\n[[sources]]\n[12,34] [[confirm]] Did that help?", false, "Answer.\n[[confirm]] Did that help?", nil},
{"with confirmation", "Answer.\n[[sources]] [12]\n[[confirm]]\nDid that help?\n[[suggestions]] [\"Yes\",\"No\"]", true, "Answer.\n[[confirm]]\nDid that help?\n[[suggestions]] [\"Yes\",\"No\"]", []int{12}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
body, ids := splitArticleSources(tt.answer, hits, tt.enabled)
if body != tt.body || !reflect.DeepEqual(ids, tt.ids) {
t.Fatalf("got %q, %v, want %q, %v", body, ids, tt.body, tt.ids)
}
})
}
}
func TestArticleCitationPrompt(t *testing.T) {
if strings.Contains(buildSystemPrompt(models.Assistant{}), "[[sources]]") {
t.Fatal("disabled assistant requests article references")
}
if !strings.Contains(buildSystemPrompt(models.Assistant{CitationsEnabled: true}), "[[sources]]") {
t.Fatal("enabled assistant does not request article references")
}
}
+148
View File
@@ -0,0 +1,148 @@
package conversation
import (
"encoding/json"
"fmt"
"html"
"strings"
"github.com/abhinavxd/libredesk/internal/conversation/models"
hcmodels "github.com/abhinavxd/libredesk/internal/helpcenter/models"
)
func (m *Manager) SetArticleReferenceStore(store articleReferenceStore) {
m.articleReferenceStore = store
}
func (m *Manager) GetArticleReferences(ids []int) ([]hcmodels.ArticleReference, error) {
if m.articleReferenceStore == nil || len(ids) == 0 {
return nil, nil
}
references, err := m.articleReferenceStore.GetArticleReferences(ids)
if err != nil {
return nil, err
}
rootURL, err := m.settingsStore.GetAppRootURL()
if err != nil {
return nil, err
}
resolved := make([]hcmodels.ArticleReference, 0, len(references))
for _, reference := range references {
reference.URL = reference.PublicURL(rootURL)
if reference.URL != "" {
resolved = append(resolved, reference)
}
}
return resolved, nil
}
func (m *Manager) RenderArticleReferences(message *models.Message) {
message.Content += m.chatArticleReferencesHTML(m.lookupArticleReferences(message.Meta)[0])
}
func (m *Manager) RenderMessagesArticleReferences(messages []models.Message) {
for i, references := range m.lookupArticleReferences(messageMetas(messages)...) {
messages[i].Content += m.chatArticleReferencesHTML(references)
}
}
func (m *Manager) chatArticleReferencesHTML(references []hcmodels.ArticleReference) string {
return articleReferencesHTML(references, m.i18n.Tc("globals.terms.articleReference", 2 /* n */))
}
func (m *Manager) emailArticleReferences(message *models.Message) string {
return emailArticleReferencesHTML(m.lookupArticleReferences(message.Meta)[0])
}
// lookupArticleReferences returns each message's references, resolved with one query for all of them.
func (m *Manager) lookupArticleReferences(metas ...json.RawMessage) [][]hcmodels.ArticleReference {
out := make([][]hcmodels.ArticleReference, len(metas))
idsByMessage := make([][]int, len(metas))
var ids []int
for i, meta := range metas {
idsByMessage[i] = messageArticleIDs(meta)
ids = append(ids, idsByMessage[i]...)
}
if len(ids) == 0 {
return out
}
references, err := m.GetArticleReferences(ids)
if err != nil {
m.lo.Error("error rendering article references", "error", err)
return out
}
byID := make(map[int]hcmodels.ArticleReference, len(references))
for _, reference := range references {
byID[reference.ID] = reference
}
for i, messageIDs := range idsByMessage {
for _, id := range messageIDs {
if reference, ok := byID[id]; ok {
out[i] = append(out[i], reference)
}
}
}
return out
}
func messageMetas(messages []models.Message) []json.RawMessage {
metas := make([]json.RawMessage, len(messages))
for i := range messages {
metas[i] = messages[i].Meta
}
return metas
}
func messageArticleIDs(meta json.RawMessage) []int {
var data struct {
AssistantID int `json:"ai_assistant_id"`
ArticleIDs []int `json:"ai_article_ids"`
}
if json.Unmarshal(meta, &data) != nil || data.AssistantID <= 0 {
return nil
}
ids := make([]int, 0, len(data.ArticleIDs))
seen := map[int]bool{}
for _, id := range data.ArticleIDs {
if id > 0 && !seen[id] {
ids = append(ids, id)
seen[id] = true
}
}
return ids
}
func articleReferencesHTML(references []hcmodels.ArticleReference, label string) string {
if len(references) == 0 {
return ""
}
var b strings.Builder
b.WriteString("<p>")
b.WriteString(html.EscapeString(label))
b.WriteString("</p><ul>")
for _, reference := range references {
b.WriteString(`<li><a href="`)
b.WriteString(html.EscapeString(reference.URL))
b.WriteString(`" target="_blank" rel="noopener noreferrer">`)
b.WriteString(html.EscapeString(reference.Title))
b.WriteString("</a></li>")
}
b.WriteString("</ul>")
return b.String()
}
func emailArticleReferencesHTML(references []hcmodels.ArticleReference) string {
if len(references) == 0 {
return ""
}
var b strings.Builder
b.WriteString("<p>")
for i, reference := range references {
if i > 0 {
b.WriteByte(' ')
}
fmt.Fprintf(&b, `<a href="%s" target="_blank" rel="noopener noreferrer">(%d)</a>`, html.EscapeString(reference.URL), i+1)
}
b.WriteString("</p>")
return b.String()
}
+22 -2
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"html"
"slices"
"strings"
"time"
@@ -202,7 +203,8 @@ func (m *Manager) sendContinuityEmail(conv models.ContinuityConversation, maxMes
// Build References and In-Reply-To headers for email threading.
references, inReplyTo := m.BuildEmailThreadingHeaders(conv.ID, sourceID)
// Render message template
content, hasReferences := m.renderContinuityEmailContent(unreadMessages, websiteURL)
message.Content = content
if err := m.RenderMessageInTemplate(linkedEmailInbox.Channel(), &message); err != nil {
// Clean up the inserted message on failure
cleanUp = true
@@ -241,6 +243,9 @@ func (m *Manager) sendContinuityEmail(conv models.ContinuityConversation, maxMes
Meta: message.Meta,
CreatedAt: message.CreatedAt,
}
if hasReferences {
outbound.AltContent = stringutil.HTML2TextMarkdownLinks(message.Content)
}
// Send the email
if err := linkedEmailInbox.Send(outbound); err != nil {
@@ -270,7 +275,22 @@ func (m *Manager) sendContinuityEmail(conv models.ContinuityConversation, maxMes
return nil
}
// buildContinuityEmailContent creates email content with conversation summary and unread messages
func (m *Manager) renderContinuityEmailContent(unreadMessages []models.ContinuityUnreadMessage, websiteURL string) (string, bool) {
messages := slices.Clone(unreadMessages)
metas := make([]json.RawMessage, len(messages))
for i := range messages {
metas[i] = messages[i].Meta
}
hasReferences := false
for i, references := range m.lookupArticleReferences(metas...) {
if len(references) > 0 {
messages[i].Content += emailArticleReferencesHTML(references)
hasReferences = true
}
}
return m.buildContinuityEmailContent(messages, websiteURL), hasReferences
}
func (m *Manager) buildContinuityEmailContent(unreadMessages []models.ContinuityUnreadMessage, websiteURL string) string {
var content strings.Builder
+8
View File
@@ -29,6 +29,7 @@ import (
csatModels "github.com/abhinavxd/libredesk/internal/csat/models"
"github.com/abhinavxd/libredesk/internal/dbutil"
"github.com/abhinavxd/libredesk/internal/envelope"
hcmodels "github.com/abhinavxd/libredesk/internal/helpcenter/models"
"github.com/abhinavxd/libredesk/internal/inbox"
imodels "github.com/abhinavxd/libredesk/internal/inbox/models"
mmodels "github.com/abhinavxd/libredesk/internal/media/models"
@@ -91,6 +92,10 @@ type notificationDispatcher interface {
Send(nmodels.Notification) ([]nmodels.DeliveryResult, error)
}
type articleReferenceStore interface {
GetArticleReferences([]int) ([]hcmodels.ArticleReference, error)
}
// Manager handles the operations related to conversations
type Manager struct {
q queries
@@ -101,6 +106,7 @@ type Manager struct {
statusStore statusStore
priorityStore priorityStore
slaStore slaStore
articleReferenceStore articleReferenceStore
settingsStore settingsStore
csatStore csatStore
webhookStore webhookStore
@@ -556,6 +562,7 @@ func (c *Manager) GetContactUnreadPreviewMessages(contactID, inboxID, limit int)
return nil, envelope.NewError(envelope.GeneralError, c.i18n.T("globals.messages.somethingWentWrong"), nil)
}
c.RenderMessagesArticleReferences(messages)
previews := make([]models.ChatMessage, 0, len(messages))
for _, message := range messages {
c.SignAvatarURL(&message.Author.AvatarURL)
@@ -2201,6 +2208,7 @@ func (m *Manager) BuildWidgetConversationResponse(conversation models.Conversati
m.ProcessCSATStatus(messages)
// Generate signed URLs for all attachments.
m.RenderMessagesArticleReferences(messages)
chatMessages := make([]models.ChatMessage, 0, len(messages))
for _, msg := range messages {
m.SignAvatarURL(&msg.Author.AvatarURL)
+26 -12
View File
@@ -188,6 +188,9 @@ func (m *Manager) sendOutgoingMessage(message models.Message) {
return
}
outbound.From = m.emailFromAddress(inb, message, selected)
if len(messageArticleIDs(message.Meta)) > 0 {
outbound.AltContent = stringutil.HTML2TextMarkdownLinks(message.Content)
}
// Set "In-Reply-To" and "References" headers for email threading.
outbound.References, outbound.InReplyTo = m.BuildEmailThreadingHeaders(message.ConversationID, outbound.SourceID)
@@ -304,7 +307,6 @@ func (m *Manager) BuildTemplateData(conversationUUID string, senderID int) (map[
return data, nil
}
// RenderMessageInTemplate renders message content in the email base template for sending.
func (m *Manager) RenderMessageInTemplate(channel string, message *models.Message) error {
switch channel {
case inbox.ChannelEmail:
@@ -313,22 +315,14 @@ func (m *Manager) RenderMessageInTemplate(channel string, message *models.Messag
return err
}
// Expose message meta flags to the template.
var (
isContinuity bool
meta map[string]any
)
if len(message.Meta) > 0 && json.Unmarshal(message.Meta, &meta) == nil {
isContinuity, _ = meta["continuity_email"].(bool)
}
data["IsContinuityEmail"] = isContinuity
message.Content, err = m.template.RenderEmailWithTemplate(data, message.Content)
content := m.emailTemplateContent(message, data)
message.Content, err = m.template.RenderEmailWithTemplate(data, content)
if err != nil {
m.lo.Error("could not render email content using template", "id", message.ID, "error", err)
return fmt.Errorf("could not render email content using template: %w", err)
}
case inbox.ChannelLiveChat, inbox.ChannelWhatsApp:
m.RenderArticleReferences(message)
return nil
default:
m.lo.Warn("unknown message channel", "channel", channel)
@@ -337,6 +331,26 @@ func (m *Manager) RenderMessageInTemplate(channel string, message *models.Messag
return nil
}
func (m *Manager) emailTemplateContent(message *models.Message, data map[string]any) string {
var (
isContinuity bool
meta map[string]any
)
if len(message.Meta) > 0 && json.Unmarshal(message.Meta, &meta) == nil {
isContinuity, _ = meta["continuity_email"].(bool)
}
data["IsContinuityEmail"] = isContinuity
if isContinuity {
data["ContinuityContent"] = message.Content
return "{{ .ContinuityContent }}"
}
if references := m.emailArticleReferences(message); references != "" {
data["ArticleReferences"] = references
return message.Content + "{{ .ArticleReferences }}"
}
return message.Content
}
// GetConversationMessages retrieves messages for a specific conversation.
func (m *Manager) GetConversationMessages(conversationUUID string, page, pageSize int, private *bool, msgTypes []string) ([]models.Message, int, error) {
var (
+297
View File
@@ -1,11 +1,20 @@
package conversation
import (
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"text/template"
"time"
"github.com/abhinavxd/libredesk/internal/conversation/models"
hcmodels "github.com/abhinavxd/libredesk/internal/helpcenter/models"
"github.com/abhinavxd/libredesk/internal/inbox"
"github.com/abhinavxd/libredesk/internal/stringutil"
"github.com/abhinavxd/libredesk/internal/testutil"
"github.com/zerodha/logf"
)
const testUUID = "d0355103-455f-4c7d-b9c7-86e9254fe119"
@@ -21,6 +30,32 @@ func (f fromTestInbox) FromAddress() string { return f.from }
func (f fromTestInbox) PrimaryAddress() string { return f.primary }
func (f fromTestInbox) FromNameTemplate() string { return "" }
type citationArticleStore struct {
references []hcmodels.ArticleReference
err error
lookups *int
}
type citationSettingsStore struct {
settingsStore
rootURL string
lookups *int
}
func (s citationArticleStore) GetArticleReferences(ids []int) ([]hcmodels.ArticleReference, error) {
if s.lookups != nil {
(*s.lookups)++
}
return s.references, s.err
}
func (s citationSettingsStore) GetAppRootURL() (string, error) {
if s.lookups != nil {
(*s.lookups)++
}
return s.rootURL, nil
}
func TestImgSrcUploadsPattern(t *testing.T) {
tests := []struct {
name string
@@ -630,3 +665,265 @@ func TestEmailFromAddress(t *testing.T) {
})
}
}
func TestMessageArticleIDs(t *testing.T) {
tests := []struct {
meta string
want []int
}{
{`{"ai_assistant_id":1,"ai_article_ids":[12,12,0,-1,34]}`, []int{12, 34}},
{`{"ai_assistant_id":1,"ai_article_ids":[]}`, []int{}},
{`{"ai_article_ids":[12]}`, nil},
{`{"ai_assistant_id":0,"ai_article_ids":[12]}`, nil},
{`{"ai_assistant_id":1,"ai_article_ids":["12"]}`, nil},
{`not-json`, nil},
}
for _, tt := range tests {
if got := messageArticleIDs(json.RawMessage(tt.meta)); !reflect.DeepEqual(got, tt.want) {
t.Errorf("messageArticleIDs(%s) = %v, want %v", tt.meta, got, tt.want)
}
}
}
func TestRenderArticleReferences(t *testing.T) {
lo := logf.New(logf.Opts{})
m := Manager{
articleReferenceStore: citationArticleStore{references: []hcmodels.ArticleReference{
{ID: 12, Title: `Refunds <script> & "policy"`, HelpCenterSlug: "support", Locale: "en", Slug: "refunds"},
{ID: 34, Title: "Delivery", HelpCenterSlug: "support", Locale: "fr", Slug: "delivery", CustomDomain: "https://help.example.com"},
}},
settingsStore: citationSettingsStore{rootURL: "https://desk.example.com/"},
i18n: testutil.NewI18n(t),
lo: &lo,
}
meta := json.RawMessage(`{"ai_assistant_id":1,"ai_article_ids":[12,34]}`)
message := models.Message{Content: "<p>Answer.</p>", Meta: meta}
m.RenderArticleReferences(&message)
for _, part := range []string{"Article references", "https://desk.example.com/hc/support/en/articles/refunds", "https://help.example.com/fr/articles/delivery", "Refunds &lt;script&gt; &amp; &#34;policy&#34;"} {
if !strings.Contains(message.Content, part) {
t.Errorf("rendered references missing %q: %s", part, message.Content)
}
}
if strings.Contains(message.Content, "<script>") {
t.Fatal("article title inserted unescaped HTML")
}
plain := stringutil.HTML2TextMarkdownLinks(message.Content)
for _, url := range []string{"https://desk.example.com/hc/support/en/articles/refunds", "https://help.example.com/fr/articles/delivery"} {
if !strings.Contains(plain, url) {
t.Errorf("plain text email is missing %s", url)
}
}
if string(meta) != string(message.Meta) || strings.Contains(string(message.Meta), "https://") {
t.Fatal("message metadata changed or persisted an absolute URL")
}
m.settingsStore = citationSettingsStore{rootURL: "https://new.example.com"}
message.Content = "<p>Answer.</p>"
m.RenderArticleReferences(&message)
if strings.Contains(message.Content, "https://desk.example.com") || !strings.Contains(message.Content, "https://new.example.com/hc/support/en/articles/refunds") {
t.Fatal("references did not follow the current root URL")
}
for _, store := range []citationArticleStore{{}, {err: errors.New("lookup failed")}} {
m.articleReferenceStore = store
message.Content = "<p>Answer.</p>"
m.RenderArticleReferences(&message)
if message.Content != "<p>Answer.</p>" {
t.Fatal("unavailable references changed the answer")
}
}
}
func TestArticleReferencePublicURL(t *testing.T) {
reference := hcmodels.ArticleReference{HelpCenterSlug: "support", Locale: "en", Slug: "refund policy"}
for _, tt := range []struct{ root, custom, want string }{
{"https://desk.example.com/", "", "https://desk.example.com/hc/support/en/articles/refund%20policy"},
{"https://desk.example.com", "https://help.example.com/path", "https://help.example.com/en/articles/refund%20policy"},
{"", "https://help.example.com", "https://help.example.com/en/articles/refund%20policy"},
{"", "", ""},
{"javascript:alert(1)", "", ""},
} {
reference.CustomDomain = tt.custom
if got := reference.PublicURL(tt.root); got != tt.want {
t.Errorf("PublicURL(%q, %q) = %q, want %q", tt.root, tt.custom, got, tt.want)
}
}
}
func TestContinuityEmailArticleReferences(t *testing.T) {
lo := logf.New(logf.Opts{})
m := Manager{
articleReferenceStore: citationArticleStore{references: []hcmodels.ArticleReference{
{ID: 12, Title: "Refund {{name}}", HelpCenterSlug: "support", Locale: "en", Slug: "refunds"},
}},
settingsStore: citationSettingsStore{rootURL: "https://desk.example.com"},
i18n: testutil.NewI18n(t),
lo: &lo,
}
messages := []models.ContinuityUnreadMessage{
{Message: models.Message{Content: "<p>AI answer.</p>", Meta: json.RawMessage(`{"ai_assistant_id":1,"ai_article_ids":[12,12,0,-1]}`)}},
{Message: models.Message{Content: "<p>Human answer. {{name}} {{ .Author.FirstName }} {{</p>"}},
}
stored := m.buildContinuityEmailContent(messages, "" /* websiteURL */)
rendered, _ := m.renderContinuityEmailContent(messages, "" /* websiteURL */)
wantURL := "https://desk.example.com/hc/support/en/articles/refunds"
if strings.Count(rendered, wantURL) != 1 || !strings.Contains(rendered, `target="_blank"`) {
t.Fatalf("offline email is missing article links: %s", rendered)
}
if !strings.Contains(rendered, ">(1)</a>") || strings.Contains(rendered, "Refund {{name}}") || strings.Contains(rendered, "Article references") {
t.Fatalf("offline email should contain numbered links: %s", rendered)
}
message := models.Message{Content: rendered, Meta: json.RawMessage(`{"continuity_email":true}`)}
data := map[string]any{"Author": map[string]any{"FirstName": "Agent"}}
content := m.emailTemplateContent(&message, data)
tmpl, err := template.New("content").Parse(content)
if err != nil {
t.Fatalf("parsing offline email: %v", err)
}
var output strings.Builder
if err := tmpl.Execute(&output, data); err != nil {
t.Fatalf("rendering offline email: %v", err)
}
if output.String() != rendered || data["IsContinuityEmail"] != true {
t.Fatal("offline email changed literal message content or lost its template flag")
}
if strings.Index(rendered, wantURL) > strings.Index(rendered, "Human answer.") {
t.Fatal("article reference is attached to the wrong reply")
}
if !strings.Contains(stringutil.HTML2TextMarkdownLinks(rendered), wantURL) {
t.Fatal("plain text offline email is missing the article URL")
}
if stored != m.buildContinuityEmailContent(messages, "" /* websiteURL */) || strings.Contains(stored, wantURL) {
t.Fatal("rendering persisted an article URL in the saved email")
}
if again, _ := m.renderContinuityEmailContent(messages, "" /* websiteURL */); again != rendered {
t.Fatal("rendering the offline email twice duplicated article references")
}
for _, store := range []citationArticleStore{{}, {err: errors.New("lookup failed")}} {
m.articleReferenceStore = store
if got, _ := m.renderContinuityEmailContent(messages, "" /* websiteURL */); got != stored {
t.Fatal("unavailable references changed the offline email")
}
}
}
func TestEmailTemplateArticleReferences(t *testing.T) {
lo := logf.New(logf.Opts{})
for _, title := range []string{"Refund {{name}}", "Refund {{ .Author.FirstName }}", "Refund {{", `<script> & "refunds"`} {
t.Run(title, func(t *testing.T) {
m := Manager{
articleReferenceStore: citationArticleStore{references: []hcmodels.ArticleReference{
{ID: 12, Title: title, HelpCenterSlug: "support", Locale: "en", Slug: "refunds"},
{ID: 34, Title: "Delivery", HelpCenterSlug: "support", Locale: "fr", Slug: "delivery", CustomDomain: "https://help.example.com"},
}},
settingsStore: citationSettingsStore{rootURL: "https://desk.example.com"},
lo: &lo,
}
message := models.Message{
Content: "<p>Hello {{ .Author.FirstName }}.</p>",
Meta: json.RawMessage(`{"ai_assistant_id":1,"ai_article_ids":[12,34]}`),
}
data := map[string]any{"Author": map[string]any{"FirstName": "Agent"}}
content := m.emailTemplateContent(&message, data)
tmpl, err := template.New("content").Parse(content)
if err != nil {
t.Fatalf("parsing email: %v", err)
}
var output strings.Builder
if err := tmpl.Execute(&output, data); err != nil {
t.Fatalf("rendering email: %v", err)
}
want := `<p>Hello Agent.</p><p><a href="https://desk.example.com/hc/support/en/articles/refunds" target="_blank" rel="noopener noreferrer">(1)</a> <a href="https://help.example.com/fr/articles/delivery" target="_blank" rel="noopener noreferrer">(2)</a></p>`
if output.String() != want || data["IsContinuityEmail"] != false {
t.Fatalf("unexpected email content: %s", output.String())
}
plain := stringutil.HTML2TextMarkdownLinks(output.String())
for _, part := range []string{"(1)", "(2)", "https://desk.example.com/hc/support/en/articles/refunds", "https://help.example.com/fr/articles/delivery"} {
if !strings.Contains(plain, part) {
t.Errorf("plain text email is missing %q: %s", part, plain)
}
}
if again := m.emailTemplateContent(&message, data); again != content {
t.Fatal("preparing the email twice duplicated article references")
}
for _, store := range []citationArticleStore{{}, {err: errors.New("lookup failed")}} {
m.articleReferenceStore = store
if got := m.emailTemplateContent(&message, data); got != message.Content {
t.Fatal("unavailable references changed the email content")
}
}
})
}
}
func TestArticleReferenceBatchLookups(t *testing.T) {
renderers := []struct {
name string
render func(*Manager, []models.Message) string
}{
{"messages", func(m *Manager, messages []models.Message) string {
m.RenderMessagesArticleReferences(messages)
var output strings.Builder
for _, message := range messages {
output.WriteString(message.Content)
}
return output.String()
}},
{"transcript", func(m *Manager, messages []models.Message) string {
return string(m.BuildTranscript(models.Conversation{}, messages, time.Time{} /* downloadedAt */))
}},
{"offline email", func(m *Manager, messages []models.Message) string {
unread := make([]models.ContinuityUnreadMessage, len(messages))
for i, message := range messages {
unread[i].Message = message
}
content, _ := m.renderContinuityEmailContent(unread, "" /* websiteURL */)
return content
}},
}
for _, renderer := range renderers {
t.Run(renderer.name, func(t *testing.T) {
lo := logf.New(logf.Opts{})
articleLookups, settingsLookups := 0, 0
m := Manager{
articleReferenceStore: citationArticleStore{
references: []hcmodels.ArticleReference{
{ID: 12, Title: "Refund policy", HelpCenterSlug: "support", Locale: "en", Slug: "refunds"},
{ID: 34, Title: "Delivery", HelpCenterSlug: "support", Locale: "en", Slug: "delivery"},
},
lookups: &articleLookups,
},
settingsStore: citationSettingsStore{rootURL: "https://desk.example.com", lookups: &settingsLookups},
i18n: testutil.NewI18n(t),
lo: &lo,
}
messages := make([]models.Message, 151)
for i := range 150 {
messages[i] = models.Message{
Content: "<p>AI answer.</p>",
Meta: json.RawMessage(`{"ai_assistant_id":1,"ai_article_ids":[34,12,34,0,-1,999]}`),
}
}
messages[150].Content = "<p>Human answer.</p>"
output := renderer.render(&m, messages)
if articleLookups != 1 || settingsLookups != 1 {
t.Fatalf("150 replies made %d article lookups and %d settings lookups", articleLookups, settingsLookups)
}
for _, slug := range []string{"delivery", "refunds"} {
if count := strings.Count(output, "https://desk.example.com/hc/support/en/articles/"+slug); count != 150 {
t.Errorf("article %s appeared %d times, want 150", slug, count)
}
}
if strings.Index(output, "/articles/delivery") > strings.Index(output, "/articles/refunds") {
t.Fatal("batch lookup changed citation order")
}
if strings.Contains(messages[150].Content, "href=") {
t.Fatal("uncited reply gained article links")
}
articleLookups, settingsLookups = 0, 0
renderer.render(&m, []models.Message{{Content: "<p>Human answer.</p>"}})
renderer.render(&m, nil /* messages */)
if articleLookups != 0 || settingsLookups != 0 {
t.Fatal("uncited or empty batches performed reference lookups")
}
})
}
}
+6 -1
View File
@@ -27,11 +27,16 @@ func (m *Manager) BuildTranscript(conversation models.Conversation, messages []m
fmt.Fprintf(&b, "%s: %s\n\n", m.i18n.T("globals.terms.downloadedAt"), downloadedAt.UTC().Format(transcriptTimeFormat))
b.WriteString(transcriptSeparator + "\n")
for _, message := range messages {
label := m.i18n.Tc("globals.terms.articleReference", 2 /* n */)
references := m.lookupArticleReferences(messageMetas(messages)...)
for i, message := range messages {
content := message.TextContent
if content == "" {
content = stringutil.HTML2Text(message.Content)
}
if len(references[i]) > 0 {
content += "\n\n" + stringutil.HTML2TextMarkdownLinks(articleReferencesHTML(references[i], label))
}
fmt.Fprintf(&b, "\n[%s] %s (%s):\n%s\n",
message.CreatedAt.UTC().Format(transcriptTimeFormat),
transcriptAuthor(message),
+28
View File
@@ -1,12 +1,14 @@
package conversation
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/abhinavxd/libredesk/internal/attachment"
"github.com/abhinavxd/libredesk/internal/conversation/models"
hcmodels "github.com/abhinavxd/libredesk/internal/helpcenter/models"
"github.com/abhinavxd/libredesk/internal/testutil"
"github.com/volatiletech/null/v9"
)
@@ -70,3 +72,29 @@ func TestBuildTranscript(t *testing.T) {
}
}
}
func TestBuildTranscriptArticleReferences(t *testing.T) {
m := Manager{
articleReferenceStore: citationArticleStore{references: []hcmodels.ArticleReference{
{ID: 12, Title: "Refund policy", HelpCenterSlug: "support", Locale: "en", Slug: "refunds"},
}},
settingsStore: citationSettingsStore{rootURL: "https://desk.example.com"},
i18n: testutil.NewI18n(t),
}
for _, text := range []string{"Answer.", ""} {
messages := []models.Message{{
Content: "<p>Answer.</p>",
TextContent: text,
Meta: json.RawMessage(`{"ai_assistant_id":1,"ai_article_ids":[12]}`),
}}
out := string(m.BuildTranscript(models.Conversation{}, messages, time.Now() /* downloadedAt */))
for _, want := range []string{"Answer.", "Article references", "Refund policy", "https://desk.example.com/hc/support/en/articles/refunds"} {
if !strings.Contains(out, want) {
t.Errorf("transcript missing %q: %s", want, out)
}
}
if strings.Contains(messages[0].Content, "https://") {
t.Fatal("transcript rendering changed the saved reply")
}
}
}
+14
View File
@@ -22,6 +22,7 @@ import (
"github.com/abhinavxd/libredesk/internal/stringutil"
"github.com/jmoiron/sqlx"
"github.com/knadh/go-i18n"
"github.com/lib/pq"
"github.com/microcosm-cc/bluemonday"
"github.com/volatiletech/null/v9"
"github.com/zerodha/logf"
@@ -204,6 +205,7 @@ type queries struct {
GetHelpCenterTreeData *sqlx.Stmt `query:"get-help-center-tree-data"`
GetPublicTreeData *sqlx.Stmt `query:"get-public-tree-data"`
GetArticleReferences *sqlx.Stmt `query:"get-article-references"`
GetPublishedArticleBySlug *sqlx.Stmt `query:"get-published-article-by-slug"`
GetPublishedArticleTranslations *sqlx.Stmt `query:"get-published-article-translations"`
GetPublishedCollectionLocales *sqlx.Stmt `query:"get-published-collection-locales"`
@@ -1004,6 +1006,18 @@ func (m *Manager) GetPublicTree(helpCenter models.HelpCenter, locale string) (mo
return models.TreeResponse{HelpCenter: helpCenter, Tree: tree}, nil
}
func (m *Manager) GetArticleReferences(ids []int) ([]models.ArticleReference, error) {
references := []models.ArticleReference{}
if len(ids) == 0 {
return references, nil
}
if err := m.q.GetArticleReferences.Select(&references, pq.Array(ids)); err != nil {
m.lo.Error("error fetching article references", "error", err)
return nil, envelope.NewError(envelope.GeneralError, m.i18n.T("globals.messages.somethingWentWrong"), nil)
}
return references, nil
}
// GetPublishedArticle retrieves a published article by help center slug and article slug, restricted to locale (empty = any).
func (m *Manager) GetPublishedArticle(helpCenterSlug, articleSlug, locale string) (models.Article, error) {
var article models.Article
+26
View File
@@ -2,6 +2,8 @@ package models
import (
"encoding/json"
"net/url"
"strings"
"time"
"github.com/volatiletech/null/v9"
@@ -19,6 +21,16 @@ const (
ColorSchemeDark = "dark"
)
type ArticleReference struct {
ID int `db:"id" json:"id"`
Title string `db:"title" json:"title"`
Slug string `db:"slug" json:"-"`
Locale string `db:"locale" json:"-"`
HelpCenterSlug string `db:"help_center_slug" json:"-"`
CustomDomain string `db:"custom_domain" json:"-"`
URL string `db:"-" json:"url"`
}
type HelpCenter struct {
ID int `db:"id" json:"id"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
@@ -220,6 +232,20 @@ type Insights struct {
NoResultSearch []SearchTermStat `json:"no_result_searches"`
}
func (r ArticleReference) PublicURL(rootURL string) string {
origin := strings.TrimRight(rootURL, "/")
prefix := "/hc/" + url.PathEscape(r.HelpCenterSlug)
if u, err := url.Parse(r.CustomDomain); err == nil && u.Host != "" && (u.Scheme == "https" || u.Scheme == "http") {
origin = u.Scheme + "://" + u.Host
prefix = ""
}
u, err := url.Parse(origin)
if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") {
return ""
}
return origin + prefix + "/" + url.PathEscape(r.Locale) + "/articles/" + url.PathEscape(r.Slug)
}
// DefaultTheme enables the show-flags for elements that must keep rendering when a stored theme predates the flag.
func DefaultTheme() Theme {
return Theme{
+21
View File
@@ -529,3 +529,24 @@ LIMIT $2;
-- name: delete-stale-search-queries
DELETE FROM help_search_queries
WHERE created_at < NOW() - ($1 * INTERVAL '1 day');
-- name: get-article-references
WITH RECURSIVE article_ancestors AS (
SELECT a.id AS article_id, c.id, c.parent_id, c.help_center_id, c.is_published
FROM help_articles a
JOIN article_collections c ON c.id = a.collection_id AND c.locale = a.locale
WHERE a.id = ANY($1) AND a.status = 'published' AND a.ai_enabled = true
UNION
SELECT p.article_id, c.id, c.parent_id, c.help_center_id, c.is_published
FROM article_collections c
JOIN article_ancestors p ON c.id = p.parent_id AND c.help_center_id = p.help_center_id
)
SELECT a.id, a.title, a.slug, a.locale, hc.slug AS help_center_slug, hc.custom_domain
FROM help_articles a
JOIN article_collections c ON c.id = a.collection_id
JOIN help_centers hc ON hc.id = c.help_center_id
WHERE a.id = ANY($1) AND a.status = 'published' AND a.ai_enabled = true AND hc.is_active = true
AND (a.locale = hc.default_locale OR hc.allowed_locales ? a.locale)
AND EXISTS (SELECT 1 FROM article_ancestors p WHERE p.article_id = a.id AND p.parent_id IS NULL)
AND NOT EXISTS (SELECT 1 FROM article_ancestors p WHERE p.article_id = a.id AND p.is_published = false)
ORDER BY array_position($1::integer[], a.id);
+3
View File
@@ -7,6 +7,9 @@ import (
)
func V2_9_0_RC9(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
if _, err := db.Exec(`ALTER TABLE ai_assistants ADD COLUMN IF NOT EXISTS citations_enabled BOOLEAN NOT NULL DEFAULT false;`); err != nil {
return err
}
if _, err := db.Exec(`
ALTER TABLE media ADD COLUMN IF NOT EXISTS uploaded_by BIGINT REFERENCES users(id) ON DELETE SET NULL;
CREATE INDEX IF NOT EXISTS index_media_on_uploaded_by ON media(uploaded_by) WHERE uploaded_by IS NOT NULL;
+1 -1
View File
@@ -24,7 +24,7 @@ var notificationTypesV2_9_0 = []string{
"automation",
}
func V2_9_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
func V2_9_0_RC8(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf) error {
_, err := db.Exec(`ALTER TYPE channels ADD VALUE IF NOT EXISTS 'whatsapp';`)
if err != nil {
return err
+6 -6
View File
@@ -18,7 +18,7 @@ func TestV2_9_0TemplateComponentsMigration(t *testing.T) {
}
db.MustExec(`ALTER TABLE whatsapp_templates DROP COLUMN component_types`)
for range 2 {
if err := V2_9_0(db, nil, nil); err != nil {
if err := V2_9_0_RC8(db, nil, nil); err != nil {
t.Fatal(err)
}
}
@@ -51,7 +51,7 @@ func TestV2_9_0PrivateNotePermissionMigration(t *testing.T) {
// Running the migration twice verifies that it does not append duplicates.
for range 2 {
if err := V2_9_0(db, nil, nil); err != nil {
if err := V2_9_0_RC8(db, nil, nil); err != nil {
t.Fatalf("running migration: %v", err)
}
}
@@ -95,7 +95,7 @@ func TestV2_9_0NotificationMigration(t *testing.T) {
`)
for range 2 {
if err := V2_9_0(db, nil, nil); err != nil {
if err := V2_9_0_RC8(db, nil, nil); err != nil {
t.Fatalf("running migration: %v", err)
}
}
@@ -160,7 +160,7 @@ func TestV2_9_0PreservesExistingQueuedEmails(t *testing.T) {
db.MustExec(`INSERT INTO users (type, email, first_name, last_name) VALUES ('agent', 'queued@example.com', 'Agent', '')`)
db.MustExec(`INSERT INTO notification_email_queue (user_id, notification_type, recipient_email, subject, content, send_at) VALUES ((SELECT id FROM users LIMIT 1), 'new_reply', 'queued@example.com', 'Reply', 'Pending reply', now())`)
for range 2 {
if err := V2_9_0(db, nil, nil); err != nil {
if err := V2_9_0_RC8(db, nil, nil); err != nil {
t.Fatal(err)
}
}
@@ -184,7 +184,7 @@ func TestWidgetCampaignMigration(t *testing.T) {
db := testutil.NewDB(t, "widget_migration")
db.MustExec(`DROP TABLE widget_campaign_deliveries`)
for range 2 {
if err := V2_9_0(db, nil, nil); err != nil {
if err := V2_9_0_RC8(db, nil, nil); err != nil {
t.Fatal(err)
}
}
@@ -211,7 +211,7 @@ func TestHelpArticleTranslationGroupMigration(t *testing.T) {
`)
for range 2 {
if err := V2_9_0(db, nil, nil); err != nil {
if err := V2_9_0_RC8(db, nil, nil); err != nil {
t.Fatal(err)
}
}
+1
View File
@@ -851,6 +851,7 @@ CREATE TABLE ai_assistants (
max_turns INTEGER NOT NULL DEFAULT 6,
fallback_team_id INTEGER NULL REFERENCES teams(id) ON DELETE SET NULL,
handoff_enabled BOOLEAN NOT NULL DEFAULT true,
citations_enabled BOOLEAN NOT NULL DEFAULT false,
languages TEXT[] NOT NULL DEFAULT '{}',
enabled BOOLEAN NOT NULL DEFAULT true,
CONSTRAINT constraint_ai_assistants_on_tone CHECK (tone IN ('friendly', 'professional', 'neutral', 'casual')),