From 8d99740a41dee31cb10667e5f18364a4bc08c464 Mon Sep 17 00:00:00 2001 From: Abhinav Raut Date: Fri, 18 Sep 2026 20:19:52 +0530 Subject: [PATCH 1/7] add custom AI tools to copilot and generate reply with an agent approval step --- cmd/ai.go | 149 +++++++++-- cmd/handlers.go | 2 + frontend/apps/main/src/api/index.js | 6 + .../main/src/features/admin/ai/ToolForm.vue | 67 ++++- .../src/features/conversation/ReplyBox.vue | 121 ++++++++- .../conversation/sidebar/CopilotPanel.vue | 133 +++++++++- frontend/shared-ui/components/SwitchField.vue | 10 +- i18n/en-US.json | 17 ++ internal/ai/agent.go | 78 +++--- internal/ai/agent_approval.go | 242 ++++++++++++++++++ internal/ai/agent_approval_test.go | 131 ++++++++++ internal/ai/ai.go | 15 +- internal/ai/copilot.go | 8 +- internal/ai/models/models.go | 48 +++- internal/ai/queries.sql | 20 +- internal/ai/tools.go | 8 +- internal/ai/toolstore.go | 22 +- internal/ai/toolstore_test.go | 2 +- internal/migrations/v2.9.0.go | 9 + schema.sql | 3 + 20 files changed, 967 insertions(+), 124 deletions(-) create mode 100644 internal/ai/agent_approval.go create mode 100644 internal/ai/agent_approval_test.go diff --git a/cmd/ai.go b/cmd/ai.go index 999ae50f..0e74be10 100644 --- a/cmd/ai.go +++ b/cmd/ai.go @@ -60,6 +60,11 @@ type snippetImportReq struct { URL string `json:"url"` } +type aiToolReq struct { + aimodels.Tool + RequiresAgentApproval *bool `json:"requires_agent_approval"` +} + // handleAICompletion runs a stored prompt over the supplied content (reply-box actions). func handleAICompletion(r *fastglue.Request) error { var ( @@ -214,12 +219,14 @@ func handleGetAITool(r *fastglue.Request) error { func handleCreateAITool(r *fastglue.Request) error { var ( app = r.Context.(*App) - req aimodels.Tool + req aiToolReq ) if err := r.Decode(&req, "json"); err != nil { return sendErrorEnvelope(r, envelope.NewError(envelope.InputError, app.i18n.T("errors.parsingRequest"), nil)) } - tool, err := app.ai.CreateTool(req) + toolInput := req.Tool + toolInput.RequiresAgentApproval = req.RequiresAgentApproval == nil || *req.RequiresAgentApproval + tool, err := app.ai.CreateTool(toolInput) if err != nil { return sendErrorEnvelope(r, err) } @@ -229,7 +236,7 @@ func handleCreateAITool(r *fastglue.Request) error { func handleUpdateAITool(r *fastglue.Request) error { var ( app = r.Context.(*App) - req aimodels.Tool + req aiToolReq ) id, err := strconv.Atoi(r.RequestCtx.UserValue("id").(string)) if err != nil { @@ -238,7 +245,17 @@ func handleUpdateAITool(r *fastglue.Request) error { if err := r.Decode(&req, "json"); err != nil { return sendErrorEnvelope(r, envelope.NewError(envelope.InputError, app.i18n.T("errors.parsingRequest"), nil)) } - tool, err := app.ai.UpdateTool(id, req) + toolInput := req.Tool + if req.RequiresAgentApproval == nil { + existing, err := app.ai.GetTool(id) + if err != nil { + return sendErrorEnvelope(r, err) + } + toolInput.RequiresAgentApproval = existing.RequiresAgentApproval + } else { + toolInput.RequiresAgentApproval = *req.RequiresAgentApproval + } + tool, err := app.ai.UpdateTool(id, toolInput) if err != nil { return sendErrorEnvelope(r, err) } @@ -352,14 +369,29 @@ func handleAIGenerateReply(r *fastglue.Request) error { } transcript = conversationTranscript(app, req.ConversationUUID) } - resp, err := app.ai.GenerateReply(r.RequestCtx, transcript, req.Instruction, ai.ToolContext{}, generateReplyTools(app, user, conv)) + toolIDs := []int(nil) + tctx := ai.ToolContext{Verified: func() bool { return true }} + scope := ai.AgentRunScope{AgentID: auser.ID, Surface: aimodels.ToolInvocationReply} + if conv != nil { + toolIDs, err = app.ai.GetEnabledGenerateReplyToolIDs() + if err != nil { + return sendErrorEnvelope(r, err) + } + tctx = agentToolContext(conv) + scope.ConversationID = conv.ID + scope.ConversationUUID = conv.UUID + } + resp, err := app.ai.GenerateReply(r.RequestCtx, transcript, req.Instruction, tctx, generateReplyTools(app, user, conv), toolIDs, scope) if err != nil { return sendErrorEnvelope(r, err) } - if strings.TrimSpace(resp) == "" { + if resp.Status == aimodels.AgentRunCompleted && strings.TrimSpace(resp.Content) == "" { return sendErrorEnvelope(r, envelope.NewError(envelope.GeneralError, app.i18n.T("ai.emptyResponse"), nil)) } - return r.SendEnvelope(stringutil.Markdown2HTML(resp)) + if resp.Status == aimodels.AgentRunCompleted { + resp.Content = stringutil.Markdown2HTML(resp.Content) + } + return r.SendEnvelope(resp) } // handleAISummarizeConversation summarizes a conversation and posts the summary as the requesting agent's private note. @@ -466,22 +498,48 @@ func handleAICopilot(r *fastglue.Request) error { } history = append(history, aimodels.ChatMessage{Role: aimodels.RoleUser, Content: req.Message}) - convoContext := conversationTranscript(app, req.ConversationUUID) - resp, err := app.ai.Copilot(r.RequestCtx, convoContext, history, ai.ToolContext{}, copilotTools(app, user, conv), persona) + scope := ai.AgentRunScope{AgentID: auser.ID, ConversationID: conv.ID, ConversationUUID: conv.UUID, Surface: aimodels.ToolInvocationCopilot, UserMessage: req.Message} + if _, approval := app.ai.PendingAgentApproval(scope); approval != nil { + return sendErrorEnvelope(r, envelope.NewError(envelope.ConflictError, app.i18n.T("ai.toolApprovalPending"), nil)) + } + toolIDs, err := app.ai.GetEnabledCopilotToolIDs() if err != nil { return sendErrorEnvelope(r, err) } - if strings.TrimSpace(resp) == "" { - return sendErrorEnvelope(r, envelope.NewError(envelope.GeneralError, app.i18n.T("ai.emptyResponse"), nil)) - } - // Persist the exchange only after a successful reply, so a failed or empty call leaves no orphaned turn. - if err := app.ai.SaveCopilotMessage(conv.ID, auser.ID, aimodels.RoleUser, req.Message); err != nil { + convoContext := conversationTranscript(app, req.ConversationUUID) + resp, err := app.ai.Copilot(r.RequestCtx, convoContext, history, agentToolContext(conv), copilotTools(app, user, conv), persona, toolIDs, scope) + if err != nil { return sendErrorEnvelope(r, err) } - if err := app.ai.SaveCopilotMessage(conv.ID, auser.ID, aimodels.RoleAssistant, resp); err != nil { - app.lo.Error("error saving copilot reply", "error", err) + if resp.Status == aimodels.AgentRunCompleted && strings.TrimSpace(resp.Content) == "" { + return sendErrorEnvelope(r, envelope.NewError(envelope.GeneralError, app.i18n.T("ai.emptyResponse"), nil)) } - return r.SendEnvelope(stringutil.Markdown2HTML(resp)) + if resp.Status == aimodels.AgentRunCompleted { + if err := app.ai.SaveCopilotMessage(conv.ID, auser.ID, aimodels.RoleUser, req.Message); err != nil { + return sendErrorEnvelope(r, err) + } + if err := app.ai.SaveCopilotMessage(conv.ID, auser.ID, aimodels.RoleAssistant, resp.Content); err != nil { + app.lo.Error("error saving copilot reply", "error", err) + } + resp.Content = stringutil.Markdown2HTML(resp.Content) + } + return r.SendEnvelope(resp) +} + +func handleApproveAIToolRun(r *fastglue.Request) error { + resp, err := decideAIToolRun(r, true) + if err != nil { + return sendErrorEnvelope(r, err) + } + return r.SendEnvelope(resp) +} + +func handleDeclineAIToolRun(r *fastglue.Request) error { + resp, err := decideAIToolRun(r, false) + if err != nil { + return sendErrorEnvelope(r, err) + } + return r.SendEnvelope(resp) } // handleGetCopilotMessages returns the requesting agent's persisted copilot chat for a conversation. @@ -506,6 +564,13 @@ func handleGetCopilotMessages(r *fastglue.Request) error { msgs[i].Content = stringutil.Markdown2HTML(msgs[i].Content) } } + scope := ai.AgentRunScope{AgentID: auser.ID, ConversationID: conv.ID, ConversationUUID: conv.UUID, Surface: aimodels.ToolInvocationCopilot} + if userMessage, approval := app.ai.PendingAgentApproval(scope); approval != nil { + if userMessage != "" { + msgs = append(msgs, aimodels.CopilotMessage{Role: aimodels.RoleUser, Content: userMessage}) + } + msgs = append(msgs, aimodels.CopilotMessage{Role: aimodels.RoleApproval, Approval: approval}) + } return r.SendEnvelope(msgs) } @@ -524,6 +589,7 @@ func handleClearCopilotMessages(r *fastglue.Request) error { if err := app.ai.ClearCopilotMessages(conv.ID, auser.ID); err != nil { return sendErrorEnvelope(r, err) } + app.ai.ClearPendingAgentRuns(ai.AgentRunScope{AgentID: auser.ID, ConversationID: conv.ID, ConversationUUID: conv.UUID, Surface: aimodels.ToolInvocationCopilot}) return r.SendEnvelope(true) } @@ -548,3 +614,52 @@ func conversationTranscript(app *App, uuid string) string { } return cmodels.Transcript(msgs, maxTranscriptMessages) } + +func decideAIToolRun(r *fastglue.Request, approved bool) (aimodels.AgentRunResult, error) { + app := r.Context.(*App) + auser := r.RequestCtx.UserValue("user").(amodels.User) + runID := r.RequestCtx.UserValue("id").(string) + scope, err := app.ai.PendingAgentRunScope(runID, auser.ID) + if err != nil { + return aimodels.AgentRunResult{}, err + } + if scope.ConversationUUID != "" { + conv, err := enforceAIConversationAccess(r, scope.ConversationUUID) + if err != nil { + return aimodels.AgentRunResult{}, err + } + if conv.ID != scope.ConversationID { + return aimodels.AgentRunResult{}, envelope.NewError(envelope.PermissionError, app.i18n.T("ai.toolApprovalUnavailable"), nil) + } + } + resp, resumedScope, err := app.ai.ResumeAgentRun(r.RequestCtx, runID, auser.ID, approved) + if err != nil { + return aimodels.AgentRunResult{}, err + } + if resp.Status == aimodels.AgentRunCompleted && strings.TrimSpace(resp.Content) == "" { + return aimodels.AgentRunResult{}, envelope.NewError(envelope.GeneralError, app.i18n.T("ai.emptyResponse"), nil) + } + if resp.Status == aimodels.AgentRunCompleted { + if resumedScope.Surface == aimodels.ToolInvocationCopilot { + if err := app.ai.SaveCopilotMessage(resumedScope.ConversationID, auser.ID, aimodels.RoleUser, resumedScope.UserMessage); err != nil { + app.lo.Error("error saving copilot message", "error", err) + } else if err := app.ai.SaveCopilotMessage(resumedScope.ConversationID, auser.ID, aimodels.RoleAssistant, resp.Content); err != nil { + app.lo.Error("error saving copilot reply", "error", err) + } + } + resp.Content = stringutil.Markdown2HTML(resp.Content) + } + return resp, nil +} + +func agentToolContext(conv *cmodels.Conversation) ai.ToolContext { + return ai.ToolContext{ + ContactID: conv.Contact.ID, + ContactExternalID: conv.Contact.ExternalUserID.String, + ContactType: conv.Contact.Type, + ConversationUUID: conv.UUID, + InboxID: conv.InboxID, + ContactEmail: func() string { return conv.Contact.Email.String }, + Verified: func() bool { return true }, + } +} diff --git a/cmd/handlers.go b/cmd/handlers.go index bd2fdc1e..b5daf2cd 100644 --- a/cmd/handlers.go +++ b/cmd/handlers.go @@ -282,6 +282,8 @@ func initHandlers(g *fastglue.Fastglue, hub *ws.Hub) { g.POST("/api/v1/ai/summarize", perm(handleAISummarizeConversation, "messages:write_private")) g.POST("/api/v1/ai/suggest-tags", auth(handleAISuggestTags)) g.POST("/api/v1/ai/copilot", auth(handleAICopilot)) + g.POST("/api/v1/ai/tool-runs/{id}/approve", auth(handleApproveAIToolRun)) + g.POST("/api/v1/ai/tool-runs/{id}/decline", auth(handleDeclineAIToolRun)) g.GET("/api/v1/ai/copilot/messages", auth(handleGetCopilotMessages)) g.DELETE("/api/v1/ai/copilot/messages", auth(handleClearCopilotMessages)) diff --git a/frontend/apps/main/src/api/index.js b/frontend/apps/main/src/api/index.js index bab5473b..ad900b3d 100644 --- a/frontend/apps/main/src/api/index.js +++ b/frontend/apps/main/src/api/index.js @@ -517,6 +517,10 @@ const aiGenerateReply = (data) => http.post('/api/v1/ai/generate-reply', data, { const aiSummarizeConversation = (data) => http.post('/api/v1/ai/summarize', data, { timeout: AI_TIMEOUT }) const aiSuggestTags = (data) => http.post('/api/v1/ai/suggest-tags', data, { timeout: AI_TIMEOUT }) const aiCopilot = (data) => http.post('/api/v1/ai/copilot', data, { timeout: AI_TIMEOUT }) +const approveAIToolRun = (id) => + http.post(`/api/v1/ai/tool-runs/${id}/approve`, {}, { timeout: AI_TIMEOUT }) +const declineAIToolRun = (id) => + http.post(`/api/v1/ai/tool-runs/${id}/decline`, {}, { timeout: AI_TIMEOUT }) const getCopilotMessages = (conversationUUID) => http.get('/api/v1/ai/copilot/messages', { params: { conversation_uuid: conversationUUID } }) const clearCopilotMessages = (conversationUUID) => @@ -818,6 +822,8 @@ export default { aiSummarizeConversation, aiSuggestTags, aiCopilot, + approveAIToolRun, + declineAIToolRun, getCopilotMessages, clearCopilotMessages, searchConversations, diff --git a/frontend/apps/main/src/features/admin/ai/ToolForm.vue b/frontend/apps/main/src/features/admin/ai/ToolForm.vue index 810bdfdc..41e6f64f 100644 --- a/frontend/apps/main/src/features/admin/ai/ToolForm.vue +++ b/frontend/apps/main/src/features/admin/ai/ToolForm.vue @@ -31,7 +31,11 @@ {{ t('globals.terms.url') }} - + @@ -119,6 +123,44 @@ +
+
+

{{ t('admin.ai.tool.agentAccess') }}

+
+ + + + + + + + + + + + + + + + + + +
+
+
+ + {{ $t('globals.terms.loading') }} +
@@ -83,23 +87,8 @@ aria-live="polite" :aria-label="$t('ai.toolApprovalTitle')" > -
-

{{ $t('ai.toolApprovalTitle') }}

-

- - - -

-
-
-

{{ $t('ai.toolApprovalArguments') }}

-
{{ formatToolArguments(msg.approval.arguments) }}
-
+

{{ $t('ai.toolApprovalTitle') }}

+
diff --git a/frontend/apps/main/src/features/conversation/ToolApprovalDetails.vue b/frontend/apps/main/src/features/conversation/ToolApprovalDetails.vue index 92c2ca1c..95f0f71a 100644 --- a/frontend/apps/main/src/features/conversation/ToolApprovalDetails.vue +++ b/frontend/apps/main/src/features/conversation/ToolApprovalDetails.vue @@ -11,7 +11,7 @@

-

{{ $t('ai.toolApprovalArguments') }}

+

{{ $t('globals.terms.arguments') }}

{{ formattedArguments }}
- {{ $t('ai.toolApprovalDecline') }} + {{ $t('globals.messages.decline') }}
diff --git a/i18n/en-US.json b/i18n/en-US.json index 36e6f29f..67f41ed7 100644 --- a/i18n/en-US.json +++ b/i18n/en-US.json @@ -188,7 +188,6 @@ "admin.ai.tool.turnOffVerificationEndpoint": "If the tool accesses sensitive data or performs actions, your endpoint must authenticate the customer itself. Every request carries an X-Libredesk-Contact-Verified header.", "admin.ai.tool.keepVerificationOn": "Keep authentication on", "admin.ai.tool.turnOffVerification": "Turn off", - "admin.ai.tool.agentAccess": "Agent access", "admin.ai.tool.availableInCopilot": "Available in Copilot", "admin.ai.tool.availableInGenerateReply": "Available in Generate Reply", "admin.ai.tool.requireAgentApproval": "Require agent approval", @@ -607,9 +606,6 @@ "ai.toolChangedBeforeApproval": "This tool changed after approval was requested and was not run. Run the AI action again to review the current tool.", "ai.toolApprovalTitle": "Review tool request", "ai.toolApprovalDescription": "The AI wants to run {tool}. Review the arguments before allowing it.", - "ai.toolApprovalArguments": "Arguments", - "ai.toolApprovalApprove": "Approve and run", - "ai.toolApprovalDecline": "Decline", "ai.toolRanResponseFailed": "The tool request finished, but the AI could not complete its response. Check the external system before running it again.", "ai.toolDeclinedResponseFailed": "The tool was declined, but the AI could not complete its response. Run the AI action again if you still need a reply.", "ai.toolApprovalPendingInput": "Approve or decline the pending tool request first", @@ -837,6 +833,7 @@ "globals.messages.additionalFeedback": "Additional feedback (optional)", "globals.messages.all": "All", "globals.messages.apply": "Apply", + "globals.messages.approveAndRun": "Approve and run", "globals.messages.areYouAbsolutelySure": "Are you absolutely sure?", "globals.messages.attachFile": "Attach file", "globals.messages.back": "Back", @@ -867,6 +864,7 @@ "globals.messages.delete": "Delete", "globals.messages.deleteAll": "Delete all", "globals.messages.deletedSuccessfully": "Deleted successfully", + "globals.messages.decline": "Decline", "globals.messages.disable": "Disable", "globals.messages.edit": "Edit", "globals.messages.empty": "{name} empty", @@ -966,10 +964,12 @@ "globals.terms.actor": "Actor | Actors", "globals.terms.admin": "Admin | Admins", "globals.terms.agent": "Agent | Agents", + "globals.terms.agentAccess": "Agent access", "globals.terms.alert": "Alert | Alerts", "globals.terms.announcement": "Announcement | Announcements", "globals.terms.apiKey": "API key | API keys", "globals.terms.appliesTo": "Applies to", + "globals.terms.arguments": "Arguments", "globals.terms.article": "Article | Articles", "globals.terms.ascending": "Ascending", "globals.terms.assignedTeam": "Assigned team", From a1117ac5fe543b0fe3f1c6b749c6d465ae634214 Mon Sep 17 00:00:00 2001 From: Abhinav Raut Date: Sat, 19 Sep 2026 01:34:43 +0530 Subject: [PATCH 7/7] send the AI agent's last reply before handing off to a human The handoff tool used to reassign the conversation right away, so the assistant's own reply was dropped and the customer saw nothing. It now records the intent, the reply goes out, then the handoff runs. --- cmd/ai.go | 3 +-- .../src/features/conversation/ReplyBox.vue | 2 +- .../conversation/sidebar/CopilotPanel.vue | 20 +++++++++---------- i18n/en-US.json | 1 - internal/aiagent/otp.go | 4 ++-- internal/aiagent/prompt.go | 4 ++-- internal/aiagent/tools.go | 7 ++++--- internal/aiagent/worker.go | 18 +++++++++-------- 8 files changed, 30 insertions(+), 29 deletions(-) diff --git a/cmd/ai.go b/cmd/ai.go index 3f60349f..ca7e7ddf 100644 --- a/cmd/ai.go +++ b/cmd/ai.go @@ -380,8 +380,6 @@ func handleAIGenerateReply(r *fastglue.Request) error { tctx = agentToolContext(conv) scope.ConversationID = conv.ID scope.ConversationUUID = conv.UUID - // The draft surface keeps no history, so a reload leaves the agent no way to answer an - // earlier request; the newest one replaces it. app.ai.ClearPendingAgentRuns(scope) } resp, err := app.ai.GenerateReply(r.RequestCtx, transcript, req.Instruction, tctx, generateReplyTools(app, user, conv), toolIDs, scope) @@ -502,6 +500,7 @@ func handleAICopilot(r *fastglue.Request) error { history = append(history, aimodels.ChatMessage{Role: aimodels.RoleUser, Content: req.Message}) scope := ai.AgentRunScope{AgentID: auser.ID, ConversationID: conv.ID, ConversationUUID: conv.UUID, Surface: aimodels.ToolInvocationCopilot, UserMessage: req.Message} + app.ai.ClearPendingAgentRuns(scope) toolIDs, err := app.ai.GetEnabledCopilotToolIDs() if err != nil { return sendErrorEnvelope(r, err) diff --git a/frontend/apps/main/src/features/conversation/ReplyBox.vue b/frontend/apps/main/src/features/conversation/ReplyBox.vue index 895ec343..5921fa35 100644 --- a/frontend/apps/main/src/features/conversation/ReplyBox.vue +++ b/frontend/apps/main/src/features/conversation/ReplyBox.vue @@ -350,7 +350,7 @@ const resolveGenerateToolApproval = async (approved) => { pendingToolConversationUUID.value = '' htmlContent.value = result.content || '' } catch (error) { - if ([404, 409].includes(error?.response?.status)) { + if ([403, 404, 409].includes(error?.response?.status)) { pendingToolApproval.value = null pendingToolConversationUUID.value = '' } diff --git a/frontend/apps/main/src/features/conversation/sidebar/CopilotPanel.vue b/frontend/apps/main/src/features/conversation/sidebar/CopilotPanel.vue index f481a97a..107bbdc6 100644 --- a/frontend/apps/main/src/features/conversation/sidebar/CopilotPanel.vue +++ b/frontend/apps/main/src/features/conversation/sidebar/CopilotPanel.vue @@ -186,10 +186,7 @@ >