mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 05:57:27 +00:00
c31445e2e1
Pro sells Patrol investigation, but every install starts in Watch only and stays there unless the operator finds the mode control. In the paid telemetry read on 12 October, 20 of 35 Pro installs with Patrol on ran Watch only, and none of those 20 ran an investigation in 30 days, so the feature they paid for never ran. An install that never saved a Patrol mode now runs Ask first when it holds a positive investigation licence. Patrol investigates new and re-detected warning and critical findings and proposes a fix; nothing changes without approval. The default only applies to a positive licence answer: a missing licence checker counts as unlicensed, so a startup window cannot spend provider budget on a free install. Free installs and explicit choices are unchanged. The Patrol header says so once, with the cost (the operator's AI provider and 30-day budget) and two actions that save Ask first or Watch only. The notice never renders on a plan-locked install. The header also stops claiming "Watching your infrastructure" while Patrol cannot run yet and reads "Patrol is not running yet" instead.
5134 lines
165 KiB
Go
5134 lines
165 KiB
Go
package ai
|
||
|
||
import (
|
||
"context"
|
||
"crypto/hmac"
|
||
cryptorand "crypto/rand"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"net/url"
|
||
"os"
|
||
"reflect"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/google/uuid"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/agentcapabilities"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/approval"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/modelboundary"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
||
aitools "github.com/rcourtman/pulse-go-rewrite/internal/ai/tools"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||
"github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"
|
||
pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
|
||
"github.com/rcourtman/pulse-go-rewrite/pkg/securityutil"
|
||
"github.com/rs/zerolog/log"
|
||
)
|
||
|
||
var modelsCacheCredentialKey = newModelsCacheCredentialKey()
|
||
|
||
func newModelsCacheCredentialKey() []byte {
|
||
key := make([]byte, 32)
|
||
if _, err := cryptorand.Read(key); err != nil {
|
||
panic(fmt.Sprintf("initialize AI models cache credential key: %v", err))
|
||
}
|
||
return key
|
||
}
|
||
|
||
func credentialCacheDigest(values ...string) string {
|
||
mac := hmac.New(sha256.New, modelsCacheCredentialKey)
|
||
for _, value := range values {
|
||
_, _ = io.WriteString(mac, strconv.Itoa(len(value)))
|
||
_, _ = io.WriteString(mac, ":")
|
||
_, _ = io.WriteString(mac, value)
|
||
}
|
||
return hex.EncodeToString(mac.Sum(nil))
|
||
}
|
||
|
||
// StateProvider is a type alias for models.SnapshotProvider.
|
||
// Kept for local convenience; all new code should use models.SnapshotProvider directly.
|
||
type StateProvider = models.SnapshotProvider
|
||
|
||
// CommandPolicy defines the interface for command security policy
|
||
type CommandPolicy interface {
|
||
Evaluate(command string) agentexec.PolicyDecision
|
||
}
|
||
|
||
// LicenseState represents the current state of the license
|
||
type LicenseState string
|
||
|
||
const (
|
||
LicenseStateNone LicenseState = "none"
|
||
LicenseStateActive LicenseState = "active"
|
||
LicenseStateExpired LicenseState = "expired"
|
||
LicenseStateGracePeriod LicenseState = "grace_period"
|
||
)
|
||
|
||
const legacyServiceToolResultModelContextChars = 8000
|
||
|
||
func legacyServiceProviderToolResultContextOptions() agentcapabilities.ProviderToolResultContextOptions {
|
||
return agentcapabilities.ProviderToolResultContextOptions{
|
||
MaxModelContentChars: legacyServiceToolResultModelContextChars,
|
||
}
|
||
}
|
||
|
||
func newLegacyServiceProviderToolResultContextProjection(toolUseID, content string, isError bool) agentcapabilities.ProviderToolResultContextProjection {
|
||
projection := agentcapabilities.NewProviderToolResultContextProjection(toolUseID, content, isError, legacyServiceProviderToolResultContextOptions())
|
||
logLegacyServiceProviderToolResultTruncation(projection.Truncation)
|
||
return projection
|
||
}
|
||
|
||
func legacyAssistantCapabilityInputSchema(name string) map[string]interface{} {
|
||
cap, ok := agentcapabilities.FindCapability(agentcapabilities.CanonicalManifest().Capabilities, name)
|
||
if !ok {
|
||
return agentcapabilities.ProviderInputSchema(agentcapabilities.InputSchema{})
|
||
}
|
||
return agentcapabilities.ProviderInputSchemaFromRaw(agentcapabilities.ToolInputSchema(cap))
|
||
}
|
||
|
||
func logLegacyServiceProviderToolResultTruncation(truncation agentcapabilities.ProviderToolResultTruncation) {
|
||
if !truncation.Applied {
|
||
return
|
||
}
|
||
|
||
log.Debug().
|
||
Int("original_chars", truncation.OriginalChars).
|
||
Int("truncated_to", truncation.MaxChars).
|
||
Int("chars_cut", truncation.TruncatedChars).
|
||
Msg("Truncated large tool result")
|
||
}
|
||
|
||
// LicenseChecker provides license feature checking for Pro features
|
||
type LicenseChecker interface {
|
||
HasFeature(feature string) bool
|
||
// GetLicenseStateString returns the current license state (none, active, expired, grace_period)
|
||
// and whether features are available (true for active/grace_period)
|
||
GetLicenseStateString() (string, bool)
|
||
}
|
||
|
||
// AgentServer defines the interface for communicating with agents
|
||
type AgentServer interface {
|
||
GetConnectedAgents() []agentexec.ConnectedAgent
|
||
ExecuteCommand(ctx context.Context, agentID string, cmd agentexec.ExecuteCommandPayload) (*agentexec.CommandResultPayload, error)
|
||
}
|
||
|
||
// ChatServiceProvider defines the interface for accessing chat functionality
|
||
// This is used by the investigation orchestrator and patrol to run AI executions
|
||
type ChatServiceProvider interface {
|
||
CreateSession(ctx context.Context) (*ChatSession, error)
|
||
ExecuteStream(ctx context.Context, req ChatExecuteRequest, callback ChatStreamCallback) error
|
||
ExecutePatrolStream(ctx context.Context, req PatrolExecuteRequest, callback ChatStreamCallback) (*PatrolStreamResponse, error)
|
||
GetMessages(ctx context.Context, sessionID string) ([]ChatMessage, error)
|
||
DeleteSession(ctx context.Context, sessionID string) error
|
||
ReloadConfig(ctx context.Context, cfg *config.AIConfig) error
|
||
}
|
||
|
||
// ChatSession represents a chat session (minimal interface for investigations)
|
||
type ChatSession struct {
|
||
ID string `json:"id"`
|
||
}
|
||
|
||
// ChatExecuteRequest represents a chat execution request
|
||
type ChatExecuteRequest struct {
|
||
Prompt string `json:"prompt"`
|
||
SessionID string `json:"session_id,omitempty"`
|
||
}
|
||
|
||
// ChatStreamCallback is called for each streaming event
|
||
type ChatStreamCallback func(event ChatStreamEvent)
|
||
|
||
// ChatStreamEvent represents a streaming event
|
||
type ChatStreamEvent struct {
|
||
Type string `json:"type"`
|
||
Data []byte `json:"data,omitempty"`
|
||
}
|
||
|
||
// ChatMessage represents a chat message
|
||
type ChatMessage struct {
|
||
ID string `json:"id"`
|
||
Role string `json:"role"`
|
||
Content string `json:"content"`
|
||
ReasoningContent string `json:"reasoning_content,omitempty"`
|
||
ToolCalls []ChatToolCall `json:"tool_calls"`
|
||
ToolResult *ChatToolResult `json:"tool_result,omitempty"`
|
||
Timestamp time.Time `json:"timestamp"`
|
||
}
|
||
|
||
func EmptyChatMessage() ChatMessage {
|
||
return ChatMessage{}.NormalizeCollections()
|
||
}
|
||
|
||
func (m ChatMessage) NormalizeCollections() ChatMessage {
|
||
if m.ToolCalls == nil {
|
||
m.ToolCalls = []ChatToolCall{}
|
||
}
|
||
for i := range m.ToolCalls {
|
||
m.ToolCalls[i] = m.ToolCalls[i].NormalizeCollections()
|
||
}
|
||
return m
|
||
}
|
||
|
||
// ChatToolCall retains observed output and result status in product history.
|
||
// Provider turns use the explicit ProviderToolCall projection.
|
||
type ChatToolCall = agentcapabilities.TranscriptToolCall
|
||
|
||
func EmptyChatToolCall() ChatToolCall {
|
||
return ChatToolCall{}.NormalizeCollections()
|
||
}
|
||
|
||
// ChatToolResult represents the result of a tool invocation. It aliases the
|
||
// shared Pulse Intelligence provider-result shape so API-facing chat messages
|
||
// and provider turns do not drift on tool result JSON.
|
||
type ChatToolResult = agentcapabilities.ProviderToolResult
|
||
|
||
// PatrolExecuteRequest represents a patrol execution request via the chat service
|
||
type PatrolExecuteRequest struct {
|
||
Prompt string `json:"prompt"`
|
||
SystemPrompt string `json:"system_prompt"`
|
||
SessionID string `json:"session_id,omitempty"`
|
||
ExecutionID string `json:"execution_id,omitempty"`
|
||
UseCase string `json:"use_case"` // "patrol" — for model selection
|
||
MaxTurns int `json:"max_turns,omitempty"`
|
||
AllowedToolNames []string `json:"allowed_tool_names,omitempty"`
|
||
}
|
||
|
||
// QuickAnalysisRequest represents a lightweight single-turn analysis request.
|
||
// ExecutionID lets higher-level workflows group ancillary calls under one
|
||
// commercial run instead of charging each helper call separately.
|
||
type QuickAnalysisRequest struct {
|
||
Prompt string `json:"prompt"`
|
||
ExecutionID string `json:"execution_id,omitempty"`
|
||
UseCase string `json:"use_case,omitempty"`
|
||
// TargetType tags the recorded usage event so cost telemetry can
|
||
// distinguish sub-workloads sharing a use case (e.g. patrol alert
|
||
// auto-resolve reviews vs the main patrol pass).
|
||
TargetType string `json:"target_type,omitempty"`
|
||
}
|
||
|
||
// PatrolStreamResponse contains the results of a patrol execution via the chat service
|
||
type PatrolStreamResponse struct {
|
||
Content string `json:"content"`
|
||
InputTokens int `json:"input_tokens"`
|
||
OutputTokens int `json:"output_tokens"`
|
||
// Prompt-cache buckets summed across the run's provider calls; disjoint
|
||
// from InputTokens, see providers.ChatResponse.
|
||
CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"`
|
||
CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"`
|
||
// ToolCalls carries calls observed by bounded Patrol follow-up passes so
|
||
// the parent run's durable audit contains every provider action.
|
||
ToolCalls []ToolCallRecord `json:"tool_calls,omitempty"`
|
||
}
|
||
|
||
// Service orchestrates AI interactions
|
||
type Service struct {
|
||
mu sync.RWMutex
|
||
orgID string
|
||
persistence *config.ConfigPersistence
|
||
provider providers.Provider
|
||
// providerInitErr records why the default provider could not be built even
|
||
// though the config claims an enabled, configured provider (for example a
|
||
// live model-catalog resolution against an Ollama server that was still
|
||
// booting). Patrol readiness reports it instead of the misleading
|
||
// "provider not configured" copy, and RetryProviderInit clears it.
|
||
providerInitErr string
|
||
cfg *config.AIConfig
|
||
agentServer AgentServer
|
||
policy CommandPolicy
|
||
stateProvider StateProvider
|
||
readState unifiedresources.ReadState
|
||
alertProvider AlertProvider
|
||
knowledgeStore *knowledge.Store
|
||
costStore *cost.Store
|
||
unifiedResourceProvider UnifiedResourceProvider
|
||
resourceExportStore unifiedresources.ResourceStore
|
||
resourceExportStoreOrgID string
|
||
patrolAutopilotPolicy func() unifiedresources.PatrolAutopilotServerPolicy
|
||
patrolObjectiveStore *PatrolObjectiveStore
|
||
patrolService *PatrolService // Background AI monitoring service
|
||
thresholdProvider ThresholdProvider // User-configured alert thresholds; nil means factory defaults
|
||
metadataProvider MetadataProvider // Enables AI to update resource URLs
|
||
incidentStore *memory.IncidentStore // Alert-scoped investigation memory; not the canonical durable resource history
|
||
chatService ChatServiceProvider // Chat service for investigation orchestrator
|
||
|
||
// Alert-triggered analysis - token-efficient real-time AI insights
|
||
alertTriggeredAnalyzer aicontracts.AlertAnalyzer
|
||
// alertAnalyzerFactory gates construction of the AlertAnalyzer.
|
||
// When nil (OSS binary), the analyzer is never created regardless of config.
|
||
alertAnalyzerFactory func(aicontracts.AlertAnalyzerDeps) aicontracts.AlertAnalyzer
|
||
|
||
limits executionLimits
|
||
|
||
modelsCache modelsCache
|
||
|
||
// patrolPreflightCache holds the most recent preflight result for the
|
||
// configured Patrol provider+model so operators can see when tools
|
||
// were last verified without re-running the call.
|
||
patrolPreflightCache patrolPreflightCache
|
||
|
||
// patrolModelReadinessCache holds the explicit, multi-scenario advisor
|
||
// evidence separately from the lightweight startup preflight.
|
||
patrolModelReadinessCache patrolModelReadinessCache
|
||
|
||
// License checker for Pro feature gating
|
||
licenseChecker LicenseChecker
|
||
}
|
||
|
||
type executionLimits struct {
|
||
chatSlots chan struct{}
|
||
patrolSlots chan struct{}
|
||
}
|
||
|
||
type providerModelsEntry struct {
|
||
at time.Time
|
||
models []providers.ModelInfo
|
||
}
|
||
|
||
type modelsCache struct {
|
||
mu sync.RWMutex
|
||
key string
|
||
providers map[string]providerModelsEntry
|
||
ttl time.Duration
|
||
}
|
||
|
||
// alertFindingRecorderAdapter adapts PatrolService.recordFinding for the
|
||
// aicontracts.FindingRecorder interface used by the enterprise alert analyzer.
|
||
type alertFindingRecorderAdapter struct {
|
||
ps *PatrolService
|
||
}
|
||
|
||
func (a *alertFindingRecorderAdapter) RecordAlertFinding(f *aicontracts.AlertAnalyzerFinding) {
|
||
if a.ps == nil || f == nil {
|
||
return
|
||
}
|
||
a.ps.recordFinding(&Finding{
|
||
ID: f.ID,
|
||
Key: f.Key,
|
||
Severity: FindingSeverity(f.Severity),
|
||
Category: FindingCategory(f.Category),
|
||
ResourceID: f.ResourceID,
|
||
ResourceName: f.ResourceName,
|
||
ResourceType: f.ResourceType,
|
||
Title: f.Title,
|
||
Description: f.Description,
|
||
Recommendation: f.Recommendation,
|
||
Evidence: f.Evidence,
|
||
AlertIdentifier: f.AlertIdentifier,
|
||
DetectedAt: f.DetectedAt,
|
||
LastSeenAt: f.LastSeenAt,
|
||
})
|
||
}
|
||
|
||
// NewService creates a new AI service
|
||
func NewService(persistence *config.ConfigPersistence, agentServer AgentServer) *Service {
|
||
// Initialize knowledge store
|
||
var knowledgeStore *knowledge.Store
|
||
costStore := cost.NewStore(cost.DefaultMaxDays)
|
||
objectiveStore := NewInMemoryPatrolObjectiveStore()
|
||
if persistence != nil {
|
||
var err error
|
||
knowledgeStore, err = knowledge.NewStore(persistence.DataDir())
|
||
if err != nil {
|
||
log.Warn().Err(err).Msg("failed to initialize knowledge store")
|
||
}
|
||
if err := costStore.SetPersistence(NewCostPersistenceAdapter(persistence)); err != nil {
|
||
log.Warn().Err(err).Msg("failed to initialize AI usage cost store")
|
||
}
|
||
objectiveStore, err = NewPatrolObjectiveStore(persistence.DataDir())
|
||
if err != nil {
|
||
objectiveStore = nil
|
||
log.Error().Err(err).Msg("failed to initialize encrypted Patrol objective store")
|
||
}
|
||
}
|
||
|
||
svc := &Service{
|
||
orgID: "default",
|
||
persistence: persistence,
|
||
agentServer: agentServer,
|
||
policy: agentexec.DefaultPolicy(),
|
||
knowledgeStore: knowledgeStore,
|
||
costStore: costStore,
|
||
patrolObjectiveStore: objectiveStore,
|
||
limits: executionLimits{
|
||
chatSlots: make(chan struct{}, 4),
|
||
patrolSlots: make(chan struct{}, 1),
|
||
},
|
||
modelsCache: modelsCache{
|
||
ttl: 5 * time.Minute,
|
||
providers: make(map[string]providerModelsEntry),
|
||
},
|
||
}
|
||
|
||
if persistence != nil {
|
||
if store, err := unifiedresources.NewSQLiteResourceStore(persistence.DataDir(), svc.orgID); err != nil {
|
||
log.Warn().Err(err).Str("orgID", svc.orgID).Msg("failed to initialize unified resource export store")
|
||
} else {
|
||
svc.resourceExportStore = store
|
||
svc.resourceExportStoreOrgID = svc.orgID
|
||
}
|
||
}
|
||
|
||
svc.loadPatrolModelReadiness()
|
||
|
||
return svc
|
||
}
|
||
|
||
// SetOrgID sets the org scope used when creating approval records.
|
||
func (s *Service) SetOrgID(orgID string) {
|
||
normalized := strings.TrimSpace(orgID)
|
||
if normalized == "" {
|
||
normalized = "default"
|
||
}
|
||
|
||
s.mu.Lock()
|
||
if s.orgID == normalized && s.resourceExportStore != nil {
|
||
s.mu.Unlock()
|
||
return
|
||
}
|
||
|
||
oldStore := s.resourceExportStore
|
||
incidentStore := s.incidentStore
|
||
persistence := s.persistence
|
||
s.orgID = normalized
|
||
s.resourceExportStore = nil
|
||
s.resourceExportStoreOrgID = ""
|
||
s.mu.Unlock()
|
||
|
||
if incidentStore != nil {
|
||
incidentStore.SetResourceTimelineStore(nil)
|
||
}
|
||
|
||
if oldStore != nil {
|
||
if err := oldStore.Close(); err != nil {
|
||
log.Warn().Err(err).Str("orgID", normalized).Msg("failed to close previous unified resource export store")
|
||
}
|
||
}
|
||
|
||
if persistence == nil {
|
||
return
|
||
}
|
||
|
||
store, err := unifiedresources.NewSQLiteResourceStore(persistence.DataDir(), normalized)
|
||
if err != nil {
|
||
log.Warn().Err(err).Str("orgID", normalized).Msg("failed to initialize unified resource export store for org")
|
||
return
|
||
}
|
||
|
||
s.mu.Lock()
|
||
if s.orgID == normalized && s.resourceExportStore == nil {
|
||
s.resourceExportStore = store
|
||
s.resourceExportStoreOrgID = normalized
|
||
incidentStore = s.incidentStore
|
||
s.mu.Unlock()
|
||
if incidentStore != nil {
|
||
incidentStore.SetResourceTimelineStore(store)
|
||
}
|
||
return
|
||
}
|
||
s.mu.Unlock()
|
||
|
||
if err := store.Close(); err != nil {
|
||
log.Warn().Err(err).Str("orgID", normalized).Msg("failed to close unused unified resource export store")
|
||
}
|
||
}
|
||
|
||
// GetOrgID returns the org scope associated with this service instance.
|
||
func (s *Service) GetOrgID() string {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
orgID := strings.TrimSpace(s.orgID)
|
||
if orgID == "" {
|
||
return "default"
|
||
}
|
||
return orgID
|
||
}
|
||
|
||
func (s *Service) acquireExecutionSlot(ctx context.Context, useCase string) (func(), error) {
|
||
normalized := strings.TrimSpace(strings.ToLower(useCase))
|
||
if normalized == "" {
|
||
normalized = "chat"
|
||
}
|
||
|
||
var slots chan struct{}
|
||
if normalized == "patrol" {
|
||
slots = s.limits.patrolSlots
|
||
} else {
|
||
slots = s.limits.chatSlots
|
||
}
|
||
|
||
timer := time.NewTimer(5 * time.Second)
|
||
defer func() {
|
||
if !timer.Stop() {
|
||
select {
|
||
case <-timer.C:
|
||
default:
|
||
}
|
||
}
|
||
}()
|
||
|
||
select {
|
||
case slots <- struct{}{}:
|
||
return func() { <-slots }, nil
|
||
case <-ctx.Done():
|
||
return nil, ctx.Err()
|
||
case <-timer.C:
|
||
return nil, fmt.Errorf("Pulse Assistant is busy - too many concurrent requests")
|
||
}
|
||
}
|
||
|
||
func (s *Service) enforceBudget(useCase string) error {
|
||
s.mu.RLock()
|
||
cfg := s.cfg
|
||
store := s.costStore
|
||
s.mu.RUnlock()
|
||
|
||
if cfg == nil || cfg.CostBudgetUSD30d <= 0 || store == nil {
|
||
return nil
|
||
}
|
||
|
||
summary := store.GetSummary(30)
|
||
if !summary.Totals.PricingKnown {
|
||
// We can't reliably enforce without pricing. Keep tracking and allow.
|
||
return nil
|
||
}
|
||
|
||
if summary.Totals.EstimatedUSD >= cfg.CostBudgetUSD30d {
|
||
normalized := strings.TrimSpace(strings.ToLower(useCase))
|
||
if normalized == "" {
|
||
normalized = "chat"
|
||
}
|
||
return &CostBudgetExceededError{
|
||
SpentUSD: summary.Totals.EstimatedUSD,
|
||
BudgetUSD: cfg.CostBudgetUSD30d,
|
||
Days: summary.EffectiveDays,
|
||
}
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// CheckBudget is a public wrapper around enforceBudget, allowing callers
|
||
// (e.g. patrol) to verify budget availability before starting expensive work.
|
||
func (s *Service) CheckBudget(useCase string) error {
|
||
return s.enforceBudget(useCase)
|
||
}
|
||
|
||
// SetReadState injects a unified read-state provider for context enrichment.
|
||
// This is forwarded to PatrolService when available.
|
||
func (s *Service) SetReadState(rs unifiedresources.ReadState) {
|
||
s.mu.Lock()
|
||
defer s.mu.Unlock()
|
||
s.readState = rs
|
||
|
||
s.initPatrolServiceLocked()
|
||
|
||
if s.patrolService != nil {
|
||
s.patrolService.SetReadState(rs)
|
||
}
|
||
}
|
||
|
||
// initPatrolServiceLocked creates the patrol service once any canonical patrol
|
||
// runtime input exists. Must be called while holding s.mu.
|
||
func (s *Service) initPatrolServiceLocked() {
|
||
if s.patrolService != nil {
|
||
return
|
||
}
|
||
if s.stateProvider == nil && s.readState == nil && s.unifiedResourceProvider == nil {
|
||
return
|
||
}
|
||
|
||
s.patrolService = NewPatrolService(s, s.stateProvider)
|
||
s.patrolService.SetObjectiveStore(s.patrolObjectiveStore)
|
||
if s.thresholdProvider != nil {
|
||
s.patrolService.SetThresholdProvider(s.thresholdProvider)
|
||
}
|
||
if s.knowledgeStore != nil {
|
||
s.patrolService.SetKnowledgeStore(s.knowledgeStore)
|
||
}
|
||
if s.incidentStore != nil {
|
||
s.patrolService.SetIncidentStore(s.incidentStore)
|
||
}
|
||
if s.readState != nil {
|
||
s.patrolService.SetReadState(s.readState)
|
||
}
|
||
if s.unifiedResourceProvider != nil {
|
||
s.patrolService.SetUnifiedResourceProvider(s.unifiedResourceProvider)
|
||
}
|
||
}
|
||
|
||
// SetStateProvider sets the state provider for infrastructure context
|
||
func (s *Service) SetStateProvider(sp StateProvider) {
|
||
s.mu.Lock()
|
||
s.stateProvider = sp
|
||
|
||
s.initPatrolServiceLocked()
|
||
if s.patrolService != nil {
|
||
s.patrolService.SetStateProvider(sp)
|
||
}
|
||
|
||
// Initialize alert-triggered analyzer if not already done (requires enterprise factory)
|
||
if s.alertTriggeredAnalyzer == nil && s.alertAnalyzerFactory != nil && s.patrolService != nil {
|
||
deps := aicontracts.AlertAnalyzerDeps{
|
||
FindingRecorder: &alertFindingRecorderAdapter{ps: s.patrolService},
|
||
IncidentRecorder: s,
|
||
}
|
||
s.alertTriggeredAnalyzer = s.alertAnalyzerFactory(deps)
|
||
}
|
||
s.mu.Unlock()
|
||
}
|
||
|
||
// GetStateProvider returns the state provider for infrastructure context
|
||
func (s *Service) GetStateProvider() StateProvider {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
return s.stateProvider
|
||
}
|
||
|
||
// GetPatrolService returns the patrol service for background monitoring
|
||
func (s *Service) GetPatrolService() *PatrolService {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
return s.patrolService
|
||
}
|
||
|
||
// GetPatrolObjectiveStore returns the organization-scoped durable operator
|
||
// objective store. It exists independently from a running Patrol service so an
|
||
// operator can retain intent while provider or infrastructure setup is offline.
|
||
func (s *Service) GetPatrolObjectiveStore() *PatrolObjectiveStore {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
return s.patrolObjectiveStore
|
||
}
|
||
|
||
// SetChatService sets the chat service for investigation orchestrator
|
||
func (s *Service) SetChatService(cs ChatServiceProvider) {
|
||
s.mu.Lock()
|
||
defer s.mu.Unlock()
|
||
s.chatService = cs
|
||
s.configurePatrolProviderFactoryLocked()
|
||
}
|
||
|
||
// GetChatService returns the chat service for investigation orchestrator
|
||
func (s *Service) GetChatService() ChatServiceProvider {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
return s.chatService
|
||
}
|
||
|
||
// GetRemediationLog returns the remediation log from the patrol service
|
||
func (s *Service) GetRemediationLog() *RemediationLog {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
if patrol == nil {
|
||
return nil
|
||
}
|
||
return patrol.GetRemediationLog()
|
||
}
|
||
|
||
// GetAlertTriggeredAnalyzer returns the alert-triggered analyzer for token-efficient real-time analysis
|
||
func (s *Service) GetAlertTriggeredAnalyzer() aicontracts.AlertAnalyzer {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
return s.alertTriggeredAnalyzer
|
||
}
|
||
|
||
// GetAIConfig returns the current AI configuration
|
||
func (s *Service) GetAIConfig() *config.AIConfig {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
return s.cfg
|
||
}
|
||
|
||
// CostStore returns the underlying usage cost store handle so callers
|
||
// that own a Service can record cost events directly. Returned value
|
||
// may be nil when persistence/initialization failed during NewService.
|
||
// Consumers must nil-check before recording.
|
||
func (s *Service) CostStore() *cost.Store {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
return s.costStore
|
||
}
|
||
|
||
// GetCostSummary returns usage rollups for the last N days.
|
||
func (s *Service) GetCostSummary(days int) cost.Summary {
|
||
s.mu.RLock()
|
||
store := s.costStore
|
||
s.mu.RUnlock()
|
||
|
||
if store == nil {
|
||
if days <= 0 {
|
||
days = 30
|
||
}
|
||
effectiveDays := days
|
||
truncated := false
|
||
if cost.DefaultMaxDays > 0 && days > cost.DefaultMaxDays {
|
||
effectiveDays = cost.DefaultMaxDays
|
||
truncated = true
|
||
}
|
||
return cost.EmptySummary(days, cost.DefaultMaxDays, effectiveDays, truncated)
|
||
}
|
||
return store.GetSummary(days)
|
||
}
|
||
|
||
// ListCostEvents returns retained AI usage events within the requested time window.
|
||
func (s *Service) ListCostEvents(days int) []cost.UsageEvent {
|
||
s.mu.RLock()
|
||
store := s.costStore
|
||
s.mu.RUnlock()
|
||
if store == nil {
|
||
return nil
|
||
}
|
||
return store.ListEvents(days)
|
||
}
|
||
|
||
// ClearCostHistory deletes retained AI usage events (admin operation).
|
||
func (s *Service) ClearCostHistory() error {
|
||
s.mu.RLock()
|
||
store := s.costStore
|
||
s.mu.RUnlock()
|
||
if store == nil {
|
||
return nil
|
||
}
|
||
return store.Clear()
|
||
}
|
||
|
||
// SetPatrolThresholdProvider sets the threshold provider for patrol
|
||
// This should be called with an AlertThresholdAdapter to connect patrol to user-configured thresholds
|
||
func (s *Service) SetPatrolThresholdProvider(provider ThresholdProvider) {
|
||
s.mu.Lock()
|
||
s.thresholdProvider = provider
|
||
patrol := s.patrolService
|
||
s.mu.Unlock()
|
||
|
||
if patrol != nil {
|
||
patrol.SetThresholdProvider(provider)
|
||
}
|
||
}
|
||
|
||
// MetricsHistoryProvider provides access to historical metrics for trend analysis
|
||
// This interface matches the monitoring.MetricsHistory methods we need
|
||
type MetricsHistoryProvider interface {
|
||
GetNodeMetrics(nodeID string, metricType string, duration time.Duration) []MetricPoint
|
||
GetGuestMetrics(guestID string, metricType string, duration time.Duration) []MetricPoint
|
||
GetAllGuestMetrics(guestID string, duration time.Duration) map[string][]MetricPoint
|
||
GetAllStorageMetrics(storageID string, duration time.Duration) map[string][]MetricPoint
|
||
}
|
||
|
||
// MetricPoint is an alias for the shared metric point type
|
||
type MetricPoint = models.MetricPoint
|
||
|
||
// SetMetricsHistoryProvider sets the metrics history provider for enriched AI context
|
||
// This enables the AI to see trends, anomalies, and predictions based on historical data
|
||
func (s *Service) SetMetricsHistoryProvider(provider MetricsHistoryProvider) {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol != nil {
|
||
patrol.SetMetricsHistoryProvider(provider)
|
||
}
|
||
}
|
||
|
||
// SetBaselineStore sets the baseline store for anomaly detection
|
||
func (s *Service) SetBaselineStore(store *BaselineStore) {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol != nil {
|
||
patrol.SetBaselineStore(store)
|
||
}
|
||
}
|
||
|
||
// SetChangeDetector sets the change detector for operational memory
|
||
func (s *Service) SetChangeDetector(detector *ChangeDetector) {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol != nil {
|
||
patrol.SetChangeDetector(detector)
|
||
}
|
||
}
|
||
|
||
// SetRemediationLog sets the remediation log for tracking fix attempts
|
||
func (s *Service) SetRemediationLog(remLog *RemediationLog) {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol != nil {
|
||
patrol.SetRemediationLog(remLog)
|
||
}
|
||
}
|
||
|
||
// SetIncidentStore sets the incident store for alert timeline memory.
|
||
func (s *Service) SetIncidentStore(store *memory.IncidentStore) {
|
||
s.mu.Lock()
|
||
defer s.mu.Unlock()
|
||
s.incidentStore = store
|
||
if store != nil {
|
||
store.SetResourceTimelineStore(s.resourceExportStore)
|
||
}
|
||
|
||
if s.patrolService != nil {
|
||
s.patrolService.SetIncidentStore(store)
|
||
}
|
||
}
|
||
|
||
// SetPatternDetector sets the pattern detector for failure prediction
|
||
func (s *Service) SetPatternDetector(detector *PatternDetector) {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol != nil {
|
||
patrol.SetPatternDetector(detector)
|
||
}
|
||
}
|
||
|
||
// SetCorrelationDetector sets the correlation detector for multi-resource correlation
|
||
func (s *Service) SetCorrelationDetector(detector *CorrelationDetector) {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol != nil {
|
||
patrol.SetCorrelationDetector(detector)
|
||
}
|
||
}
|
||
|
||
// SetLicenseChecker sets the license checker for Pro feature gating
|
||
func (s *Service) SetLicenseChecker(checker LicenseChecker) {
|
||
s.mu.Lock()
|
||
defer s.mu.Unlock()
|
||
s.licenseChecker = checker
|
||
}
|
||
|
||
// SetAlertAnalyzerFactory registers the factory that creates the enterprise
|
||
// AlertTriggeredAnalyzer. When nil (OSS binary), no analyzer is created.
|
||
func (s *Service) SetAlertAnalyzerFactory(fn func(aicontracts.AlertAnalyzerDeps) aicontracts.AlertAnalyzer) {
|
||
s.mu.Lock()
|
||
defer s.mu.Unlock()
|
||
s.alertAnalyzerFactory = fn
|
||
}
|
||
|
||
// HasLicenseFeature checks if a Pro feature is licensed (returns true if no license checker is set)
|
||
func (s *Service) HasLicenseFeature(feature string) bool {
|
||
s.mu.RLock()
|
||
checker := s.licenseChecker
|
||
s.mu.RUnlock()
|
||
|
||
if checker == nil {
|
||
// No license checker means no enforcement (development mode or feature not gated)
|
||
return true
|
||
}
|
||
return checker.HasFeature(feature)
|
||
}
|
||
|
||
// PatrolInvestigationLicensed reports a positive licence answer for Patrol
|
||
// investigation. Unlike HasLicenseFeature it treats a missing checker as not
|
||
// licensed, so the Pro default mode never applies before the licence is known.
|
||
func (s *Service) PatrolInvestigationLicensed() bool {
|
||
s.mu.RLock()
|
||
checker := s.licenseChecker
|
||
s.mu.RUnlock()
|
||
return checker != nil && checker.HasFeature(FeatureAIAutoFix)
|
||
}
|
||
|
||
const minCommunityPatrolInterval = 1 * time.Hour
|
||
|
||
// GetEffectivePatrolAutonomyLevel returns the autonomy level that should be enforced at runtime
|
||
// for the current license state.
|
||
//
|
||
// Community tier is locked to "monitor" (findings-only). Higher levels imply investigation and
|
||
// are gated behind ai_autofix (Pro/Cloud). A licensed install that never chose a mode runs
|
||
// "approval" (Ask first), so the investigation it pays for is on by default.
|
||
func (s *Service) GetEffectivePatrolAutonomyLevel() string {
|
||
cfg := s.GetConfig()
|
||
if cfg == nil {
|
||
return config.PatrolAutonomyMonitor
|
||
}
|
||
if !s.HasLicenseFeature(FeatureAIAutoFix) {
|
||
return config.PatrolAutonomyMonitor
|
||
}
|
||
if !cfg.PatrolAutonomyLevelChosen() {
|
||
if s.PatrolInvestigationLicensed() {
|
||
return config.PatrolAutonomyApproval
|
||
}
|
||
return config.PatrolAutonomyMonitor
|
||
}
|
||
s.mu.RLock()
|
||
orgID := s.orgID
|
||
policyProvider := s.patrolAutopilotPolicy
|
||
s.mu.RUnlock()
|
||
policy := unifiedresources.CurrentPatrolAutopilotServerPolicy(time.Now().UTC())
|
||
if policyProvider != nil {
|
||
policy = policyProvider()
|
||
}
|
||
level, _ := cfg.GetEffectivePatrolAutonomyWithPolicy(orgID, policy)
|
||
return level
|
||
}
|
||
|
||
// SetPatrolAutopilotServerPolicyProvider keeps runtime admission on the same
|
||
// server-owned clock and current-version source as the API boundary.
|
||
func (s *Service) SetPatrolAutopilotServerPolicyProvider(provider func() unifiedresources.PatrolAutopilotServerPolicy) {
|
||
if s == nil {
|
||
return
|
||
}
|
||
s.mu.Lock()
|
||
defer s.mu.Unlock()
|
||
s.patrolAutopilotPolicy = provider
|
||
}
|
||
|
||
// ApplyPatrolAutonomyConfig publishes an already-persisted autonomy mutation
|
||
// without rebuilding provider clients. Callers must persist first and invoke
|
||
// this while holding the policy-mutation coordinator so revocation and runtime
|
||
// admission observe one ordered boundary.
|
||
func (s *Service) ApplyPatrolAutonomyConfig(cfg *config.AIConfig) error {
|
||
if s == nil || cfg == nil {
|
||
return fmt.Errorf("Pulse Patrol config is unavailable")
|
||
}
|
||
s.mu.Lock()
|
||
defer s.mu.Unlock()
|
||
if s.cfg == nil {
|
||
s.cfg = config.NewDefaultAIConfig()
|
||
}
|
||
s.cfg.PatrolAutonomyLevel = cfg.PatrolAutonomyLevel
|
||
s.cfg.PatrolFullModeUnlocked = cfg.PatrolFullModeUnlocked
|
||
s.cfg.PatrolInvestigationBudget = cfg.PatrolInvestigationBudget
|
||
s.cfg.PatrolInvestigationTimeoutSec = cfg.PatrolInvestigationTimeoutSec
|
||
s.cfg.PatrolAutopilotAcknowledgements = clonePatrolAutopilotAcknowledgements(cfg.PatrolAutopilotAcknowledgements)
|
||
s.cfg.PatrolAutopilotRevocations = append([]unifiedresources.PatrolAutopilotRevocation(nil), cfg.PatrolAutopilotRevocations...)
|
||
if cfg.PatrolAutopilotActivation == nil {
|
||
s.cfg.PatrolAutopilotActivation = nil
|
||
} else {
|
||
activation := *cfg.PatrolAutopilotActivation
|
||
s.cfg.PatrolAutopilotActivation = &activation
|
||
}
|
||
if s.patrolService != nil {
|
||
orchestrator := s.patrolService.GetInvestigationOrchestrator()
|
||
if updater, ok := orchestrator.(aicontracts.InvestigationExecutionLimitUpdater); ok {
|
||
updater.SetInvestigationExecutionLimits(
|
||
s.cfg.GetPatrolInvestigationBudget(),
|
||
s.cfg.GetPatrolInvestigationTimeout(),
|
||
)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func clonePatrolAutopilotAcknowledgements(records []unifiedresources.PatrolAutopilotAcknowledgement) []unifiedresources.PatrolAutopilotAcknowledgement {
|
||
cloned := append([]unifiedresources.PatrolAutopilotAcknowledgement(nil), records...)
|
||
for index := range cloned {
|
||
cloned[index].AcceptedScope = append([]string(nil), cloned[index].AcceptedScope...)
|
||
}
|
||
return cloned
|
||
}
|
||
|
||
func (s *Service) getEffectivePatrolInterval(cfg *config.AIConfig) time.Duration {
|
||
if cfg == nil {
|
||
return 0
|
||
}
|
||
interval := cfg.GetPatrolInterval()
|
||
if interval <= 0 {
|
||
return interval
|
||
}
|
||
if !s.HasLicenseFeature(FeatureAIAutoFix) && interval < minCommunityPatrolInterval {
|
||
return minCommunityPatrolInterval
|
||
}
|
||
return interval
|
||
}
|
||
|
||
func (s *Service) patrolConfigFromAIConfig(cfg *config.AIConfig) PatrolConfig {
|
||
patrolCfg := DefaultPatrolConfig()
|
||
if cfg == nil {
|
||
if IsDemoMode() {
|
||
// Demo runtimes simulate patrol without provider settings.
|
||
patrolCfg.Enabled = true
|
||
return patrolCfg
|
||
}
|
||
patrolCfg.Enabled = false
|
||
patrolCfg.RuntimeBlockedReason = "Pulse Intelligence settings could not be loaded from persistence."
|
||
patrolCfg.RuntimeBlockedCause = PatrolFailureCauseSettingsPersistence
|
||
return patrolCfg
|
||
}
|
||
|
||
patrolCfg.Enabled = cfg.IsPatrolEnabled()
|
||
patrolCfg.Interval = s.getEffectivePatrolInterval(cfg)
|
||
patrolCfg.AnalyzeNodes = cfg.PatrolAnalyzeNodes
|
||
patrolCfg.AnalyzeGuests = cfg.PatrolAnalyzeGuests
|
||
patrolCfg.AnalyzeDocker = cfg.PatrolAnalyzeDocker
|
||
patrolCfg.AnalyzeStorage = cfg.PatrolAnalyzeStorage
|
||
if IsDemoMode() {
|
||
// Demo/mock runtimes simulate patrol runs without a provider; never
|
||
// carry a readiness blocker into the demo loop.
|
||
patrolCfg.Enabled = true
|
||
return patrolCfg
|
||
}
|
||
if patrolCfg.Enabled {
|
||
if readiness := EvaluatePatrolConfigReadiness(cfg); !readiness.Ready {
|
||
patrolCfg.RuntimeBlockedReason = readiness.Summary
|
||
patrolCfg.RuntimeBlockedCause = readiness.Cause
|
||
}
|
||
}
|
||
return patrolCfg
|
||
}
|
||
|
||
// GetLicenseState returns the current license state and whether features are available
|
||
func (s *Service) GetLicenseState() (string, bool) {
|
||
s.mu.RLock()
|
||
checker := s.licenseChecker
|
||
s.mu.RUnlock()
|
||
|
||
if checker == nil {
|
||
// No license checker means development mode - treat as active
|
||
return string(LicenseStateActive), true
|
||
}
|
||
return checker.GetLicenseStateString()
|
||
}
|
||
|
||
// Feature constants are imported from the license package for compile-time consistency.
|
||
// This ensures the ai package always uses the same feature strings as the license system.
|
||
const (
|
||
FeatureAIPatrol = pkglicensing.FeatureAIPatrol
|
||
FeatureAIAlerts = pkglicensing.FeatureAIAlerts
|
||
FeatureAIAutoFix = pkglicensing.FeatureAIAutoFix
|
||
)
|
||
|
||
const DevDisableBackgroundAIEnv = "PULSE_DEV_DISABLE_BACKGROUND_AI"
|
||
|
||
// BackgroundAutomationDisabledForDev returns true when the local development
|
||
// runtime should avoid automatic AI work that can burn provider quota while a
|
||
// developer is only trying to run the app.
|
||
func BackgroundAutomationDisabledForDev() bool {
|
||
if os.Getenv("PULSE_DEV") != "true" && !strings.EqualFold(os.Getenv("NODE_ENV"), "development") {
|
||
return false
|
||
}
|
||
|
||
value := strings.TrimSpace(os.Getenv(DevDisableBackgroundAIEnv))
|
||
if value == "" {
|
||
return false
|
||
}
|
||
|
||
disabled, err := strconv.ParseBool(value)
|
||
if err != nil {
|
||
return false
|
||
}
|
||
return disabled
|
||
}
|
||
|
||
func patrolEventTriggerConfigFromAIConfig(cfg *config.AIConfig) PatrolEventTriggerConfig {
|
||
if cfg == nil {
|
||
return PatrolEventTriggerConfig{}
|
||
}
|
||
return PatrolEventTriggerConfig{
|
||
AlertTriggersEnabled: cfg.IsPatrolAlertTriggersEnabled(),
|
||
AnomalyTriggersEnabled: cfg.IsPatrolAnomalyTriggersEnabled(),
|
||
}
|
||
}
|
||
|
||
// StartPatrol starts the background patrol service
|
||
func (s *Service) StartPatrol(ctx context.Context) {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
alertAnalyzer := s.alertTriggeredAnalyzer
|
||
cfg := s.cfg
|
||
licenseChecker := s.licenseChecker
|
||
s.mu.RUnlock()
|
||
|
||
if patrol == nil {
|
||
log.Debug().Msg("patrol service not initialized, cannot start")
|
||
return
|
||
}
|
||
|
||
if cfg == nil || !cfg.IsPatrolEnabled() {
|
||
// Demo runtimes still start the loop: the release demo instance boots
|
||
// with mock fixtures off until the license sync authorizes them, and
|
||
// each tick re-checks IsDemoMode() live, so the loop self-heals into
|
||
// simulated patrol cycles once fixtures enable. Requires a loaded
|
||
// config because the setup below dereferences it.
|
||
if cfg == nil || !IsDemoRuntimeIntended() {
|
||
log.Debug().Msg("AI Patrol not enabled")
|
||
return
|
||
}
|
||
log.Info().Msg("AI Patrol: starting loop for demo runtime despite disabled config")
|
||
}
|
||
|
||
// Check license for Patrol fix actions (Pro only) - Patrol itself is free with BYOK
|
||
if licenseChecker != nil && !licenseChecker.HasFeature(FeatureAIAutoFix) {
|
||
log.Info().Msg("Patrol safe remediation requires Pulse Pro license - fixes will require manual approval")
|
||
}
|
||
|
||
// Configure patrol from AI config (preserve defaults for resource types not in AI config).
|
||
patrolCfg := s.patrolConfigFromAIConfig(cfg)
|
||
patrol.SetConfig(patrolCfg)
|
||
patrol.SetProactiveMode(cfg.UseProactiveThresholds)
|
||
patrol.SetEventTriggerConfig(patrolEventTriggerConfigFromAIConfig(cfg))
|
||
|
||
if BackgroundAutomationDisabledForDev() {
|
||
patrol.SetEventTriggerBlock(BackgroundAutomationEventTriggerBlock())
|
||
if alertAnalyzer != nil {
|
||
alertAnalyzer.SetEnabled(false)
|
||
}
|
||
log.Info().
|
||
Str("env", DevDisableBackgroundAIEnv).
|
||
Msg("Pulse dev background AI disabled; Patrol scheduler and alert-triggered AI are not started")
|
||
// The dev guard protects provider quota; a simulated demo cycle costs
|
||
// nothing, so mock-mode dev still gets a populated patrol surface.
|
||
if IsDemoMode() {
|
||
patrol.startDemoPatrolWarmup()
|
||
}
|
||
return
|
||
}
|
||
|
||
patrol.SetEventTriggerBlock(PatrolEventTriggerBlock{})
|
||
patrol.Start(ctx)
|
||
|
||
// Configure alert-triggered analyzer (also Pro-only)
|
||
if alertAnalyzer != nil {
|
||
// Only enable if licensed for AI Alerts
|
||
enabled := cfg.IsAlertTriggeredAnalysisEnabled()
|
||
if enabled && licenseChecker != nil && !licenseChecker.HasFeature(FeatureAIAlerts) {
|
||
log.Info().Msg("AI Alert Analysis requires Pulse Pro license - alert-triggered analysis disabled")
|
||
enabled = false
|
||
}
|
||
alertAnalyzer.SetEnabled(enabled)
|
||
alertAnalyzer.Start() // Start cleanup goroutine
|
||
log.Info().
|
||
Bool("enabled", enabled).
|
||
Msg("Alert-triggered AI analysis configured")
|
||
}
|
||
|
||
// In demo/mock mode, populate the patrol surface promptly with a
|
||
// simulated cycle instead of waiting for the first scheduled tick.
|
||
if IsDemoMode() {
|
||
patrol.startDemoPatrolWarmup()
|
||
}
|
||
}
|
||
|
||
// StopPatrol stops the background patrol service
|
||
func (s *Service) StopPatrol() {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
alertAnalyzer := s.alertTriggeredAnalyzer
|
||
s.mu.RUnlock()
|
||
|
||
if patrol != nil {
|
||
patrol.Stop()
|
||
}
|
||
if alertAnalyzer != nil {
|
||
alertAnalyzer.Stop()
|
||
}
|
||
}
|
||
|
||
// Stop stops all background services
|
||
func (s *Service) Stop() {
|
||
s.StopPatrol()
|
||
|
||
s.mu.Lock()
|
||
store := s.resourceExportStore
|
||
incidentStore := s.incidentStore
|
||
costStore := s.costStore
|
||
s.resourceExportStore = nil
|
||
s.resourceExportStoreOrgID = ""
|
||
defer s.mu.Unlock()
|
||
|
||
if incidentStore != nil {
|
||
incidentStore.SetResourceTimelineStore(nil)
|
||
}
|
||
|
||
if costStore != nil {
|
||
if err := costStore.Flush(); err != nil {
|
||
log.Warn().Err(err).Msg("failed to flush AI usage cost store")
|
||
}
|
||
}
|
||
|
||
if store != nil {
|
||
if err := store.Close(); err != nil {
|
||
log.Warn().Err(err).Msg("failed to close unified resource export store")
|
||
}
|
||
}
|
||
}
|
||
|
||
// ReconfigurePatrol updates the patrol configuration without restarting
|
||
// Call this after changing patrol settings to apply them immediately
|
||
func (s *Service) ReconfigurePatrol() {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
alertAnalyzer := s.alertTriggeredAnalyzer
|
||
cfg := s.cfg
|
||
licenseChecker := s.licenseChecker
|
||
s.mu.RUnlock()
|
||
|
||
if patrol == nil || cfg == nil {
|
||
return
|
||
}
|
||
|
||
// Update patrol configuration (preserve defaults for resource types not in AI config).
|
||
patrolCfg := s.patrolConfigFromAIConfig(cfg)
|
||
patrol.SetConfig(patrolCfg)
|
||
|
||
// Update proactive threshold mode
|
||
patrol.SetProactiveMode(cfg.UseProactiveThresholds)
|
||
patrol.SetEventTriggerConfig(patrolEventTriggerConfigFromAIConfig(cfg))
|
||
|
||
if BackgroundAutomationDisabledForDev() {
|
||
patrol.SetEventTriggerBlock(BackgroundAutomationEventTriggerBlock())
|
||
patrol.Stop()
|
||
if alertAnalyzer != nil {
|
||
alertAnalyzer.SetEnabled(false)
|
||
}
|
||
log.Info().
|
||
Str("env", DevDisableBackgroundAIEnv).
|
||
Msg("Pulse dev background AI disabled; Patrol background automation stopped")
|
||
return
|
||
}
|
||
|
||
patrol.SetEventTriggerBlock(PatrolEventTriggerBlock{})
|
||
|
||
log.Info().
|
||
Bool("enabled", patrolCfg.Enabled).
|
||
Dur("interval", patrolCfg.Interval).
|
||
Bool("proactiveThresholds", cfg.UseProactiveThresholds).
|
||
Msg("Patrol configuration updated")
|
||
|
||
// Update alert-triggered analyzer (re-check license on each config change)
|
||
if alertAnalyzer != nil {
|
||
enabled := cfg.IsAlertTriggeredAnalysisEnabled()
|
||
// Re-check license - don't allow re-enabling without valid license
|
||
if enabled && licenseChecker != nil && !licenseChecker.HasFeature(FeatureAIAlerts) {
|
||
log.Debug().Msg("alert-triggered analysis requires Pulse Pro license - staying disabled")
|
||
enabled = false
|
||
}
|
||
alertAnalyzer.SetEnabled(enabled)
|
||
}
|
||
}
|
||
|
||
// enrichRequestFromFinding looks up a patrol finding by ID and enriches the request
|
||
// with the finding's context (node, resource ID, resource type, etc.)
|
||
// This ensures proper command routing when the AI helps fix a patrol finding.
|
||
// The function modifies the request in place.
|
||
func (s *Service) enrichRequestFromFinding(req *ExecuteRequest) {
|
||
if req.FindingID == "" {
|
||
return
|
||
}
|
||
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol == nil {
|
||
log.Debug().Str("finding_id", req.FindingID).Msg("cannot enrich request - patrol service not available")
|
||
return
|
||
}
|
||
|
||
findings := patrol.GetFindings()
|
||
if findings == nil {
|
||
log.Debug().Str("finding_id", req.FindingID).Msg("cannot enrich request - findings store not available")
|
||
return
|
||
}
|
||
|
||
finding := findings.Get(req.FindingID)
|
||
if finding == nil {
|
||
log.Debug().Str("finding_id", req.FindingID).Msg("cannot enrich request - finding not found")
|
||
return
|
||
}
|
||
|
||
// Ensure context map exists
|
||
if req.Context == nil {
|
||
req.Context = make(map[string]interface{})
|
||
}
|
||
|
||
// Inject finding context (only if not already set)
|
||
if _, ok := req.Context["node"]; !ok && finding.Node != "" {
|
||
req.Context["node"] = finding.Node
|
||
}
|
||
if _, ok := req.Context["guestName"]; !ok && finding.ResourceName != "" {
|
||
req.Context["guestName"] = finding.ResourceName
|
||
}
|
||
if req.TargetID == "" && finding.ResourceID != "" {
|
||
req.TargetID = finding.ResourceID
|
||
}
|
||
if req.TargetType == "" && finding.ResourceType != "" {
|
||
req.TargetType = finding.ResourceType
|
||
}
|
||
|
||
// Also store the finding details for reference in context
|
||
req.Context["finding_resource_id"] = finding.ResourceID
|
||
req.Context["finding_resource_name"] = finding.ResourceName
|
||
req.Context["finding_resource_type"] = finding.ResourceType
|
||
if finding.Node != "" {
|
||
req.Context["finding_node"] = finding.Node
|
||
}
|
||
|
||
log.Debug().
|
||
Str("finding_id", req.FindingID).
|
||
Str("node", finding.Node).
|
||
Str("resource_id", finding.ResourceID).
|
||
Str("resource_name", finding.ResourceName).
|
||
Str("resource_type", finding.ResourceType).
|
||
Msg("Enriched request with finding context")
|
||
}
|
||
|
||
// GuestInfo contains information about a guest (VM or container) found by VMID lookup
|
||
type GuestInfo struct {
|
||
Node string
|
||
Name string
|
||
Type string // semantic type: "system-container" or "vm"
|
||
Technology string // implementation: "lxc", "qemu", etc.
|
||
Instance string // PVE instance ID (for multi-cluster disambiguation)
|
||
}
|
||
|
||
// lookupNodeForVMID looks up which node owns a given VMID using the state provider
|
||
// Returns the node name and guest name if found, empty strings otherwise
|
||
// If targetInstance is provided, only matches guests from that instance (for multi-cluster setups)
|
||
func (s *Service) lookupNodeForVMID(vmID int) (node string, guestName string, guestType string) {
|
||
guests := s.lookupGuestsByVMID(vmID, "")
|
||
if len(guests) == 1 {
|
||
return guests[0].Node, guests[0].Name, guests[0].Type
|
||
}
|
||
if len(guests) > 1 {
|
||
// Multiple matches - VMID collision across instances
|
||
// Log warning and return first match (caller should use lookupGuestsByVMID with instance filter)
|
||
log.Warn().
|
||
Int("vmid", vmID).
|
||
Int("matches", len(guests)).
|
||
Msg("VMID collision detected - multiple guests with same VMID across instances")
|
||
return guests[0].Node, guests[0].Name, guests[0].Type
|
||
}
|
||
return "", "", ""
|
||
}
|
||
|
||
// lookupGuestsByVMID finds all guests with the given VMID
|
||
// If targetInstance is non-empty, only returns guests from that instance
|
||
func (s *Service) lookupGuestsByVMID(vmID int, targetInstance string) []GuestInfo {
|
||
s.mu.RLock()
|
||
rs := s.readState
|
||
s.mu.RUnlock()
|
||
|
||
if rs == nil {
|
||
return nil
|
||
}
|
||
|
||
var results []GuestInfo
|
||
|
||
// Check containers via ReadState
|
||
for _, ct := range rs.Containers() {
|
||
if ct.VMID() == vmID {
|
||
if targetInstance == "" || ct.Instance() == targetInstance {
|
||
results = append(results, GuestInfo{
|
||
Node: ct.Node(),
|
||
Name: ct.Name(),
|
||
Type: "system-container",
|
||
Technology: "lxc",
|
||
Instance: ct.Instance(),
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
// Check VMs via ReadState
|
||
for _, vm := range rs.VMs() {
|
||
if vm.VMID() == vmID {
|
||
if targetInstance == "" || vm.Instance() == targetInstance {
|
||
results = append(results, GuestInfo{
|
||
Node: vm.Node(),
|
||
Name: vm.Name(),
|
||
Type: "vm",
|
||
Technology: "qemu",
|
||
Instance: vm.Instance(),
|
||
})
|
||
}
|
||
}
|
||
}
|
||
|
||
return results
|
||
}
|
||
|
||
// extractVMIDFromCommand parses pct/qm/vzdump commands to extract the VMID being targeted
|
||
// Returns the VMID, whether it requires node-specific routing, and whether found
|
||
// Some commands (like vzdump) can run from any cluster node, others (like pct exec) must run on the owning node
|
||
func extractVMIDFromCommand(command string) (vmID int, requiresOwnerNode bool, found bool) {
|
||
// Commands that MUST run on the node that owns the guest
|
||
// These interact directly with the container/VM runtime
|
||
nodeSpecificPatterns := []string{
|
||
// pct commands - match any pct subcommand followed by VMID
|
||
// Covers: exec, enter, start, stop, shutdown, reboot, status, push, pull, mount, unmount, etc.
|
||
`(?:^|\s)pct\s+\w+\s+(\d+)`,
|
||
// qm commands - match any qm subcommand followed by VMID
|
||
// Covers: start, stop, shutdown, reset, status, guest exec, monitor, etc.
|
||
`(?:^|\s)qm\s+(?:guest\s+)?\w+\s+(\d+)`,
|
||
}
|
||
|
||
// Commands that can run from any cluster node (cluster-aware)
|
||
// vzdump uses the cluster to route to the right node automatically
|
||
clusterAwarePatterns := []string{
|
||
`(?:^|\s)vzdump\s+(\d+)`,
|
||
// pvesh commands can specify node in path, so we don't force routing
|
||
}
|
||
|
||
// Check node-specific commands first (higher priority)
|
||
for _, pattern := range nodeSpecificPatterns {
|
||
re := regexp.MustCompile(pattern)
|
||
if matches := re.FindStringSubmatch(command); len(matches) > 1 {
|
||
if v, err := strconv.Atoi(matches[1]); err == nil {
|
||
return v, true, true
|
||
}
|
||
}
|
||
}
|
||
|
||
// Check cluster-aware commands (don't force node routing)
|
||
for _, pattern := range clusterAwarePatterns {
|
||
re := regexp.MustCompile(pattern)
|
||
if matches := re.FindStringSubmatch(command); len(matches) > 1 {
|
||
if v, err := strconv.Atoi(matches[1]); err == nil {
|
||
return v, false, true
|
||
}
|
||
}
|
||
}
|
||
|
||
return 0, false, false
|
||
}
|
||
|
||
// formatApprovalNeededToolResult returns a structured tool marker for commands
|
||
// that require approval. The shared marker vocabulary lives in
|
||
// agentcapabilities so Assistant and external-agent bridges cannot drift.
|
||
func formatApprovalNeededToolResult(command, toolID, reason, approvalID string) string {
|
||
return agentcapabilities.ApprovalRequiredToolMarker(
|
||
command,
|
||
toolID,
|
||
reason,
|
||
approvalID,
|
||
"Ask the user to click the approval button shown in the UI.",
|
||
)
|
||
}
|
||
|
||
func resolveRunCommandApprovalTarget(req ExecuteRequest, runOnHost bool, targetHost string) (targetType, targetID, targetName string) {
|
||
normalizedTargetHost := strings.ToLower(strings.TrimSpace(targetHost))
|
||
targetType = agentcapabilities.NormalizeActionTargetType(req.TargetType)
|
||
targetID = strings.TrimSpace(req.TargetID)
|
||
|
||
if runOnHost {
|
||
targetType = agentcapabilities.ActionTargetTypeAgent
|
||
if normalizedTargetHost != "" {
|
||
targetID = normalizedTargetHost
|
||
targetName = normalizedTargetHost
|
||
}
|
||
}
|
||
|
||
if targetType == "" {
|
||
targetType = agentcapabilities.ActionTargetTypeAgent
|
||
}
|
||
|
||
if targetType == agentcapabilities.ActionTargetTypeAgent && targetID == "" {
|
||
if node, ok := req.Context["node"].(string); ok && node != "" {
|
||
targetID = strings.ToLower(strings.TrimSpace(node))
|
||
} else if node, ok := req.Context["hostname"].(string); ok && node != "" {
|
||
targetID = strings.ToLower(strings.TrimSpace(node))
|
||
} else if node, ok := req.Context["host_name"].(string); ok && node != "" {
|
||
targetID = strings.ToLower(strings.TrimSpace(node))
|
||
}
|
||
}
|
||
|
||
if targetName == "" {
|
||
targetName = strings.TrimSpace(targetHost)
|
||
}
|
||
if targetName == "" {
|
||
targetName = targetID
|
||
}
|
||
|
||
return targetType, targetID, targetName
|
||
}
|
||
|
||
func createRunCommandApprovalRecord(orgID, command, toolID string, req ExecuteRequest, runOnHost bool, targetHost, reason string) string {
|
||
store := approval.GetStore()
|
||
if store == nil {
|
||
log.Debug().Msg("approval store not available, run_command approval will not be persisted")
|
||
return ""
|
||
}
|
||
|
||
targetType, targetID, targetName := resolveRunCommandApprovalTarget(req, runOnHost, targetHost)
|
||
approvalReq := &approval.ApprovalRequest{
|
||
OrgID: strings.TrimSpace(orgID),
|
||
ToolID: toolID,
|
||
Command: command,
|
||
TargetType: targetType,
|
||
TargetID: targetID,
|
||
TargetName: targetName,
|
||
Context: reason,
|
||
}
|
||
if err := store.CreateApproval(approvalReq); err != nil {
|
||
log.Warn().Err(err).Msg("failed to create run_command approval record")
|
||
return ""
|
||
}
|
||
|
||
return approvalReq.ID
|
||
}
|
||
|
||
// formatPolicyBlockedToolResult returns a structured tool marker for commands
|
||
// blocked by policy.
|
||
func formatPolicyBlockedToolResult(command, reason string) string {
|
||
return agentcapabilities.PolicyBlockedToolMarker(command, reason)
|
||
}
|
||
|
||
func approvalNeededFromToolCall(req ExecuteRequest, tc providers.ToolCall, result string) (ApprovalNeededData, bool) {
|
||
if !agentcapabilities.HasApprovalRequiredToolMarker(result) {
|
||
return ApprovalNeededData{}, false
|
||
}
|
||
if tc.Name != agentcapabilities.LegacyAssistantRunCommandToolName {
|
||
return ApprovalNeededData{}, false
|
||
}
|
||
|
||
parsed, parsedOK := agentcapabilities.ParseApprovalRequiredToolMarkerData(result)
|
||
if parsedOK && strings.TrimSpace(parsed.Command) == "" {
|
||
parsedOK = false
|
||
}
|
||
cmd, _ := tc.Input[agentcapabilities.LegacyAssistantCommandArgumentName].(string)
|
||
runOnHost, _ := tc.Input[agentcapabilities.LegacyAssistantRunOnHostArgumentName].(bool)
|
||
targetHost, _ := tc.Input[agentcapabilities.LegacyAssistantTargetHostArgumentName].(string)
|
||
|
||
if targetHost == "" {
|
||
if node, ok := req.Context["node"].(string); ok && node != "" {
|
||
targetHost = node
|
||
} else if node, ok := req.Context["hostname"].(string); ok && node != "" {
|
||
targetHost = node
|
||
} else if node, ok := req.Context["host_name"].(string); ok && node != "" {
|
||
targetHost = node
|
||
}
|
||
}
|
||
|
||
if parsedOK {
|
||
cmd = parsed.Command
|
||
toolID := strings.TrimSpace(parsed.ToolID)
|
||
if toolID == "" {
|
||
toolID = tc.ID
|
||
}
|
||
return ApprovalNeededData{
|
||
Command: cmd,
|
||
ToolID: toolID,
|
||
ToolName: tc.Name,
|
||
RunOnHost: runOnHost,
|
||
TargetHost: targetHost,
|
||
ApprovalID: strings.TrimSpace(parsed.ApprovalID),
|
||
}, true
|
||
}
|
||
|
||
return ApprovalNeededData{
|
||
Command: cmd,
|
||
ToolID: tc.ID,
|
||
ToolName: tc.Name,
|
||
RunOnHost: runOnHost,
|
||
TargetHost: targetHost,
|
||
}, true
|
||
}
|
||
|
||
// LoadConfig loads the AI configuration and initializes the provider
|
||
func (s *Service) LoadConfig() error {
|
||
s.mu.RLock()
|
||
persistence := s.persistence
|
||
orgID := s.orgID
|
||
s.mu.RUnlock()
|
||
|
||
if persistence == nil {
|
||
s.mu.Lock()
|
||
s.provider = nil
|
||
s.cfg = nil
|
||
s.mu.Unlock()
|
||
return fmt.Errorf("Pulse Assistant config persistence unavailable")
|
||
}
|
||
|
||
cfg, err := persistence.LoadAIConfig()
|
||
if err != nil {
|
||
return fmt.Errorf("failed to load Pulse Assistant config: %w", err)
|
||
}
|
||
var providerClient providers.Provider
|
||
var providerInitErr string
|
||
|
||
// Don't initialize provider if AI is not enabled or not configured
|
||
if cfg != nil && cfg.Enabled && cfg.IsConfigured() {
|
||
providerClient, providerInitErr = buildConfiguredProvider(context.Background(), cfg, orgID, true)
|
||
}
|
||
|
||
s.mu.Lock()
|
||
s.cfg = cfg
|
||
s.provider = providerClient
|
||
s.providerInitErr = providerInitErr
|
||
s.mu.Unlock()
|
||
|
||
return nil
|
||
}
|
||
|
||
// buildConfiguredProvider resolves the effective model and constructs the
|
||
// default provider client for an enabled, configured AI config. It returns a
|
||
// nil provider plus a redacted failure summary when either step fails — for
|
||
// example when model resolution needs the provider's live catalog and that
|
||
// server is still booting. Callers own storing the result under s.mu.
|
||
// stampModel controls whether the resolved model is written back to cfg.Model:
|
||
// LoadConfig owns the config instance it just loaded, while retry callers
|
||
// share s.cfg and must not mutate it without the lock.
|
||
func buildConfiguredProvider(ctx context.Context, cfg *config.AIConfig, orgID string, stampModel bool) (providers.Provider, string) {
|
||
selectedModel, resolveErr := ResolveConfiguredModel(ctx, cfg)
|
||
if resolveErr != nil {
|
||
log.Warn().Err(resolveErr).Str("orgID", orgID).Msg("AI enabled but no effective provider model could be resolved")
|
||
return nil, summarizePatrolRuntimeFailureDetail(resolveErr.Error(), false)
|
||
}
|
||
if stampModel {
|
||
cfg.Model = selectedModel
|
||
}
|
||
selectedProvider, _ := config.ParseModelString(selectedModel)
|
||
|
||
nextProvider, providerErr := providers.NewForModel(cfg, selectedModel)
|
||
if providerErr != nil {
|
||
log.Warn().
|
||
Err(providerErr).
|
||
Str("selected_model", selectedModel).
|
||
Str("selected_provider", selectedProvider).
|
||
Strs("configured_providers", cfg.GetConfiguredProviders()).
|
||
Msg("AI enabled but selected provider could not be initialized")
|
||
return nil, summarizePatrolRuntimeFailureDetail(providerErr.Error(), false)
|
||
}
|
||
|
||
log.Info().
|
||
Str("provider", selectedProvider).
|
||
Str("model", selectedModel).
|
||
Str("control_level", cfg.GetControlLevel()).
|
||
Bool("autonomous", cfg.IsAutonomous()).
|
||
Msg("AI service initialized")
|
||
return nextProvider, ""
|
||
}
|
||
|
||
// ProviderInitError returns the redacted reason the default provider could not
|
||
// be initialized, or "" when the provider is healthy or intentionally absent.
|
||
func (s *Service) ProviderInitError() string {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
return s.providerInitErr
|
||
}
|
||
|
||
// RetryProviderInit rebuilds the default provider when the config claims an
|
||
// enabled, configured provider but initialization previously failed. Provider
|
||
// construction can depend on a live model-catalog call, so a boot-time race
|
||
// (Pulse starting before a local Ollama server) must not strand the runtime
|
||
// until the next settings save; Patrol calls this once per scheduled run.
|
||
// Returns true when a usable provider is available afterwards.
|
||
func (s *Service) RetryProviderInit(ctx context.Context) bool {
|
||
s.mu.RLock()
|
||
cfg := s.cfg
|
||
provider := s.provider
|
||
orgID := s.orgID
|
||
s.mu.RUnlock()
|
||
|
||
if provider != nil {
|
||
return true
|
||
}
|
||
if cfg == nil || !cfg.Enabled || !cfg.IsConfigured() {
|
||
return false
|
||
}
|
||
|
||
nextProvider, initErr := buildConfiguredProvider(ctx, cfg, orgID, false)
|
||
|
||
s.mu.Lock()
|
||
defer s.mu.Unlock()
|
||
// A concurrent LoadConfig may have replaced provider or cfg; never clobber
|
||
// a healthy provider with a failed retry.
|
||
if s.provider != nil {
|
||
return true
|
||
}
|
||
if s.cfg != cfg {
|
||
return s.provider != nil
|
||
}
|
||
s.provider = nextProvider
|
||
s.providerInitErr = initErr
|
||
return nextProvider != nil
|
||
}
|
||
|
||
// IsEnabled returns true if AI is enabled and configured
|
||
func (s *Service) IsEnabled() bool {
|
||
// In demo mode, AI is always considered enabled (using mock backend)
|
||
if IsDemoMode() {
|
||
return true
|
||
}
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
if s.cfg != nil && s.cfg.Enabled && s.provider != nil {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
type patrolProviderFactorySetter interface {
|
||
SetPatrolProviderFactory(func(modelStr string) (providers.StreamingProvider, error))
|
||
}
|
||
|
||
func (s *Service) configurePatrolProviderFactoryLocked() {
|
||
setter, ok := s.chatService.(patrolProviderFactorySetter)
|
||
if !ok {
|
||
return
|
||
}
|
||
setter.SetPatrolProviderFactory(s.createPatrolProviderForModel)
|
||
}
|
||
|
||
func (s *Service) createPatrolProviderForModel(modelStr string) (providers.StreamingProvider, error) {
|
||
s.mu.RLock()
|
||
cfg := s.cfg
|
||
defaultProvider := s.provider
|
||
s.mu.RUnlock()
|
||
|
||
providerName, _ := config.ParseModelString(modelStr)
|
||
if providerName == config.AIProviderQuickstart {
|
||
return nil, fmt.Errorf("quickstart provider is retired; configure a provider API key or Ollama")
|
||
}
|
||
|
||
provider, err := providers.NewForModel(cfg, modelStr)
|
||
if err != nil {
|
||
if streaming, ok := defaultProvider.(providers.StreamingProvider); ok {
|
||
return streaming, nil
|
||
}
|
||
return nil, err
|
||
}
|
||
streaming, ok := provider.(providers.StreamingProvider)
|
||
if !ok {
|
||
return nil, fmt.Errorf("patrol provider %q does not support streaming", provider.Name())
|
||
}
|
||
return streaming, nil
|
||
}
|
||
|
||
// QuickAnalysis performs a lightweight AI analysis for simple decisions.
|
||
// This is used for things like determining if an alert should be auto-resolved.
|
||
// It uses a single-turn, no-tools call for efficiency.
|
||
func (s *Service) QuickAnalysis(ctx context.Context, req QuickAnalysisRequest) (string, error) {
|
||
s.mu.RLock()
|
||
provider := s.provider
|
||
cfg := s.cfg
|
||
costStore := s.costStore
|
||
s.mu.RUnlock()
|
||
|
||
if provider == nil {
|
||
return "", fmt.Errorf("Pulse Assistant is not enabled or configured")
|
||
}
|
||
|
||
// Use a fast model for quick analysis if available
|
||
model := ""
|
||
if cfg != nil && cfg.PatrolModel != "" {
|
||
model = cfg.PatrolModel
|
||
}
|
||
sanitizerModel := model
|
||
if sanitizerModel == "" && cfg != nil {
|
||
sanitizerModel = cfg.GetChatModel()
|
||
}
|
||
|
||
messages := []providers.Message{
|
||
{
|
||
Role: "system",
|
||
Content: "You are a brief, decisive infrastructure monitoring assistant. Give short, direct answers.",
|
||
},
|
||
{
|
||
Role: "user",
|
||
Content: req.Prompt,
|
||
},
|
||
}
|
||
|
||
executionID := strings.TrimSpace(req.ExecutionID)
|
||
if executionID == "" {
|
||
executionID = uuid.NewString()
|
||
}
|
||
|
||
chatReq := providers.ChatRequest{
|
||
Messages: messages,
|
||
Model: model,
|
||
ExecutionID: executionID,
|
||
}
|
||
if sanitizer := s.requestSanitizerForModel(sanitizerModel); sanitizer != nil {
|
||
chatReq = sanitizer(chatReq)
|
||
}
|
||
|
||
useCase := strings.TrimSpace(req.UseCase)
|
||
if useCase == "" {
|
||
useCase = "quick_analysis"
|
||
}
|
||
|
||
resp, err := provider.Chat(ctx, chatReq)
|
||
if err != nil {
|
||
return "", fmt.Errorf("Pulse Assistant analysis failed: %w", err)
|
||
}
|
||
|
||
// Record token usage in the operator-facing cost ledger so quick
|
||
// analysis (alert auto-resolve, etc.) shows up alongside chat and
|
||
// patrol spend. Recording happens before the empty-content guard
|
||
// because the operator was billed regardless.
|
||
if costStore != nil {
|
||
providerName, _ := config.ParseModelString(model)
|
||
if providerName == "" {
|
||
providerName = provider.Name()
|
||
}
|
||
costStore.Record(cost.UsageEvent{
|
||
Timestamp: time.Now(),
|
||
Provider: providerName,
|
||
RequestModel: model,
|
||
ResponseModel: resp.Model,
|
||
UseCase: useCase,
|
||
TargetType: strings.TrimSpace(req.TargetType),
|
||
InputTokens: resp.InputTokens,
|
||
OutputTokens: resp.OutputTokens,
|
||
CacheCreationInputTokens: resp.CacheCreationInputTokens,
|
||
CacheReadInputTokens: resp.CacheReadInputTokens,
|
||
})
|
||
}
|
||
|
||
if resp.Content == "" {
|
||
return "", fmt.Errorf("Pulse Assistant returned empty response")
|
||
}
|
||
|
||
return resp.Content, nil
|
||
}
|
||
|
||
func (s *Service) requestSanitizerForModel(model string) func(providers.ChatRequest) providers.ChatRequest {
|
||
s.mu.RLock()
|
||
urp := s.unifiedResourceProvider
|
||
s.mu.RUnlock()
|
||
return modelboundary.RequestSanitizerForModel(model, urp)
|
||
}
|
||
|
||
// GetConfig returns a copy of the current AI config
|
||
func (s *Service) GetConfig() *config.AIConfig {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
if s.cfg == nil {
|
||
return nil
|
||
}
|
||
cfg := *s.cfg
|
||
cfg.PatrolAutopilotAcknowledgements = clonePatrolAutopilotAcknowledgements(s.cfg.PatrolAutopilotAcknowledgements)
|
||
cfg.PatrolAutopilotRevocations = append([]unifiedresources.PatrolAutopilotRevocation(nil), s.cfg.PatrolAutopilotRevocations...)
|
||
if s.cfg.PatrolAutopilotActivation != nil {
|
||
activation := *s.cfg.PatrolAutopilotActivation
|
||
cfg.PatrolAutopilotActivation = &activation
|
||
}
|
||
return &cfg
|
||
}
|
||
|
||
// GetDebugContext returns debug information about what context would be sent to the AI
|
||
func (s *Service) GetDebugContext(req ExecuteRequest) map[string]interface{} {
|
||
s.mu.RLock()
|
||
rs := s.readState
|
||
agentServer := s.agentServer
|
||
cfg := s.cfg
|
||
s.mu.RUnlock()
|
||
|
||
result := make(map[string]interface{})
|
||
|
||
// ReadState info
|
||
result["has_state_provider"] = rs != nil
|
||
if rs != nil {
|
||
nodes := rs.Nodes()
|
||
vms := rs.VMs()
|
||
containers := rs.Containers()
|
||
dockerHosts := rs.DockerHosts()
|
||
hosts := rs.Hosts()
|
||
pbsInstances := rs.PBSInstances()
|
||
|
||
result["state_summary"] = map[string]interface{}{
|
||
"nodes": len(nodes),
|
||
"vms": len(vms),
|
||
"containers": len(containers),
|
||
"docker_hosts": len(dockerHosts),
|
||
"hosts": len(hosts),
|
||
"pbs_instances": len(pbsInstances),
|
||
}
|
||
|
||
// List some VMs/containers for verification
|
||
var vmNames []string
|
||
for _, vm := range vms {
|
||
vmNames = append(vmNames, fmt.Sprintf("%s (VMID:%d, node:%s)", vm.Name(), vm.VMID(), vm.Node()))
|
||
}
|
||
if len(vmNames) > 10 {
|
||
vmNames = vmNames[:10]
|
||
}
|
||
result["sample_vms"] = vmNames
|
||
|
||
var ctNames []string
|
||
for _, ct := range containers {
|
||
ctNames = append(ctNames, fmt.Sprintf("%s (VMID:%d, node:%s)", ct.Name(), ct.VMID(), ct.Node()))
|
||
}
|
||
if len(ctNames) > 10 {
|
||
ctNames = ctNames[:10]
|
||
}
|
||
result["sample_containers"] = ctNames
|
||
|
||
var hostNames []string
|
||
for _, h := range hosts {
|
||
hostNames = append(hostNames, h.Hostname())
|
||
}
|
||
result["host_names"] = hostNames
|
||
|
||
var dockerHostNames []string
|
||
for _, dh := range dockerHosts {
|
||
dockerHostNames = append(dockerHostNames, fmt.Sprintf("%s (%d containers)", dh.Name(), dh.ChildCount()))
|
||
}
|
||
result["docker_host_names"] = dockerHostNames
|
||
}
|
||
|
||
// Agent info
|
||
result["has_agent_server"] = agentServer != nil
|
||
if agentServer != nil {
|
||
agents := agentServer.GetConnectedAgents()
|
||
var agentNames []string
|
||
for _, a := range agents {
|
||
agentNames = append(agentNames, a.Hostname)
|
||
}
|
||
result["connected_agents"] = agentNames
|
||
}
|
||
|
||
// Config info
|
||
result["has_config"] = cfg != nil
|
||
if cfg != nil {
|
||
result["custom_context_length"] = len(cfg.CustomContext)
|
||
if len(cfg.CustomContext) > 200 {
|
||
result["custom_context_preview"] = cfg.CustomContext[:200] + "..."
|
||
} else {
|
||
result["custom_context_preview"] = cfg.CustomContext
|
||
}
|
||
}
|
||
|
||
// Build and include the system prompt
|
||
systemPrompt := s.buildSystemPrompt(req, "")
|
||
result["system_prompt_length"] = len(systemPrompt)
|
||
result["system_prompt"] = systemPrompt
|
||
|
||
return result
|
||
}
|
||
|
||
// IsAutonomous returns true if autonomous mode is enabled AND licensed.
|
||
// Autonomous mode requires the ai_autofix license feature (Pro tier).
|
||
func (s *Service) IsAutonomous() bool {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
if s.cfg == nil || !s.cfg.IsAutonomous() {
|
||
return false
|
||
}
|
||
// Autonomous mode requires Pro license with ai_autofix feature
|
||
if s.licenseChecker != nil && !s.licenseChecker.HasFeature(FeatureAIAutoFix) {
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
func (s *Service) isAutonomousForRequest(req ExecuteRequest) bool {
|
||
if req.AutonomousMode != nil && !*req.AutonomousMode {
|
||
return false
|
||
}
|
||
return s.IsAutonomous()
|
||
}
|
||
|
||
// ConversationMessage represents a message in conversation history
|
||
type ConversationMessage struct {
|
||
Role string `json:"role"` // "user" or "assistant"
|
||
Content string `json:"content"`
|
||
}
|
||
|
||
// ExecuteRequest represents a request to execute an AI prompt
|
||
type ExecuteRequest struct {
|
||
Prompt string `json:"prompt"`
|
||
TargetType string `json:"target_type,omitempty"` // "agent", "system-container", "vm"
|
||
TargetID string `json:"target_id,omitempty"`
|
||
Context map[string]interface{} `json:"context,omitempty"` // Current metrics, state, etc.
|
||
SystemPrompt string `json:"system_prompt,omitempty"` // Override system prompt
|
||
History []ConversationMessage `json:"history,omitempty"` // Previous conversation messages
|
||
FindingID string `json:"finding_id,omitempty"` // If fixing a patrol finding, the ID to resolve
|
||
Model string `json:"model,omitempty"` // Override model for this request (for user selection in chat)
|
||
UseCase string `json:"use_case,omitempty"` // "chat" or "patrol" - determines which default model to use
|
||
AutonomousMode *bool `json:"autonomous_mode,omitempty"` // Per-request execution override; false keeps scoped handoffs approval-required
|
||
RequireCommandApproval bool `json:"require_command_approval,omitempty"` // Force every run_command tool call through operator approval
|
||
}
|
||
|
||
func assistantContextScopeForExecuteRequest(req ExecuteRequest) string {
|
||
if strings.TrimSpace(req.FindingID) != "" {
|
||
return "patrol_finding"
|
||
}
|
||
if strings.TrimSpace(req.TargetType) != "" || strings.TrimSpace(req.TargetID) != "" {
|
||
return "resource_context"
|
||
}
|
||
if len(req.Context) > 0 {
|
||
return "operator_context"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// ExecuteResponse represents the AI's response
|
||
type ExecuteResponse struct {
|
||
Content string `json:"content"`
|
||
Model string `json:"model"`
|
||
InputTokens int `json:"input_tokens"`
|
||
OutputTokens int `json:"output_tokens"`
|
||
// Prompt-cache buckets summed across the loop's provider calls; disjoint
|
||
// from InputTokens, see providers.ChatResponse.
|
||
CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"`
|
||
CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"`
|
||
ToolCalls []ToolExecution `json:"tool_calls"` // Commands that were executed
|
||
PendingApprovals []ApprovalNeededData `json:"pending_approvals"` // Commands that require approval (non-streaming)
|
||
}
|
||
|
||
func EmptyExecuteResponse() ExecuteResponse {
|
||
return ExecuteResponse{}.NormalizeCollections()
|
||
}
|
||
|
||
func (r ExecuteResponse) NormalizeCollections() ExecuteResponse {
|
||
if r.ToolCalls == nil {
|
||
r.ToolCalls = []ToolExecution{}
|
||
}
|
||
if r.PendingApprovals == nil {
|
||
r.PendingApprovals = []ApprovalNeededData{}
|
||
}
|
||
return r
|
||
}
|
||
|
||
// ToolExecution represents a tool that was executed during the AI conversation
|
||
type ToolExecution struct {
|
||
Name string `json:"name"`
|
||
Input string `json:"input"` // Human-readable input (e.g., the command)
|
||
Output string `json:"output"` // Result of execution
|
||
Success bool `json:"success"`
|
||
}
|
||
|
||
// getModelForRequest determines which model to use for a request
|
||
// Priority: explicit override > use case default > config default
|
||
func (s *Service) getModelForRequest(req ExecuteRequest) string {
|
||
// If request has explicit model override, use it
|
||
if req.Model != "" {
|
||
return req.Model
|
||
}
|
||
|
||
s.mu.RLock()
|
||
cfg := s.cfg
|
||
s.mu.RUnlock()
|
||
|
||
if cfg == nil {
|
||
return ""
|
||
}
|
||
|
||
// Use case-specific model selection
|
||
switch req.UseCase {
|
||
case "patrol":
|
||
return cfg.GetPatrolModel()
|
||
case "chat":
|
||
return cfg.GetChatModel()
|
||
default:
|
||
// Default to chat model for interactive requests
|
||
return cfg.GetChatModel()
|
||
}
|
||
}
|
||
|
||
// StreamEvent represents an event during AI execution for streaming
|
||
type StreamEvent struct {
|
||
Type string `json:"type"` // "tool_start", "tool_end", "content", "done", "error", "approval_needed"
|
||
Data interface{} `json:"data,omitempty"`
|
||
}
|
||
|
||
// StreamCallback is called for each event during streaming execution
|
||
type StreamCallback func(event StreamEvent)
|
||
|
||
// ToolStartData is sent when a tool execution begins
|
||
type ToolStartData struct {
|
||
Name string `json:"name"`
|
||
Input string `json:"input"`
|
||
}
|
||
|
||
// ToolEndData is sent when a tool execution completes
|
||
type ToolEndData struct {
|
||
Name string `json:"name"`
|
||
Input string `json:"input"`
|
||
Output string `json:"output"`
|
||
Success bool `json:"success"`
|
||
}
|
||
|
||
// ApprovalNeededData is sent when a command needs user approval
|
||
type ApprovalNeededData struct {
|
||
Command string `json:"command"`
|
||
ToolID string `json:"tool_id"` // ID to reference when approving
|
||
ToolName string `json:"tool_name"` // "run_command"
|
||
RunOnHost bool `json:"run_on_host"`
|
||
TargetHost string `json:"target_host,omitempty"` // Explicit host to route to
|
||
ApprovalID string `json:"approval_id,omitempty"` // ID of the approval record in the store
|
||
}
|
||
|
||
// Execute sends a prompt to the AI and returns the response
|
||
// If tools are available and the AI requests them, it executes them in a loop
|
||
func (s *Service) Execute(ctx context.Context, req ExecuteRequest) (*ExecuteResponse, error) {
|
||
release, err := s.acquireExecutionSlot(ctx, req.UseCase)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer release()
|
||
|
||
if err := s.enforceBudget(req.UseCase); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Enrich request with finding context if this is a "help me fix" request
|
||
s.enrichRequestFromFinding(&req)
|
||
|
||
s.mu.RLock()
|
||
defaultProvider := s.provider
|
||
agentServer := s.agentServer
|
||
cfg := s.cfg
|
||
costStore := s.costStore
|
||
s.mu.RUnlock()
|
||
|
||
// Determine the model to use for this request
|
||
modelString := s.getModelForRequest(req)
|
||
requestSanitizer := s.requestSanitizerForModel(modelString)
|
||
|
||
// Create a provider for this specific model (supports multi-provider switching)
|
||
provider, err := providers.NewForModel(cfg, modelString)
|
||
if err != nil {
|
||
// Fall back to default provider if model-specific provider can't be created
|
||
log.Debug().Err(err).Str("model", modelString).Msg("could not create provider for model, using default")
|
||
provider = defaultProvider
|
||
}
|
||
|
||
if provider == nil {
|
||
// In demo mode, return mock response if no provider is configured
|
||
if IsDemoMode() {
|
||
return GenerateDemoAIResponse(req.Prompt), nil
|
||
}
|
||
return nil, fmt.Errorf("Pulse Assistant is not enabled or configured")
|
||
}
|
||
|
||
// Build the system prompt
|
||
systemPrompt := req.SystemPrompt
|
||
if systemPrompt == "" {
|
||
systemPrompt = s.buildSystemPrompt(req, modelString)
|
||
}
|
||
|
||
// Check if agent is available for this target
|
||
hasAgent := s.hasAgentForTarget(req)
|
||
|
||
// Build tools list if agent is available
|
||
var tools []providers.Tool
|
||
if hasAgent && agentServer != nil {
|
||
tools = s.getTools()
|
||
systemPrompt += `
|
||
|
||
## Available Tools
|
||
You have access to tools to execute commands on the target system. You should:
|
||
1. Use run_command to investigate issues, gather information, and PERFORM actions
|
||
2. Actually execute the commands - don't just explain what commands to run
|
||
3. For Proxmox operations (resize disk, manage containers/VMs), run commands on the AGENT (target_type=agent)
|
||
4. For operations inside a system container, use target_type=system-container
|
||
|
||
Examples of actions you can perform:
|
||
- Resize LXC disk: pct resize <vmid> rootfs +10G (run on host)
|
||
- Check disk usage: df -h (run on container)
|
||
- View processes: ps aux --sort=-%mem | head -20
|
||
- Check logs: tail -100 /var/log/syslog
|
||
|
||
Always execute the commands rather than telling the user how to do it.`
|
||
}
|
||
|
||
// Inject previously learned knowledge about this guest
|
||
if s.knowledgeStore != nil {
|
||
guestID := s.getGuestID(req)
|
||
if guestID != "" {
|
||
if knowledgeContext := s.knowledgeStore.FormatForContext(guestID); knowledgeContext != "" {
|
||
systemPrompt += knowledgeContext
|
||
}
|
||
}
|
||
}
|
||
|
||
// Build initial messages with conversation history
|
||
var messages []providers.Message
|
||
for _, histMsg := range req.History {
|
||
messages = append(messages, providers.Message{
|
||
Role: histMsg.Role,
|
||
Content: histMsg.Content,
|
||
})
|
||
}
|
||
messages = append(messages, providers.Message{Role: "user", Content: req.Prompt})
|
||
|
||
var toolExecutions []ToolExecution
|
||
totalInputTokens := 0
|
||
totalOutputTokens := 0
|
||
totalCacheCreationTokens := 0
|
||
totalCacheReadTokens := 0
|
||
var finalContent string
|
||
var model string
|
||
|
||
// Agentic loop - keep going while AI requests tools within the shared
|
||
// provider-call budget.
|
||
loopGuard := newServiceToolLoopGuard()
|
||
for {
|
||
if err := s.enforceBudget(req.UseCase); err != nil {
|
||
return nil, err
|
||
}
|
||
if err := loopGuard.beforeProviderCall(); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
chatReq := providers.ChatRequest{
|
||
Messages: messages,
|
||
Model: modelString,
|
||
System: systemPrompt,
|
||
MaxTokens: 4096,
|
||
Tools: tools,
|
||
}
|
||
if requestSanitizer != nil {
|
||
chatReq = requestSanitizer(chatReq)
|
||
}
|
||
|
||
resp, err := provider.Chat(ctx, chatReq)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Pulse Assistant request failed: %w", err)
|
||
}
|
||
|
||
if costStore != nil {
|
||
providerName, _ := config.ParseModelString(modelString)
|
||
if providerName == "" {
|
||
providerName = provider.Name()
|
||
}
|
||
costStore.Record(cost.UsageEvent{
|
||
Timestamp: time.Now(),
|
||
Provider: providerName,
|
||
RequestModel: modelString,
|
||
ResponseModel: resp.Model,
|
||
UseCase: req.UseCase,
|
||
ContextScope: assistantContextScopeForExecuteRequest(req),
|
||
InputTokens: resp.InputTokens,
|
||
OutputTokens: resp.OutputTokens,
|
||
CacheCreationInputTokens: resp.CacheCreationInputTokens,
|
||
CacheReadInputTokens: resp.CacheReadInputTokens,
|
||
TargetType: req.TargetType,
|
||
TargetID: req.TargetID,
|
||
FindingID: req.FindingID,
|
||
})
|
||
}
|
||
|
||
totalInputTokens += resp.InputTokens
|
||
totalOutputTokens += resp.OutputTokens
|
||
totalCacheCreationTokens += resp.CacheCreationInputTokens
|
||
totalCacheReadTokens += resp.CacheReadInputTokens
|
||
model = resp.Model
|
||
finalContent = resp.Content
|
||
|
||
// If no tool calls, we're done
|
||
if len(resp.ToolCalls) == 0 || resp.StopReason != "tool_use" {
|
||
break
|
||
}
|
||
if err := s.observeDeniedServiceToolCalls(loopGuard, resp.ToolCalls); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Add assistant's response with tool calls to messages
|
||
messages = append(messages, providers.Message{
|
||
Role: "assistant",
|
||
Content: resp.Content,
|
||
ReasoningContent: resp.ReasoningContent, // DeepSeek thinking mode
|
||
ToolCalls: resp.ToolCalls,
|
||
})
|
||
|
||
// Execute each tool call and add results
|
||
var pendingApprovals []ApprovalNeededData
|
||
for _, tc := range resp.ToolCalls {
|
||
result, execution := s.executeTool(ctx, req, tc)
|
||
toolExecutions = append(toolExecutions, execution)
|
||
|
||
if approval, ok := approvalNeededFromToolCall(req, tc, result); ok {
|
||
pendingApprovals = append(pendingApprovals, approval)
|
||
continue
|
||
}
|
||
|
||
// Add tool result to messages
|
||
projection := newLegacyServiceProviderToolResultContextProjection(tc.ID, result, !execution.Success)
|
||
messages = append(messages, providers.Message{
|
||
Role: "user",
|
||
ToolResult: &projection.Model,
|
||
})
|
||
|
||
}
|
||
|
||
// Stop the agentic loop when approvals are required.
|
||
// The caller can execute approvals via /api/ai/run-command and continue.
|
||
if len(pendingApprovals) > 0 {
|
||
return &ExecuteResponse{
|
||
Content: finalContent,
|
||
Model: model,
|
||
InputTokens: totalInputTokens,
|
||
OutputTokens: totalOutputTokens,
|
||
CacheCreationInputTokens: totalCacheCreationTokens,
|
||
CacheReadInputTokens: totalCacheReadTokens,
|
||
ToolCalls: toolExecutions,
|
||
PendingApprovals: pendingApprovals,
|
||
}, nil
|
||
}
|
||
}
|
||
|
||
return &ExecuteResponse{
|
||
Content: finalContent,
|
||
Model: model,
|
||
InputTokens: totalInputTokens,
|
||
OutputTokens: totalOutputTokens,
|
||
CacheCreationInputTokens: totalCacheCreationTokens,
|
||
CacheReadInputTokens: totalCacheReadTokens,
|
||
ToolCalls: toolExecutions,
|
||
}, nil
|
||
}
|
||
|
||
// ExecuteStream sends a prompt to the AI and streams events via callback
|
||
// This allows the UI to show real-time progress during tool execution
|
||
func (s *Service) ExecuteStream(ctx context.Context, req ExecuteRequest, callback StreamCallback) (*ExecuteResponse, error) {
|
||
release, err := s.acquireExecutionSlot(ctx, req.UseCase)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer release()
|
||
|
||
if err := s.enforceBudget(req.UseCase); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Enrich request with finding context if this is a "help me fix" request
|
||
s.enrichRequestFromFinding(&req)
|
||
|
||
s.mu.RLock()
|
||
defaultProvider := s.provider
|
||
agentServer := s.agentServer
|
||
cfg := s.cfg
|
||
costStore := s.costStore
|
||
s.mu.RUnlock()
|
||
|
||
// Determine the model to use for this request
|
||
modelString := s.getModelForRequest(req)
|
||
requestSanitizer := s.requestSanitizerForModel(modelString)
|
||
|
||
// Create a provider for this specific model (supports multi-provider switching)
|
||
provider, err := providers.NewForModel(cfg, modelString)
|
||
if err != nil {
|
||
// Fall back to default provider if model-specific provider can't be created
|
||
log.Debug().Err(err).Str("model", modelString).Msg("could not create provider for model, using default")
|
||
provider = defaultProvider
|
||
}
|
||
|
||
if provider == nil {
|
||
// In demo mode, simulate streaming response if no provider is configured
|
||
if IsDemoMode() {
|
||
return GenerateDemoAIStream(req.Prompt, callback)
|
||
}
|
||
return nil, fmt.Errorf("Pulse Assistant is not enabled or configured")
|
||
}
|
||
|
||
// Build the system prompt
|
||
systemPrompt := req.SystemPrompt
|
||
if systemPrompt == "" {
|
||
systemPrompt = s.buildSystemPrompt(req, modelString)
|
||
}
|
||
|
||
// Debug log the system prompt length and key sections
|
||
log.Debug().
|
||
Int("prompt_length", len(systemPrompt)).
|
||
Bool("has_infrastructure_map", strings.Contains(systemPrompt, "## Infrastructure Map")).
|
||
Bool("has_docker_hosts", strings.Contains(systemPrompt, "### Docker Hosts")).
|
||
Bool("has_standalone_hosts", strings.Contains(systemPrompt, "### Standalone Hosts")).
|
||
Bool("has_guests", strings.Contains(systemPrompt, "### All Guests")).
|
||
Msg("AI system prompt built")
|
||
|
||
// Check if agent is available for this target
|
||
hasAgent := s.hasAgentForTarget(req)
|
||
|
||
// Build tools list if agent is available
|
||
var tools []providers.Tool
|
||
if hasAgent && agentServer != nil {
|
||
tools = s.getTools()
|
||
systemPrompt += `
|
||
|
||
## Available Tools
|
||
You have access to tools to execute commands on the target system. You should:
|
||
1. Use run_command to investigate issues, gather information, and PERFORM actions
|
||
2. Actually execute the commands - don't just explain what commands to run
|
||
3. For Proxmox operations (resize disk, manage containers/VMs), run commands on the AGENT (target_type=agent)
|
||
4. For operations inside a system container, use target_type=system-container
|
||
|
||
Examples of actions you can perform:
|
||
- Resize LXC disk: pct resize <vmid> rootfs +10G (run on host)
|
||
- Check disk usage: df -h (run on container)
|
||
- View processes: ps aux --sort=-%mem | head -20
|
||
- Check logs: tail -100 /var/log/syslog
|
||
|
||
Always execute the commands rather than telling the user how to do it.`
|
||
}
|
||
|
||
// Inject previously learned knowledge about this guest
|
||
if s.knowledgeStore != nil {
|
||
guestID := s.getGuestID(req)
|
||
if guestID != "" {
|
||
if knowledgeContext := s.knowledgeStore.FormatForContext(guestID); knowledgeContext != "" {
|
||
log.Debug().
|
||
Str("guest_id", guestID).
|
||
Int("context_length", len(knowledgeContext)).
|
||
Msg("Injecting saved knowledge into AI context")
|
||
systemPrompt += knowledgeContext
|
||
} else {
|
||
log.Debug().Str("guest_id", guestID).Msg("no saved knowledge for guest")
|
||
}
|
||
}
|
||
}
|
||
|
||
// Build initial messages with conversation history
|
||
var messages []providers.Message
|
||
for _, histMsg := range req.History {
|
||
messages = append(messages, providers.Message{
|
||
Role: histMsg.Role,
|
||
Content: histMsg.Content,
|
||
})
|
||
}
|
||
messages = append(messages, providers.Message{Role: "user", Content: req.Prompt})
|
||
|
||
var toolExecutions []ToolExecution
|
||
totalInputTokens := 0
|
||
totalOutputTokens := 0
|
||
totalCacheCreationTokens := 0
|
||
totalCacheReadTokens := 0
|
||
var finalContent string
|
||
var model string
|
||
|
||
// Agentic loop - keep going while AI requests tools within the same
|
||
// provider-call budget as non-streaming execution.
|
||
loopGuard := newServiceToolLoopGuard()
|
||
for {
|
||
if err := s.enforceBudget(req.UseCase); err != nil {
|
||
callback(StreamEvent{Type: "error", Data: map[string]string{"message": err.Error()}})
|
||
return nil, err
|
||
}
|
||
if err := loopGuard.beforeProviderCall(); err != nil {
|
||
emitServiceToolLoopFailure(callback, err)
|
||
return nil, err
|
||
}
|
||
iteration := loopGuard.providerCalls
|
||
log.Debug().
|
||
Int("iteration", iteration).
|
||
Int("message_count", len(messages)).
|
||
Int("system_prompt_length", len(systemPrompt)).
|
||
Int("tools_count", len(tools)).
|
||
Msg("Calling AI provider...")
|
||
|
||
// Send a processing event so the frontend knows we're making an AI call
|
||
// This is especially important after tool execution when the next AI call can take a while
|
||
if iteration > 1 {
|
||
callback(StreamEvent{Type: "processing", Data: fmt.Sprintf("Analyzing results (iteration %d)...", iteration)})
|
||
}
|
||
|
||
chatReq := providers.ChatRequest{
|
||
Messages: messages,
|
||
Model: modelString,
|
||
System: systemPrompt,
|
||
MaxTokens: 4096,
|
||
Tools: tools,
|
||
}
|
||
if requestSanitizer != nil {
|
||
chatReq = requestSanitizer(chatReq)
|
||
}
|
||
|
||
resp, err := provider.Chat(ctx, chatReq)
|
||
if err != nil {
|
||
log.Error().Err(err).Int("iteration", iteration).Msg("AI provider call failed")
|
||
callback(StreamEvent{Type: "error", Data: map[string]string{"message": err.Error()}})
|
||
return nil, fmt.Errorf("Pulse Assistant request failed: %w", err)
|
||
}
|
||
|
||
if costStore != nil {
|
||
providerName, _ := config.ParseModelString(modelString)
|
||
if providerName == "" {
|
||
providerName = provider.Name()
|
||
}
|
||
costStore.Record(cost.UsageEvent{
|
||
Timestamp: time.Now(),
|
||
Provider: providerName,
|
||
RequestModel: modelString,
|
||
ResponseModel: resp.Model,
|
||
UseCase: req.UseCase,
|
||
ContextScope: assistantContextScopeForExecuteRequest(req),
|
||
InputTokens: resp.InputTokens,
|
||
OutputTokens: resp.OutputTokens,
|
||
CacheCreationInputTokens: resp.CacheCreationInputTokens,
|
||
CacheReadInputTokens: resp.CacheReadInputTokens,
|
||
TargetType: req.TargetType,
|
||
TargetID: req.TargetID,
|
||
FindingID: req.FindingID,
|
||
})
|
||
}
|
||
|
||
log.Debug().Int("iteration", iteration).Msg("AI provider returned successfully")
|
||
|
||
totalInputTokens += resp.InputTokens
|
||
totalOutputTokens += resp.OutputTokens
|
||
totalCacheCreationTokens += resp.CacheCreationInputTokens
|
||
totalCacheReadTokens += resp.CacheReadInputTokens
|
||
model = resp.Model
|
||
finalContent = resp.Content
|
||
|
||
// Stream thinking/reasoning content if present (DeepSeek reasoner)
|
||
if resp.ReasoningContent != "" {
|
||
callback(StreamEvent{Type: "thinking", Data: resp.ReasoningContent})
|
||
}
|
||
|
||
// Stream intermediate content so users see the AI's explanations between tool calls
|
||
// This gives users visibility into the AI's reasoning as it works, not just at the end
|
||
if resp.Content != "" {
|
||
callback(StreamEvent{Type: "content", Data: resp.Content})
|
||
}
|
||
|
||
log.Debug().
|
||
Int("tool_calls", len(resp.ToolCalls)).
|
||
Str("stop_reason", resp.StopReason).
|
||
Int("iteration", iteration).
|
||
Int("total_input_tokens", totalInputTokens).
|
||
Int("total_output_tokens", totalOutputTokens).
|
||
Int("content_length", len(resp.Content)).
|
||
Bool("has_content", resp.Content != "").
|
||
Msg("AI streaming iteration complete")
|
||
|
||
// If no tool calls, we're done
|
||
if len(resp.ToolCalls) == 0 || resp.StopReason != "tool_use" {
|
||
log.Info().
|
||
Int("tool_calls", len(resp.ToolCalls)).
|
||
Str("stop_reason", resp.StopReason).
|
||
Int("iteration", iteration).
|
||
Msg("AI streaming loop ending - no more tool calls or stop_reason != tool_use")
|
||
break
|
||
}
|
||
if err := s.observeDeniedServiceToolCalls(loopGuard, resp.ToolCalls); err != nil {
|
||
emitServiceToolLoopFailure(callback, err)
|
||
return nil, err
|
||
}
|
||
|
||
// Add assistant's response with tool calls to messages
|
||
messages = append(messages, providers.Message{
|
||
Role: "assistant",
|
||
Content: resp.Content,
|
||
ReasoningContent: resp.ReasoningContent, // DeepSeek thinking mode
|
||
ToolCalls: resp.ToolCalls,
|
||
})
|
||
|
||
// Execute each tool call and add results
|
||
// Track if any command needs approval - if so, we'll stop the loop after processing
|
||
anyNeedsApproval := false
|
||
for _, tc := range resp.ToolCalls {
|
||
toolInput := s.getToolInputDisplay(tc)
|
||
class := agentcapabilities.ClassifyLegacyAssistantInvocation(tc.Name)
|
||
if !s.legacyAssistantInvocationPolicy().Allows(tc.Name, class) {
|
||
result := agentcapabilities.ToolResultText(agentcapabilities.NewInvocationBlockedToolResult(tc.Name, class))
|
||
execution := ToolExecution{Name: tc.Name, Input: toolInput, Output: result, Success: false}
|
||
toolExecutions = append(toolExecutions, execution)
|
||
callback(StreamEvent{Type: "tool_start", Data: ToolStartData{Name: tc.Name, Input: toolInput}})
|
||
callback(StreamEvent{Type: "tool_end", Data: ToolEndData{Name: tc.Name, Input: toolInput, Output: result, Success: false}})
|
||
projection := newLegacyServiceProviderToolResultContextProjection(tc.ID, result, true)
|
||
messages = append(messages, providers.Message{Role: "user", ToolResult: &projection.Model})
|
||
continue
|
||
}
|
||
|
||
// Check if this command needs approval
|
||
needsApproval := false
|
||
approvalID := ""
|
||
approvalReason := "Command requires user approval"
|
||
if tc.Name == agentcapabilities.LegacyAssistantRunCommandToolName {
|
||
cmd, _ := tc.Input[agentcapabilities.LegacyAssistantCommandArgumentName].(string)
|
||
runOnHost, _ := tc.Input[agentcapabilities.LegacyAssistantRunOnHostArgumentName].(bool)
|
||
targetHost, _ := tc.Input[agentcapabilities.LegacyAssistantTargetHostArgumentName].(string)
|
||
s.mu.RLock()
|
||
approvalOrgID := s.orgID
|
||
s.mu.RUnlock()
|
||
|
||
// If AI didn't specify target_host, try to get it from request context
|
||
// This is crucial for proper routing when the command is approved
|
||
if targetHost == "" {
|
||
if node, ok := req.Context["node"].(string); ok && node != "" {
|
||
targetHost = node
|
||
} else if node, ok := req.Context["hostname"].(string); ok && node != "" {
|
||
targetHost = node
|
||
} else if node, ok := req.Context["host_name"].(string); ok && node != "" {
|
||
targetHost = node
|
||
}
|
||
}
|
||
|
||
isAuto := s.isAutonomousForRequest(req)
|
||
policyDecision := s.policy.Evaluate(cmd)
|
||
log.Debug().
|
||
Bool("autonomous", isAuto).
|
||
Str("policy_decision", string(policyDecision)).
|
||
Str("command", cmd).
|
||
Str("target_host", targetHost).
|
||
Msg("Checking command policy/approval")
|
||
|
||
// Always block commands blocked by policy (even in autonomous mode).
|
||
if policyDecision == agentexec.PolicyBlock {
|
||
result := formatPolicyBlockedToolResult(cmd, "This command is blocked by security policy")
|
||
execution := ToolExecution{
|
||
Name: tc.Name,
|
||
Input: toolInput,
|
||
Output: result,
|
||
Success: false,
|
||
}
|
||
toolExecutions = append(toolExecutions, execution)
|
||
|
||
callback(StreamEvent{
|
||
Type: "tool_start",
|
||
Data: ToolStartData{Name: tc.Name, Input: toolInput},
|
||
})
|
||
callback(StreamEvent{
|
||
Type: "tool_end",
|
||
Data: ToolEndData{Name: tc.Name, Input: toolInput, Output: result, Success: false},
|
||
})
|
||
|
||
projection := newLegacyServiceProviderToolResultContextProjection(tc.ID, result, true)
|
||
messages = append(messages, providers.Message{
|
||
Role: "user",
|
||
ToolResult: &projection.Model,
|
||
})
|
||
continue
|
||
}
|
||
|
||
if req.RequireCommandApproval || isAuto || policyDecision == agentexec.PolicyRequireApproval {
|
||
needsApproval = true
|
||
anyNeedsApproval = true
|
||
approvalReason = "Security policy requires approval"
|
||
if req.RequireCommandApproval {
|
||
approvalReason = "This handoff requires operator approval before command execution"
|
||
} else if isAuto {
|
||
approvalReason = "Autonomous model sessions cannot dispatch raw commands; operator approval is required"
|
||
}
|
||
approvalID = createRunCommandApprovalRecord(
|
||
approvalOrgID,
|
||
cmd,
|
||
tc.ID,
|
||
req,
|
||
runOnHost,
|
||
targetHost,
|
||
approvalReason,
|
||
)
|
||
callback(StreamEvent{
|
||
Type: "approval_needed",
|
||
Data: ApprovalNeededData{
|
||
Command: cmd,
|
||
ToolID: tc.ID,
|
||
ToolName: tc.Name,
|
||
RunOnHost: runOnHost,
|
||
TargetHost: targetHost,
|
||
ApprovalID: approvalID,
|
||
},
|
||
})
|
||
}
|
||
}
|
||
|
||
var result string
|
||
var execution ToolExecution
|
||
|
||
if needsApproval {
|
||
// Don't execute - command needs user approval
|
||
// We'll break out of the loop after processing all tool calls
|
||
// Note: We don't add to toolExecutions here because the approval_needed event
|
||
// already tells the frontend to show the approval UI
|
||
cmd, _ := tc.Input[agentcapabilities.LegacyAssistantCommandArgumentName].(string)
|
||
result = formatApprovalNeededToolResult(cmd, tc.ID, approvalReason, approvalID)
|
||
execution = ToolExecution{
|
||
Name: tc.Name,
|
||
Input: toolInput,
|
||
Output: result,
|
||
Success: true, // Not an error; awaiting approval
|
||
}
|
||
} else {
|
||
// Stream tool start event
|
||
callback(StreamEvent{
|
||
Type: "tool_start",
|
||
Data: ToolStartData{Name: tc.Name, Input: toolInput},
|
||
})
|
||
|
||
result, execution = s.executeTool(ctx, req, tc)
|
||
toolExecutions = append(toolExecutions, execution)
|
||
|
||
// Stream tool end event
|
||
callback(StreamEvent{
|
||
Type: "tool_end",
|
||
Data: ToolEndData{Name: tc.Name, Input: toolInput, Output: result, Success: execution.Success},
|
||
})
|
||
|
||
// Log to remediation log for operational memory
|
||
// Only log run_command since that's the main remediation action
|
||
if tc.Name == agentcapabilities.LegacyAssistantRunCommandToolName {
|
||
s.logRemediation(req, toolInput, result, execution.Success)
|
||
}
|
||
}
|
||
|
||
// Add tool result to messages
|
||
projection := newLegacyServiceProviderToolResultContextProjection(tc.ID, result, !execution.Success)
|
||
messages = append(messages, providers.Message{
|
||
Role: "user",
|
||
ToolResult: &projection.Model,
|
||
})
|
||
}
|
||
|
||
// If any command needed approval, stop the agentic loop here.
|
||
// Don't call the AI again with "COMMAND_BLOCKED" results - this causes duplicate
|
||
// approval requests and confusing "click the button" messages.
|
||
// The frontend will show approval buttons, and user action will continue the conversation.
|
||
if anyNeedsApproval {
|
||
log.Info().
|
||
Int("pending_approvals", len(resp.ToolCalls)).
|
||
Int("iteration", iteration).
|
||
Msg("Stopping AI loop - commands need user approval")
|
||
// Use the AI's current response as final content (if any)
|
||
// This preserves any explanation the AI provided before requesting the command
|
||
break
|
||
}
|
||
}
|
||
|
||
// Don't stream finalContent here - it was already streamed in the iteration above
|
||
// Sending it again causes duplicate responses (issue #947)
|
||
callback(StreamEvent{Type: "done"})
|
||
|
||
return &ExecuteResponse{
|
||
Content: finalContent,
|
||
Model: model,
|
||
InputTokens: totalInputTokens,
|
||
OutputTokens: totalOutputTokens,
|
||
CacheCreationInputTokens: totalCacheCreationTokens,
|
||
CacheReadInputTokens: totalCacheReadTokens,
|
||
ToolCalls: toolExecutions,
|
||
}, nil
|
||
}
|
||
|
||
// getToolInputDisplay returns a human-readable display of tool input
|
||
func (s *Service) getToolInputDisplay(tc providers.ToolCall) string {
|
||
switch tc.Name {
|
||
case agentcapabilities.LegacyAssistantRunCommandToolName:
|
||
cmd, _ := tc.Input[agentcapabilities.LegacyAssistantCommandArgumentName].(string)
|
||
if runOnHost, ok := tc.Input[agentcapabilities.LegacyAssistantRunOnHostArgumentName].(bool); ok && runOnHost {
|
||
return fmt.Sprintf("[host] %s", cmd)
|
||
}
|
||
return cmd
|
||
case agentcapabilities.LegacyAssistantFetchURLToolName:
|
||
fetchURL, _ := tc.Input[agentcapabilities.LegacyAssistantURLArgumentName].(string)
|
||
return fetchURL
|
||
case agentcapabilities.LegacyAssistantSetResourceURLToolName:
|
||
resourceType, _ := tc.Input[agentcapabilities.LegacyAssistantResourceTypeArgumentName].(string)
|
||
resourceURL, _ := tc.Input[agentcapabilities.LegacyAssistantURLArgumentName].(string)
|
||
return fmt.Sprintf("Set %s URL: %s", resourceType, resourceURL)
|
||
default:
|
||
return fmt.Sprintf("%v", tc.Input)
|
||
}
|
||
}
|
||
|
||
// logRemediation logs a tool execution to the remediation log for operational memory
|
||
// This enables learning from past fix attempts
|
||
// ONLY logs commands that perform actual ACTIONS (restarts, resizes, cleanups, fixes)
|
||
// Skips diagnostic commands (df, grep, cat, tail, ps) to avoid noise
|
||
func (s *Service) logRemediation(req ExecuteRequest, command, output string, success bool) {
|
||
// First, check if this is an actionable command worth logging
|
||
// Diagnostic/read-only commands don't provide value in the achievement log
|
||
if !isActionableCommand(command) {
|
||
log.Debug().
|
||
Str("command", command).
|
||
Msg("Skipping diagnostic command - not logging to remediation history")
|
||
return
|
||
}
|
||
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol == nil {
|
||
return
|
||
}
|
||
|
||
remLog := patrol.GetRemediationLog()
|
||
if remLog == nil {
|
||
return
|
||
}
|
||
|
||
// Determine outcome
|
||
outcome := OutcomeUnknown
|
||
if success {
|
||
outcome = OutcomeResolved
|
||
} else {
|
||
outcome = OutcomeFailed
|
||
}
|
||
|
||
// Extract problem from the original prompt (first 200 chars as summary)
|
||
problem := req.Prompt
|
||
if len(problem) > 200 {
|
||
problem = problem[:200] + "..."
|
||
}
|
||
|
||
// Generate a meaningful summary from the command that describes what was achieved
|
||
// This is what gets displayed in the Pulse Intelligence Impact section
|
||
summary := generateRemediationSummary(command, req.TargetType, req.Context)
|
||
|
||
// Get resource name from context if available
|
||
resourceName := ""
|
||
if req.Context != nil {
|
||
if name, ok := req.Context["name"].(string); ok {
|
||
resourceName = name
|
||
}
|
||
}
|
||
|
||
// Truncate output for storage
|
||
truncatedOutput := output
|
||
const maxOutputLen = 1000
|
||
if len(truncatedOutput) > maxOutputLen {
|
||
truncatedOutput = truncatedOutput[:maxOutputLen] + "..."
|
||
}
|
||
|
||
if err := remLog.Log(RemediationRecord{
|
||
ResourceID: req.TargetID,
|
||
ResourceType: req.TargetType,
|
||
ResourceName: resourceName,
|
||
FindingID: req.FindingID,
|
||
Problem: problem,
|
||
Summary: summary,
|
||
Action: command,
|
||
Output: truncatedOutput,
|
||
Outcome: outcome,
|
||
Automatic: req.UseCase == "patrol", // Patrol runs are automatic
|
||
}); err != nil {
|
||
log.Warn().Err(err).Str("resource_id", req.TargetID).Msg("Failed to log ACTION to Pulse Intelligence Impact")
|
||
}
|
||
|
||
log.Info().
|
||
Str("resource_id", req.TargetID).
|
||
Str("resource_name", resourceName).
|
||
Str("command", command).
|
||
Str("summary", summary).
|
||
Bool("success", success).
|
||
Msg("Logged ACTION to Pulse Intelligence Impact")
|
||
}
|
||
|
||
// isActionableCommand returns true if the command performs an actual action
|
||
// that's worth logging as an achievement, false for read-only diagnostics
|
||
func isActionableCommand(cmd string) bool {
|
||
cmd = strings.TrimSpace(cmd)
|
||
|
||
// Strip [host] or [hostname] prefix if present
|
||
if strings.HasPrefix(cmd, "[") {
|
||
if idx := strings.Index(cmd, "]"); idx != -1 {
|
||
cmd = strings.TrimSpace(cmd[idx+1:])
|
||
}
|
||
}
|
||
|
||
// Commands that PERFORM ACTIONS (should be logged)
|
||
actionPatterns := []string{
|
||
"docker restart", "docker start", "docker stop", "docker rm",
|
||
"docker compose up", "docker compose down", "docker compose restart",
|
||
"systemctl restart", "systemctl start", "systemctl stop", "systemctl enable", "systemctl disable",
|
||
"service restart", "service start", "service stop",
|
||
"pct resize", "pct start", "pct stop", "pct shutdown", "pct reboot",
|
||
"qm resize", "qm start", "qm stop", "qm shutdown", "qm reboot",
|
||
"rm -", "rm /", // File deletion/cleanup
|
||
"chmod", "chown", // Permission fixes
|
||
"mkdir", // Creating directories
|
||
"mv ", "cp ", // File operations
|
||
"echo >", "tee ", // Writing to files
|
||
"apt install", "apt upgrade", "apt remove",
|
||
"yum install", "dnf install",
|
||
"pip install", "npm install",
|
||
"kill ", "pkill ", "killall ",
|
||
"reboot", "shutdown",
|
||
}
|
||
|
||
for _, pattern := range actionPatterns {
|
||
if strings.Contains(cmd, pattern) {
|
||
return true
|
||
}
|
||
}
|
||
|
||
// Everything else is diagnostic (df, grep, cat, tail, ps, ls, etc.)
|
||
return false
|
||
}
|
||
|
||
// generateRemediationSummary creates a human-readable summary of what a command achieved
|
||
// This is used to display meaningful descriptions in the Pulse Intelligence Impact section
|
||
func generateRemediationSummary(command string, _ string, context map[string]interface{}) string {
|
||
cmd := strings.TrimSpace(command)
|
||
|
||
// Extract target name from context
|
||
targetName := ""
|
||
if context != nil {
|
||
if name, ok := context["name"].(string); ok && name != "" {
|
||
targetName = name
|
||
} else if name, ok := context["containerName"].(string); ok && name != "" {
|
||
targetName = name
|
||
} else if name, ok := context["guestName"].(string); ok && name != "" {
|
||
targetName = name
|
||
}
|
||
}
|
||
|
||
// Extract meaningful path or service from command
|
||
extractPath := func() string {
|
||
// Look for common path patterns
|
||
pathPatterns := []string{
|
||
`/[\w/._-]+`, // Unix paths
|
||
}
|
||
for _, pattern := range pathPatterns {
|
||
re := regexp.MustCompile(pattern)
|
||
if match := re.FindString(cmd); match != "" {
|
||
// Extract the meaningful part (last 2 segments)
|
||
parts := strings.Split(strings.Trim(match, "/"), "/")
|
||
if len(parts) > 2 {
|
||
parts = parts[len(parts)-2:]
|
||
}
|
||
return "/" + strings.Join(parts, "/")
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// Extract container/service name from docker commands
|
||
extractDockerTarget := func() string {
|
||
// docker ps --filter name=XXX
|
||
if match := regexp.MustCompile(`name[=:]\s*(\w+)`).FindStringSubmatch(cmd); len(match) > 1 {
|
||
return match[1]
|
||
}
|
||
// docker restart XXX, docker start XXX, etc.
|
||
if match := regexp.MustCompile(`docker\s+(?:restart|start|stop|logs)\s+([\w.-]+)`).FindStringSubmatch(cmd); len(match) > 1 {
|
||
return match[1]
|
||
}
|
||
return ""
|
||
}
|
||
|
||
path := extractPath()
|
||
dockerTarget := extractDockerTarget()
|
||
|
||
// Generate summary based on command type
|
||
switch {
|
||
case strings.Contains(cmd, "docker restart") || strings.Contains(cmd, "docker start"):
|
||
if dockerTarget != "" {
|
||
return fmt.Sprintf("Restarted %s container", dockerTarget)
|
||
}
|
||
return "Restarted container"
|
||
|
||
case strings.Contains(cmd, "docker stop"):
|
||
if dockerTarget != "" {
|
||
return fmt.Sprintf("Stopped %s container", dockerTarget)
|
||
}
|
||
return "Stopped container"
|
||
|
||
case strings.Contains(cmd, "docker ps"):
|
||
if dockerTarget != "" {
|
||
return fmt.Sprintf("Verified %s container is running", dockerTarget)
|
||
}
|
||
return "Checked container status"
|
||
|
||
case strings.Contains(cmd, "docker logs"):
|
||
if dockerTarget != "" {
|
||
return fmt.Sprintf("Retrieved %s logs", dockerTarget)
|
||
}
|
||
return "Retrieved container logs"
|
||
|
||
case strings.Contains(cmd, "systemctl restart"):
|
||
if match := regexp.MustCompile(`systemctl\s+restart\s+(\S+)`).FindStringSubmatch(cmd); len(match) > 1 {
|
||
return fmt.Sprintf("Restarted %s service", match[1])
|
||
}
|
||
return "Restarted system service"
|
||
|
||
case strings.Contains(cmd, "systemctl status"):
|
||
if match := regexp.MustCompile(`systemctl\s+status\s+(\S+)`).FindStringSubmatch(cmd); len(match) > 1 {
|
||
return fmt.Sprintf("Checked %s service status", match[1])
|
||
}
|
||
return "Checked service status"
|
||
|
||
case strings.Contains(cmd, "df ") || strings.Contains(cmd, "du "):
|
||
if path != "" {
|
||
// Check for known services in path
|
||
pathLower := strings.ToLower(path)
|
||
if strings.Contains(pathLower, "frigate") {
|
||
return "Analyzed Frigate storage usage"
|
||
}
|
||
if strings.Contains(pathLower, "plex") {
|
||
return "Analyzed Plex storage usage"
|
||
}
|
||
if strings.Contains(pathLower, "recordings") {
|
||
return "Analyzed recordings storage"
|
||
}
|
||
return fmt.Sprintf("Analyzed %s storage", path)
|
||
}
|
||
return "Analyzed disk usage"
|
||
|
||
case strings.Contains(cmd, "grep") && (strings.Contains(cmd, "config") || strings.Contains(cmd, "conf") || strings.Contains(cmd, ".yml") || strings.Contains(cmd, ".yaml")):
|
||
if path != "" && strings.Contains(strings.ToLower(path), "frigate") {
|
||
return "Inspected Frigate configuration"
|
||
}
|
||
if path != "" {
|
||
return fmt.Sprintf("Inspected %s configuration", path)
|
||
}
|
||
return "Inspected configuration"
|
||
|
||
case strings.Contains(cmd, "tail") || strings.Contains(cmd, "journalctl"):
|
||
if targetName != "" {
|
||
return fmt.Sprintf("Reviewed %s logs", targetName)
|
||
}
|
||
return "Reviewed system logs"
|
||
|
||
case strings.Contains(cmd, "pct resize"):
|
||
if match := regexp.MustCompile(`pct\s+resize\s+(\d+)`).FindStringSubmatch(cmd); len(match) > 1 {
|
||
return fmt.Sprintf("Resized container %s disk", match[1])
|
||
}
|
||
return "Resized container disk"
|
||
|
||
case strings.Contains(cmd, "qm resize"):
|
||
if match := regexp.MustCompile(`qm\s+resize\s+(\d+)`).FindStringSubmatch(cmd); len(match) > 1 {
|
||
return fmt.Sprintf("Resized VM %s disk", match[1])
|
||
}
|
||
return "Resized VM disk"
|
||
|
||
case strings.Contains(cmd, "ping") || strings.Contains(cmd, "curl"):
|
||
return "Tested network connectivity"
|
||
|
||
case strings.Contains(cmd, "free") || strings.Contains(cmd, "meminfo"):
|
||
return "Checked memory usage"
|
||
|
||
case strings.Contains(cmd, "ps aux") || strings.Contains(cmd, "top"):
|
||
return "Analyzed running processes"
|
||
|
||
case strings.Contains(cmd, "rm "):
|
||
return "Cleaned up files"
|
||
|
||
case strings.Contains(cmd, "chmod") || strings.Contains(cmd, "chown"):
|
||
return "Fixed file permissions"
|
||
|
||
default:
|
||
// Generic fallback - try to use target name if available
|
||
if targetName != "" {
|
||
return fmt.Sprintf("Ran diagnostics on %s", targetName)
|
||
}
|
||
return "Ran system diagnostics"
|
||
}
|
||
}
|
||
|
||
// hasAgentForTarget checks if we have an agent connection for the given target.
|
||
// This uses the same routing logic as command execution to determine if the target
|
||
// can be reached, including cluster peer routing for Proxmox clusters.
|
||
func (s *Service) hasAgentForTarget(req ExecuteRequest) bool {
|
||
// Check for nil interface or nil underlying value
|
||
// Note: A typed nil pointer assigned to an interface is NOT nil according to Go's == operator
|
||
// We need to use reflection to check if the underlying value is nil
|
||
if s.agentServer == nil {
|
||
return false
|
||
}
|
||
// Check if the interface contains a typed nil pointer
|
||
v := reflect.ValueOf(s.agentServer)
|
||
if v.Kind() == reflect.Ptr && v.IsNil() {
|
||
return false
|
||
}
|
||
|
||
agents := s.agentServer.GetConnectedAgents()
|
||
if len(agents) == 0 {
|
||
return false
|
||
}
|
||
|
||
// For agent targets with no specific context, any connected agent will do.
|
||
if req.TargetType == "agent" && len(req.Context) == 0 {
|
||
return true
|
||
}
|
||
|
||
// Try to determine the target node from the request context
|
||
// This mirrors the logic in routeToAgent
|
||
targetNode := ""
|
||
|
||
// Check context fields for the target node
|
||
hostFields := []string{"node", "host", "guest_node", "hostname", "host_name", "target_host"}
|
||
for _, field := range hostFields {
|
||
if value, ok := req.Context[field].(string); ok && value != "" {
|
||
targetNode = strings.ToLower(value)
|
||
break
|
||
}
|
||
}
|
||
|
||
// If no target node found in context, try the unified resource provider
|
||
if targetNode == "" {
|
||
s.mu.RLock()
|
||
urp := s.unifiedResourceProvider
|
||
s.mu.RUnlock()
|
||
|
||
if urp != nil {
|
||
resourceName := ""
|
||
if name, ok := req.Context["containerName"].(string); ok && name != "" {
|
||
resourceName = name
|
||
} else if name, ok := req.Context["name"].(string); ok && name != "" {
|
||
resourceName = name
|
||
} else if name, ok := req.Context["guestName"].(string); ok && name != "" {
|
||
resourceName = name
|
||
}
|
||
|
||
if resourceName != "" {
|
||
if host := urp.FindContainerHost(resourceName); host != "" {
|
||
targetNode = strings.ToLower(host)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// If we still don't have a target node, check for single agent scenario
|
||
if targetNode == "" {
|
||
// For unknown targets, we need at least one agent
|
||
// The actual routing will determine which one to use
|
||
return len(agents) >= 1
|
||
}
|
||
|
||
// Check if we have a direct agent match or a cluster peer
|
||
for _, agent := range agents {
|
||
if strings.EqualFold(agent.Hostname, targetNode) {
|
||
return true
|
||
}
|
||
}
|
||
|
||
// Try cluster peer routing (for Proxmox clusters)
|
||
if peerAgentID := s.findClusterPeerAgent(targetNode, agents); peerAgentID != "" {
|
||
return true
|
||
}
|
||
|
||
return false
|
||
}
|
||
|
||
// getTools returns the available tools for AI
|
||
func (s *Service) legacyAssistantInvocationPolicy() aitools.InvocationPolicy {
|
||
s.mu.RLock()
|
||
cfg := s.cfg
|
||
s.mu.RUnlock()
|
||
level := aitools.ControlLevelReadOnly
|
||
if cfg != nil {
|
||
level = aitools.ControlLevel(cfg.GetControlLevel())
|
||
}
|
||
return aitools.InvocationPolicy{
|
||
ControlLevel: level,
|
||
HasExecuteAuthority: true,
|
||
ExecuteAuthorityBound: true,
|
||
DenyInfrastructureMutations: true,
|
||
PulseStateAllowlist: map[string]bool{},
|
||
Profile: aitools.ProfileInteractiveAssistant,
|
||
}
|
||
}
|
||
|
||
func (s *Service) getTools() []providers.Tool {
|
||
candidates := append([]providers.Tool{}, agentcapabilities.LegacyAssistantUtilityProviderTools()...)
|
||
candidates = append(candidates,
|
||
providers.Tool{
|
||
Name: agentcapabilities.ResolveFindingCapabilityName,
|
||
Description: "Mark an AI patrol finding as resolved after successfully fixing the issue. Use the finding ID shown in your Patrol Finding Context section. Call this after verifying the fix worked - do NOT ask the user for the finding ID.",
|
||
InputSchema: legacyAssistantCapabilityInputSchema(agentcapabilities.ResolveFindingCapabilityName),
|
||
},
|
||
providers.Tool{
|
||
Name: agentcapabilities.DismissFindingCapabilityName,
|
||
Description: "Dismiss an AI patrol finding when it's not actually an issue or is expected behavior. Use this instead of " + agentcapabilities.ResolveFindingCapabilityName + " when the finding is a false positive or the configuration is intentional. This creates a suppression rule to prevent similar findings from being raised again.",
|
||
InputSchema: legacyAssistantCapabilityInputSchema(agentcapabilities.DismissFindingCapabilityName),
|
||
},
|
||
)
|
||
|
||
// Add web search tool for Anthropic provider
|
||
if s.provider != nil && s.provider.Name() == "anthropic" {
|
||
candidates = append(candidates, providers.Tool{
|
||
Type: "web_search_20250305",
|
||
Name: "web_search",
|
||
MaxUses: 3, // Limit searches per request to control costs
|
||
})
|
||
}
|
||
|
||
policy := s.legacyAssistantInvocationPolicy()
|
||
available := make([]providers.Tool, 0, len(candidates))
|
||
for _, candidate := range candidates {
|
||
if candidate.Name == "web_search" || policy.Allows(candidate.Name, agentcapabilities.ClassifyLegacyAssistantInvocation(candidate.Name)) {
|
||
available = append(available, candidate)
|
||
}
|
||
}
|
||
return available
|
||
}
|
||
|
||
// executeTool executes a tool call and returns the result
|
||
func (s *Service) executeTool(ctx context.Context, req ExecuteRequest, tc providers.ToolCall) (string, ToolExecution) {
|
||
execution := ToolExecution{
|
||
Name: tc.Name,
|
||
Success: false,
|
||
}
|
||
class := agentcapabilities.ClassifyLegacyAssistantInvocation(tc.Name)
|
||
if !s.legacyAssistantInvocationPolicy().Allows(tc.Name, class) {
|
||
execution.Output = agentcapabilities.ToolResultText(agentcapabilities.NewInvocationBlockedToolResult(tc.Name, class))
|
||
return execution.Output, execution
|
||
}
|
||
|
||
switch tc.Name {
|
||
case agentcapabilities.LegacyAssistantRunCommandToolName:
|
||
command, _ := tc.Input[agentcapabilities.LegacyAssistantCommandArgumentName].(string)
|
||
runOnHost, _ := tc.Input[agentcapabilities.LegacyAssistantRunOnHostArgumentName].(bool)
|
||
targetHost, _ := tc.Input[agentcapabilities.LegacyAssistantTargetHostArgumentName].(string)
|
||
execution.Input = command
|
||
if runOnHost && targetHost != "" {
|
||
execution.Input = fmt.Sprintf("[%s] %s", targetHost, command)
|
||
} else if runOnHost {
|
||
execution.Input = fmt.Sprintf("[host] %s", command)
|
||
}
|
||
|
||
if command == "" {
|
||
execution.Output = "Error: command is required"
|
||
return execution.Output, execution
|
||
}
|
||
|
||
// Enforce security policy.
|
||
// - Blocked commands are ALWAYS blocked (even in autonomous mode).
|
||
// - Approval-required commands only require approval when not in autonomous mode.
|
||
decision := s.policy.Evaluate(command)
|
||
if decision == agentexec.PolicyBlock {
|
||
execution.Output = formatPolicyBlockedToolResult(command, "This command is blocked by security policy")
|
||
return execution.Output, execution
|
||
}
|
||
isAuto := s.isAutonomousForRequest(req)
|
||
if req.RequireCommandApproval || isAuto || decision == agentexec.PolicyRequireApproval {
|
||
s.mu.RLock()
|
||
approvalOrgID := s.orgID
|
||
s.mu.RUnlock()
|
||
approvalReason := "Security policy requires approval"
|
||
if req.RequireCommandApproval {
|
||
approvalReason = "This handoff requires operator approval before command execution"
|
||
} else if isAuto {
|
||
approvalReason = "Autonomous model sessions cannot dispatch raw commands; operator approval is required"
|
||
}
|
||
approvalID := createRunCommandApprovalRecord(
|
||
approvalOrgID,
|
||
command,
|
||
tc.ID,
|
||
req,
|
||
runOnHost,
|
||
targetHost,
|
||
approvalReason,
|
||
)
|
||
execution.Output = formatApprovalNeededToolResult(command, tc.ID, approvalReason, approvalID)
|
||
execution.Success = true // Not an error, just needs approval
|
||
return execution.Output, execution
|
||
}
|
||
|
||
// Build execution request with proper targeting
|
||
execReq := req
|
||
|
||
// If target_host is explicitly specified by AI, use it for routing
|
||
if targetHost != "" {
|
||
// Ensure Context map exists
|
||
if execReq.Context == nil {
|
||
execReq.Context = make(map[string]interface{})
|
||
} else {
|
||
// Make a copy to avoid modifying the original
|
||
newContext := make(map[string]interface{})
|
||
for k, v := range req.Context {
|
||
newContext[k] = v
|
||
}
|
||
execReq.Context = newContext
|
||
}
|
||
// Set the node explicitly - this takes priority in routing
|
||
execReq.Context["node"] = targetHost
|
||
log.Debug().
|
||
Str("target_host", targetHost).
|
||
Str("command", command).
|
||
Msg("AI explicitly specified target_host for command routing")
|
||
}
|
||
|
||
// If run_on_host is true, override the target type to run on host
|
||
if runOnHost {
|
||
log.Debug().
|
||
Str("command", command).
|
||
Str("target_host", targetHost).
|
||
Str("original_target_type", req.TargetType).
|
||
Str("original_target_id", req.TargetID).
|
||
Msg("run_on_host=true - overriding target type to 'agent'")
|
||
execReq.TargetType = "agent"
|
||
execReq.TargetID = ""
|
||
} else {
|
||
log.Debug().
|
||
Str("command", command).
|
||
Str("target_type", req.TargetType).
|
||
Str("target_id", req.TargetID).
|
||
Bool("run_on_host", runOnHost).
|
||
Msg("Executing command with current target type")
|
||
}
|
||
|
||
// Execute via agent
|
||
result, err := s.executeOnAgent(ctx, execReq, command)
|
||
recordIncident := func(success bool, output string) {
|
||
alertIdentifier := extractAlertIdentifier(req.Context)
|
||
resourceID := extractIncidentResourceIdentifier(req.TargetID, req.Context)
|
||
details := map[string]any{
|
||
"resource_id": resourceID,
|
||
"resource_type": req.TargetType,
|
||
"run_on_host": runOnHost,
|
||
}
|
||
if targetHost != "" {
|
||
details["target_host"] = targetHost
|
||
}
|
||
if alertIdentifier != "" {
|
||
s.mu.RLock()
|
||
incidentStore := s.incidentStore
|
||
s.mu.RUnlock()
|
||
if incidentStore != nil {
|
||
incidentStore.RecordCommand(alertIdentifier, command, success, output, details)
|
||
}
|
||
}
|
||
s.recordCanonicalResourceChange(unifiedresources.BuildCommandExecutionChange(
|
||
resourceID,
|
||
alertIdentifier,
|
||
"agent:pulse-assistant",
|
||
command,
|
||
success,
|
||
output,
|
||
details,
|
||
))
|
||
}
|
||
if err != nil {
|
||
recordIncident(false, result)
|
||
execution.Output = fmt.Sprintf("Error executing command: %s", err)
|
||
return execution.Output, execution
|
||
}
|
||
|
||
recordIncident(true, result)
|
||
execution.Output = result
|
||
execution.Success = true
|
||
return result, execution
|
||
|
||
case agentcapabilities.LegacyAssistantFetchURLToolName:
|
||
urlStr, _ := tc.Input[agentcapabilities.LegacyAssistantURLArgumentName].(string)
|
||
execution.Input = urlStr
|
||
|
||
if urlStr == "" {
|
||
execution.Output = "Error: url is required"
|
||
return execution.Output, execution
|
||
}
|
||
|
||
// Fetch the URL
|
||
result, err := s.fetchURL(ctx, urlStr)
|
||
if err != nil {
|
||
execution.Output = fmt.Sprintf("Error fetching URL: %s", err)
|
||
return execution.Output, execution
|
||
}
|
||
|
||
execution.Output = result
|
||
execution.Success = true
|
||
return result, execution
|
||
|
||
case agentcapabilities.LegacyAssistantSetResourceURLToolName:
|
||
resourceType, _ := tc.Input[agentcapabilities.LegacyAssistantResourceTypeArgumentName].(string)
|
||
resourceID, _ := tc.Input[agentcapabilities.LegacyAssistantResourceIDArgumentName].(string)
|
||
resourceURL, _ := tc.Input[agentcapabilities.LegacyAssistantURLArgumentName].(string)
|
||
execution.Input = fmt.Sprintf("%s %s -> %s", resourceType, resourceID, resourceURL)
|
||
|
||
if resourceType == "" {
|
||
execution.Output = "Error: resource_type is required (use canonical v6 types: vm, system-container, oci-container, app-container, agent, node, docker-host)"
|
||
return execution.Output, execution
|
||
}
|
||
if resourceID == "" {
|
||
// Try to get the resource ID from the request context
|
||
if req.TargetID != "" {
|
||
resourceID = req.TargetID
|
||
} else {
|
||
execution.Output = "Error: resource_id is required"
|
||
return execution.Output, execution
|
||
}
|
||
}
|
||
|
||
// Allow empty URL to clear the setting
|
||
// if resourceURL == "" {
|
||
// execution.Output = "Error: url is required"
|
||
// return execution.Output, execution
|
||
// }
|
||
|
||
// Update the metadata
|
||
if err := s.SetResourceURL(resourceType, resourceID, resourceURL); err != nil {
|
||
execution.Output = fmt.Sprintf("Error setting URL: %s", err)
|
||
return execution.Output, execution
|
||
}
|
||
|
||
execution.Output = fmt.Sprintf("Successfully set URL for %s '%s' to: %s\nThe URL is now visible in the Pulse dashboard as a clickable link.", resourceType, resourceID, resourceURL)
|
||
execution.Success = true
|
||
return execution.Output, execution
|
||
|
||
case agentcapabilities.ResolveFindingCapabilityName:
|
||
findingID, _ := tc.Input[agentcapabilities.FindingIDArgumentName].(string)
|
||
resolutionNote, _ := tc.Input[agentcapabilities.ResolutionNoteArgumentName].(string)
|
||
execution.Input = fmt.Sprintf("finding: %s, note: %s", findingID, resolutionNote)
|
||
|
||
// If no finding ID provided by AI, check the request context
|
||
if findingID == "" {
|
||
findingID = req.FindingID
|
||
}
|
||
|
||
if findingID == "" {
|
||
execution.Output = "Error: finding_id is required. The finding ID should be provided in the request context when helping fix a patrol finding."
|
||
return execution.Output, execution
|
||
}
|
||
|
||
// Get the patrol service to resolve the finding
|
||
s.mu.RLock()
|
||
patrolService := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrolService == nil {
|
||
execution.Output = "Error: Patrol service not available"
|
||
return execution.Output, execution
|
||
}
|
||
|
||
// Resolve the finding
|
||
err := patrolService.ResolveFinding(findingID, resolutionNote)
|
||
if err != nil {
|
||
execution.Output = fmt.Sprintf("Error resolving finding: %s", err)
|
||
return execution.Output, execution
|
||
}
|
||
|
||
execution.Output = fmt.Sprintf("Finding resolved! The Patrol finding has been marked as fixed.\nID: %s", findingID)
|
||
if resolutionNote != "" {
|
||
execution.Output += fmt.Sprintf("\nResolution: %s", resolutionNote)
|
||
}
|
||
execution.Success = true
|
||
return execution.Output, execution
|
||
|
||
case agentcapabilities.DismissFindingCapabilityName:
|
||
findingID, _ := tc.Input[agentcapabilities.FindingIDArgumentName].(string)
|
||
reason, _ := tc.Input[agentcapabilities.ReasonArgumentName].(string)
|
||
note, _ := tc.Input[agentcapabilities.NoteArgumentName].(string)
|
||
execution.Input = fmt.Sprintf("finding: %s, reason: %s, note: %s", findingID, reason, note)
|
||
|
||
// If no finding ID provided by AI, check the request context
|
||
if findingID == "" {
|
||
findingID = req.FindingID
|
||
}
|
||
|
||
if findingID == "" {
|
||
execution.Output = "Error: finding_id is required. The finding ID should be provided in the request context when helping fix a patrol finding."
|
||
return execution.Output, execution
|
||
}
|
||
|
||
// Validate reason
|
||
validReasons := map[string]bool{"not_an_issue": true, "expected_behavior": true, "will_fix_later": true}
|
||
if !validReasons[reason] {
|
||
execution.Output = "Error: reason must be one of: not_an_issue, expected_behavior, will_fix_later"
|
||
return execution.Output, execution
|
||
}
|
||
|
||
// Get the patrol service to dismiss the finding
|
||
s.mu.RLock()
|
||
patrolService := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrolService == nil {
|
||
execution.Output = "Error: Patrol service not available"
|
||
return execution.Output, execution
|
||
}
|
||
|
||
// Dismiss the finding (this will also create a suppression rule for expected_behavior/not_an_issue)
|
||
err := patrolService.DismissFinding(findingID, reason, note)
|
||
if err != nil {
|
||
execution.Output = fmt.Sprintf("Error dismissing finding: %s", err)
|
||
return execution.Output, execution
|
||
}
|
||
|
||
// Format a helpful response based on reason
|
||
var resultMsg string
|
||
if reason == "will_fix_later" {
|
||
remindAt := time.Now().Add(DefaultWillFixLaterRemindAfter).Format("2006-01-02")
|
||
resultMsg = fmt.Sprintf("Finding dismissed as '%s'. Pulse Patrol will stay quiet on this until %s; if the finding is still tripping after that date, it will surface again with a 'reminded' lifecycle event.\nID: %s\nNote: %s", reason, remindAt, findingID, note)
|
||
} else {
|
||
resultMsg = fmt.Sprintf("Finding dismissed as '%s' and suppression rule created. Similar findings for this resource will not be raised again.\nID: %s\nNote: %s", reason, findingID, note)
|
||
}
|
||
execution.Output = resultMsg
|
||
execution.Success = true
|
||
return execution.Output, execution
|
||
|
||
default:
|
||
execution.Output = fmt.Sprintf("Unknown tool: %s", tc.Name)
|
||
return execution.Output, execution
|
||
}
|
||
}
|
||
|
||
// getGuestID returns a unique identifier for the guest based on the request
|
||
func (s *Service) getGuestID(req ExecuteRequest) string {
|
||
// Build a consistent guest ID from the target information
|
||
if req.TargetType == "" || req.TargetID == "" {
|
||
return ""
|
||
}
|
||
|
||
// For Proxmox targets, include the node info
|
||
// Format: instance-node-type-vmid or instance-targetid
|
||
return fmt.Sprintf("%s-%s", req.TargetType, req.TargetID)
|
||
}
|
||
|
||
// GetGuestKnowledge returns all knowledge for a guest
|
||
func (s *Service) GetGuestKnowledge(guestID string) (*knowledge.GuestKnowledge, error) {
|
||
if s.knowledgeStore == nil {
|
||
return nil, fmt.Errorf("knowledge store not available")
|
||
}
|
||
return s.knowledgeStore.GetKnowledge(guestID)
|
||
}
|
||
|
||
// SaveGuestNote saves a note for a guest
|
||
func (s *Service) SaveGuestNote(guestID, guestName, guestType, category, title, content string) error {
|
||
if s.knowledgeStore == nil {
|
||
return fmt.Errorf("knowledge store not available")
|
||
}
|
||
return s.knowledgeStore.SaveNote(guestID, guestName, guestType, category, title, content)
|
||
}
|
||
|
||
// DeleteGuestNote deletes a note from a guest
|
||
func (s *Service) DeleteGuestNote(guestID, noteID string) error {
|
||
if s.knowledgeStore == nil {
|
||
return fmt.Errorf("knowledge store not available")
|
||
}
|
||
return s.knowledgeStore.DeleteNote(guestID, noteID)
|
||
}
|
||
|
||
// GetKnowledgeStore returns the knowledge store for external use
|
||
// This is used by components like the investigation orchestrator to get
|
||
// infrastructure context for proposing correct CLI commands
|
||
func (s *Service) GetKnowledgeStore() *knowledge.Store {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
return s.knowledgeStore
|
||
}
|
||
|
||
// fetchURL fetches content from a URL with size limits and timeout
|
||
func (s *Service) fetchURL(ctx context.Context, urlStr string) (string, error) {
|
||
parsedURL, err := parseAndValidateFetchURL(ctx, urlStr)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
// Create HTTP client with timeout and safe transport to prevent SSRF/DNS rebinding
|
||
client := &http.Client{
|
||
Timeout: 30 * time.Second,
|
||
Transport: &http.Transport{
|
||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||
// Use a dialer with reasonable timeout
|
||
d := net.Dialer{
|
||
Timeout: 10 * time.Second,
|
||
}
|
||
conn, err := d.DialContext(ctx, network, addr)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Validate the actual connected IP address
|
||
// This prevents DNS rebinding attacks
|
||
if tcpConn, ok := conn.(*net.TCPConn); ok {
|
||
remoteAddr := tcpConn.RemoteAddr().(*net.TCPAddr)
|
||
if isBlockedFetchIP(remoteAddr.IP) {
|
||
conn.Close()
|
||
return nil, fmt.Errorf("URL resolves to blocked IP address: %s", remoteAddr.IP)
|
||
}
|
||
}
|
||
|
||
return conn, nil
|
||
},
|
||
DisableKeepAlives: true, // One-off requests
|
||
},
|
||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||
if len(via) >= 3 {
|
||
return fmt.Errorf("too many redirects")
|
||
}
|
||
// Still validate the URL structure and initial resolution for failsafe
|
||
if _, err := parseAndValidateFetchURL(ctx, req.URL.String()); err != nil {
|
||
return err
|
||
}
|
||
return nil
|
||
},
|
||
}
|
||
|
||
// Create request with context
|
||
req, err := http.NewRequestWithContext(ctx, "GET", parsedURL.String(), nil)
|
||
if err != nil {
|
||
return "", fmt.Errorf("invalid URL: %w", err)
|
||
}
|
||
|
||
// Set a reasonable user agent
|
||
req.Header.Set("User-Agent", "Pulse-AI/1.0")
|
||
|
||
// Make the request
|
||
resp, err := client.Do(req)
|
||
if err != nil {
|
||
return "", fmt.Errorf("request failed: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
// Read response with size limit (64KB)
|
||
const maxSize = 64 * 1024
|
||
limitedReader := io.LimitReader(resp.Body, maxSize)
|
||
body, err := io.ReadAll(limitedReader)
|
||
if err != nil {
|
||
return "", fmt.Errorf("failed to read response: %w", err)
|
||
}
|
||
|
||
// Build result with status info
|
||
result := fmt.Sprintf("HTTP %d %s\n", resp.StatusCode, resp.Status)
|
||
result += fmt.Sprintf("Content-Type: %s\n", resp.Header.Get("Content-Type"))
|
||
result += fmt.Sprintf("Content-Length: %d bytes\n\n", len(body))
|
||
result += string(body)
|
||
|
||
if len(body) == maxSize {
|
||
result += "\n\n[Response truncated at 64KB]"
|
||
}
|
||
|
||
return result, nil
|
||
}
|
||
|
||
func parseAndValidateFetchURL(ctx context.Context, urlStr string) (*url.URL, error) {
|
||
clean := strings.TrimSpace(urlStr)
|
||
if clean == "" {
|
||
return nil, fmt.Errorf("url is required")
|
||
}
|
||
|
||
parsed, err := url.Parse(clean)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("invalid URL: %w", err)
|
||
}
|
||
if !parsed.IsAbs() {
|
||
return nil, fmt.Errorf("URL must be absolute")
|
||
}
|
||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||
return nil, fmt.Errorf("only http/https URLs are allowed")
|
||
}
|
||
if parsed.User != nil {
|
||
return nil, fmt.Errorf("URLs with embedded credentials are not allowed")
|
||
}
|
||
if parsed.Fragment != "" {
|
||
return nil, fmt.Errorf("URL fragments are not allowed")
|
||
}
|
||
|
||
host := parsed.Hostname()
|
||
if host == "" {
|
||
return nil, fmt.Errorf("URL must include a host")
|
||
}
|
||
|
||
if isBlockedFetchHost(host) {
|
||
return nil, fmt.Errorf("URL host is blocked")
|
||
}
|
||
|
||
if ip := net.ParseIP(host); ip != nil {
|
||
if isBlockedFetchIP(ip) {
|
||
return nil, fmt.Errorf("URL IP is blocked")
|
||
}
|
||
return parsed, nil
|
||
}
|
||
|
||
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||
if err != nil {
|
||
// DNS failures are surfaced directly to the caller.
|
||
return nil, fmt.Errorf("failed to resolve host: %w", err)
|
||
}
|
||
for _, addr := range addrs {
|
||
if isBlockedFetchIP(addr.IP) {
|
||
return nil, fmt.Errorf("URL host resolves to a blocked address")
|
||
}
|
||
}
|
||
|
||
return parsed, nil
|
||
}
|
||
|
||
func isBlockedFetchHost(host string) bool {
|
||
h := strings.TrimSpace(strings.ToLower(host))
|
||
if h == "localhost" || h == "localhost." {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func isBlockedFetchIP(ip net.IP) bool {
|
||
if ip == nil {
|
||
return true
|
||
}
|
||
if ip.IsLoopback() || ip.IsUnspecified() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
|
||
// Allow loopback only if explicitly permitted (for local development)
|
||
if ip.IsLoopback() && os.Getenv("PULSE_AI_ALLOW_LOOPBACK") == "true" {
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
// SECURITY: Block private IP ranges (RFC1918) to prevent SSRF attacks
|
||
// Private ranges: 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16
|
||
if ip.IsPrivate() {
|
||
// Allow private IPs only if explicitly permitted
|
||
if os.Getenv("PULSE_AI_ALLOW_PRIVATE_IPS") == "true" {
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
// Block multicast and other non-unicast targets.
|
||
if !ip.IsGlobalUnicast() {
|
||
return true
|
||
}
|
||
// SECURITY: every check above reads the literal address bytes, so an IPv6
|
||
// transition address (NAT64, 6to4, Teredo, ISATAP) tunnels to an internal
|
||
// IPv4 destination none of them can see. Hold the embedded destination to
|
||
// the same policy, including the loopback and private-IP escape hatches.
|
||
for _, embedded := range securityutil.EmbeddedIPv4Candidates(ip) {
|
||
if isBlockedFetchIP(embedded) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// sanitizeError cleans up error messages to remove internal networking details
|
||
// that are not helpful to users or AI models (IP addresses, port numbers, etc.)
|
||
func sanitizeError(err error) error {
|
||
if err == nil {
|
||
return nil
|
||
}
|
||
|
||
errMsg := err.Error()
|
||
|
||
// Replace raw TCP connection details with generic message
|
||
// e.g., "write tcp 192.0.2.10:7655->198.51.100.20:58004: i/o timeout"
|
||
// becomes "connection to agent timed out"
|
||
if strings.Contains(errMsg, "i/o timeout") {
|
||
if strings.Contains(errMsg, "failed to send command") {
|
||
return fmt.Errorf("connection to agent timed out - the agent may be disconnected or unreachable")
|
||
}
|
||
return fmt.Errorf("network timeout - the target may be unreachable")
|
||
}
|
||
|
||
// Replace "write tcp ... connection refused" style errors
|
||
if strings.Contains(errMsg, "connection refused") {
|
||
return fmt.Errorf("connection refused - the agent may not be running on the target host")
|
||
}
|
||
|
||
// Replace "no such host" errors
|
||
if strings.Contains(errMsg, "no such host") {
|
||
return fmt.Errorf("host not found - verify the hostname is correct and DNS is working")
|
||
}
|
||
|
||
// Replace "context deadline exceeded" with friendlier message
|
||
if strings.Contains(errMsg, "context deadline exceeded") {
|
||
return fmt.Errorf("operation timed out - the command may have taken too long")
|
||
}
|
||
|
||
return err
|
||
}
|
||
|
||
func hasExplicitHostRoutingContext(ctx map[string]interface{}) bool {
|
||
if len(ctx) == 0 {
|
||
return false
|
||
}
|
||
for _, key := range []string{"node", "host", "guest_node", "hostname", "host_name", "target_host"} {
|
||
if value, ok := ctx[key].(string); ok && strings.TrimSpace(value) != "" {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func normalizeExecuteTargetType(raw, targetID string, ctx map[string]interface{}) (string, error) {
|
||
targetType := agentcapabilities.NormalizeActionTargetType(raw)
|
||
switch targetType {
|
||
case agentcapabilities.ActionTargetTypeAgent, agentcapabilities.ActionTargetTypeSystemContainer, agentcapabilities.ActionTargetTypeVM:
|
||
return targetType, nil
|
||
case "":
|
||
if strings.TrimSpace(targetID) == "" && hasExplicitHostRoutingContext(ctx) {
|
||
return agentcapabilities.ActionTargetTypeAgent, nil
|
||
}
|
||
return "", fmt.Errorf("target_type is required (agent, system-container, or vm)")
|
||
default:
|
||
return "", fmt.Errorf("unsupported target_type %q (allowed: %s)", raw, agentcapabilities.ActionTargetTypeAllowedDescription())
|
||
}
|
||
}
|
||
|
||
func extractVMIDFromContext(ctx map[string]interface{}) (string, bool) {
|
||
if ctx == nil {
|
||
return "", false
|
||
}
|
||
vmID, ok := ctx["vmid"]
|
||
if !ok {
|
||
return "", false
|
||
}
|
||
|
||
switch v := vmID.(type) {
|
||
case float64:
|
||
if v > 0 {
|
||
return fmt.Sprintf("%.0f", v), true
|
||
}
|
||
case int:
|
||
if v > 0 {
|
||
return fmt.Sprintf("%d", v), true
|
||
}
|
||
case int64:
|
||
if v > 0 {
|
||
return fmt.Sprintf("%d", v), true
|
||
}
|
||
case string:
|
||
if parsed := extractVMIDFromTargetID(v); parsed > 0 {
|
||
return strconv.Itoa(parsed), true
|
||
}
|
||
}
|
||
|
||
return "", false
|
||
}
|
||
|
||
// executeOnAgent executes a command via the agent WebSocket
|
||
func (s *Service) executeOnAgent(ctx context.Context, req ExecuteRequest, command string) (string, error) {
|
||
return s.executeOnAgentWithApproval(ctx, req, command, "")
|
||
}
|
||
|
||
func (s *Service) executeOnAgentWithApproval(ctx context.Context, req ExecuteRequest, command, approvalID string) (string, error) {
|
||
return s.executeOnAgentWithAuthorization(ctx, req, command, approvalID, "", "")
|
||
}
|
||
|
||
func (s *Service) executeOnAgentWithAuthorization(ctx context.Context, req ExecuteRequest, command, approvalID, orgID, actionID string) (string, error) {
|
||
if s.agentServer == nil {
|
||
return "", fmt.Errorf("agent server not available")
|
||
}
|
||
|
||
normalizedTargetType, err := normalizeExecuteTargetType(req.TargetType, req.TargetID, req.Context)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
normalizedReq := req
|
||
normalizedReq.TargetType = normalizedTargetType
|
||
|
||
// Find the appropriate agent using robust routing
|
||
agents := s.agentServer.GetConnectedAgents()
|
||
|
||
// Use the new robust routing logic
|
||
routeResult, err := s.routeToAgent(normalizedReq, command, agents)
|
||
if err != nil {
|
||
// Check if this is a routing error that should ask for clarification
|
||
if routingErr, ok := err.(*RoutingError); ok && routingErr.AskForClarification {
|
||
// Return a message that encourages the AI to ask the user for clarification
|
||
// instead of just failing with an error
|
||
return routingErr.ForAI(), nil
|
||
}
|
||
// Return actionable error message for other errors
|
||
return "", err
|
||
}
|
||
|
||
// Log any warnings from routing
|
||
for _, warning := range routeResult.Warnings {
|
||
log.Warn().Str("warning", warning).Msg("routing warning")
|
||
}
|
||
|
||
agentID := routeResult.AgentID
|
||
|
||
log.Debug().
|
||
Str("agent_id", agentID).
|
||
Str("agent_hostname", routeResult.AgentHostname).
|
||
Str("target_node", routeResult.TargetNode).
|
||
Str("routing_method", routeResult.RoutingMethod).
|
||
Bool("cluster_peer", routeResult.ClusterPeer).
|
||
Msg("Command routed to agent")
|
||
|
||
targetID := strings.TrimSpace(req.TargetID)
|
||
if normalizedTargetType == "system-container" || normalizedTargetType == "vm" {
|
||
if vmID, ok := extractVMIDFromContext(req.Context); ok {
|
||
targetID = vmID
|
||
} else if extracted := extractVMIDFromTargetID(req.TargetID); extracted > 0 {
|
||
targetID = strconv.Itoa(extracted)
|
||
} else {
|
||
return "", fmt.Errorf("%s target requires numeric VMID in context.vmid or target_id", normalizedTargetType)
|
||
}
|
||
}
|
||
|
||
dispatchTargetType := normalizedTargetType
|
||
if normalizedTargetType == "system-container" {
|
||
// Agent transport uses "container" for LXC command routing.
|
||
dispatchTargetType = "container"
|
||
}
|
||
|
||
requestID := uuid.New().String()
|
||
|
||
// Automatically force non-interactive mode for package managers
|
||
// This prevents hanging when apt/dpkg asks for confirmation or configuration
|
||
if strings.Contains(command, "apt") || strings.Contains(command, "dpkg") {
|
||
if !strings.Contains(command, "DEBIAN_FRONTEND=") {
|
||
command = "export DEBIAN_FRONTEND=noninteractive; " + command
|
||
}
|
||
}
|
||
|
||
cmd := agentexec.ExecuteCommandPayload{
|
||
RequestID: requestID,
|
||
Command: command,
|
||
ApprovalID: strings.TrimSpace(approvalID),
|
||
TargetType: dispatchTargetType,
|
||
TargetID: targetID,
|
||
Timeout: 300, // 5 minutes - commands like du, backups, etc. can take a while
|
||
}
|
||
if strings.TrimSpace(approvalID) != "" {
|
||
cmd.BindCommandAuthorization(orgID, actionID)
|
||
}
|
||
|
||
result, err := s.agentServer.ExecuteCommand(ctx, agentID, cmd)
|
||
if err != nil {
|
||
return "", sanitizeError(err)
|
||
}
|
||
|
||
if !result.Success {
|
||
if result.Error != "" {
|
||
return "", fmt.Errorf("%s", result.Error)
|
||
}
|
||
if result.Stderr != "" {
|
||
return result.Stderr, nil // Return stderr as output, not error
|
||
}
|
||
}
|
||
|
||
output := result.Stdout
|
||
if result.Stderr != "" && result.Stdout != "" {
|
||
output = fmt.Sprintf("%s\n\nSTDERR:\n%s", result.Stdout, result.Stderr)
|
||
} else if result.Stderr != "" {
|
||
output = result.Stderr
|
||
}
|
||
|
||
return output, nil
|
||
}
|
||
|
||
// RunCommandRequest represents a request to run a single command
|
||
type RunCommandRequest struct {
|
||
Command string `json:"command"`
|
||
ApprovalID string `json:"approval_id,omitempty"` // Consumed by API handler before execution
|
||
TargetType string `json:"target_type"` // "agent", "system-container", "vm"
|
||
TargetID string `json:"target_id"`
|
||
RunOnHost bool `json:"run_on_host"` // If true, run on host instead of target
|
||
VMID string `json:"vmid,omitempty"`
|
||
TargetHost string `json:"target_host,omitempty"` // Explicit host for routing
|
||
OrgID string `json:"-"`
|
||
ActionID string `json:"-"`
|
||
}
|
||
|
||
// RunCommandResponse represents the result of running a command
|
||
type RunCommandResponse struct {
|
||
Output string `json:"output"`
|
||
Success bool `json:"success"`
|
||
Error string `json:"error,omitempty"`
|
||
}
|
||
|
||
// RunCommand executes a single command via the agent (used for approved commands)
|
||
func (s *Service) RunCommand(ctx context.Context, req RunCommandRequest) (*RunCommandResponse, error) {
|
||
if s.agentServer == nil {
|
||
return &RunCommandResponse{Success: false, Error: "Agent server not available"}, nil
|
||
}
|
||
|
||
// Build an ExecuteRequest from the RunCommandRequest
|
||
execReq := ExecuteRequest{
|
||
TargetType: req.TargetType,
|
||
TargetID: req.TargetID,
|
||
Context: make(map[string]interface{}),
|
||
}
|
||
|
||
// If running on host, override target type
|
||
if req.RunOnHost {
|
||
execReq.TargetType = agentcapabilities.ActionTargetTypeAgent
|
||
// Keep the original target info for routing
|
||
}
|
||
|
||
// Add VMID to context if provided
|
||
if req.VMID != "" {
|
||
execReq.Context["vmid"] = req.VMID
|
||
}
|
||
|
||
// If target_host is specified, set it in context for routing
|
||
if req.TargetHost != "" {
|
||
execReq.Context["node"] = req.TargetHost
|
||
log.Debug().
|
||
Str("target_host", req.TargetHost).
|
||
Str("command", req.Command).
|
||
Msg("RunCommand using explicit target_host for routing")
|
||
}
|
||
|
||
output, err := s.executeOnAgentWithAuthorization(ctx, execReq, req.Command, req.ApprovalID, req.OrgID, req.ActionID)
|
||
if err != nil {
|
||
return &RunCommandResponse{
|
||
Success: false,
|
||
Error: err.Error(),
|
||
Output: output,
|
||
}, nil
|
||
}
|
||
|
||
return &RunCommandResponse{
|
||
Success: true,
|
||
Output: output,
|
||
}, nil
|
||
}
|
||
|
||
// buildSystemPrompt creates the system prompt based on the request context
|
||
func (s *Service) buildSystemPrompt(req ExecuteRequest, destinationModel string) string {
|
||
commandApproval := `Pulse has a built-in approval system:
|
||
- Safe commands (read-only: ls, df, cat, ps) execute immediately
|
||
- Destructive commands (rm, service restart, apt install) require user approval
|
||
- Blocked commands are never executed
|
||
Execute commands normally - Pulse handles the approval flow automatically.`
|
||
if req.RequireCommandApproval {
|
||
commandApproval = `This handoff is approval-bound:
|
||
- Do not run commands just because they look safe.
|
||
- Any run_command tool call will pause for explicit operator approval before execution.
|
||
- Prefer existing metrics, alert context, and known system state before requesting command approval.
|
||
- Blocked commands are never executed.`
|
||
}
|
||
|
||
prompt := fmt.Sprintf(`You are Pulse's diagnostic assistant for Proxmox, Docker, and Kubernetes homelabs.
|
||
|
||
## Command Approval
|
||
%s
|
||
|
||
## Command Execution
|
||
- run_on_host=true: Run on PVE/Docker host (pct, qm, vzdump, docker commands)
|
||
- run_on_host=false: Run inside the container/VM
|
||
- target_host: Required when using run_on_host=true - use the node name from context
|
||
|
||
Example for LXC 106 on node 'minipc':
|
||
- Inside container: run_command(command="df -h", run_on_host=false)
|
||
- On host: run_command(command="pct exec 106 -- df -h", run_on_host=true, target_host="minipc")
|
||
|
||
If you get "Configuration file does not exist" - wrong host, check target_host.
|
||
|
||
## LXC Management
|
||
Pulse manages LXC containers agentlessly - there is no Pulse process inside containers.
|
||
- run_on_host=false: Pulse routes commands into the LXC via the PVE host
|
||
- run_on_host=true + target_host: Runs pct/qm commands on the PVE host directly
|
||
|
||
## URL Discovery
|
||
When finding web URLs for resources:
|
||
1. Check listening ports: ss -tlnp | grep LISTEN
|
||
2. Get IP: hostname -I | awk '{print $1}'
|
||
3. Test with fetch_url to verify
|
||
4. Save with set_resource_url tool
|
||
|
||
resource_id format: {instance}:{node}:{vmid} (colons, not dashes)
|
||
Example: "homelab:pve-node:201" for VMID 201 on node pve-node in instance homelab
|
||
|
||
## Installing/Updating Pulse
|
||
curl -sSL https://raw.githubusercontent.com/rcourtman/Pulse/main/install.sh | bash
|
||
After: systemctl enable pulse && systemctl start pulse
|
||
Latest version: https://api.github.com/repos/rcourtman/Pulse/releases/latest`, commandApproval)
|
||
|
||
// Add custom context from AI settings (user's infrastructure description)
|
||
s.mu.RLock()
|
||
cfg := s.cfg
|
||
s.mu.RUnlock()
|
||
if cfg != nil && cfg.CustomContext != "" {
|
||
prompt += "\n\n## User's Infrastructure Description\n"
|
||
prompt += "The user has provided this context about their infrastructure:\n\n"
|
||
prompt += cfg.CustomContext
|
||
}
|
||
|
||
// Add connected infrastructure info via unified resource model
|
||
s.mu.RLock()
|
||
hasUnifiedResourceProvider := s.unifiedResourceProvider != nil
|
||
s.mu.RUnlock()
|
||
|
||
if hasUnifiedResourceProvider {
|
||
if strings.TrimSpace(destinationModel) != "" {
|
||
prompt += s.buildUnifiedResourceContextForModel(destinationModel)
|
||
} else {
|
||
prompt += s.buildUnifiedResourceContext()
|
||
}
|
||
} else {
|
||
log.Warn().Msg("AI context: unified resource provider not available, infrastructure context will be limited")
|
||
}
|
||
|
||
// Add user annotations from all resources (global context)
|
||
prompt += s.buildUserAnnotationsContext()
|
||
|
||
// Add current alert status - this gives AI awareness of active issues
|
||
prompt += s.buildAlertContext()
|
||
|
||
// Add incident memory for alert or resource context
|
||
alertIdentifier := ""
|
||
resourceID := req.TargetID
|
||
if req.Context != nil {
|
||
alertIdentifier = extractAlertIdentifier(req.Context)
|
||
|
||
if resourceID == "" {
|
||
if val, ok := req.Context["resourceId"].(string); ok && val != "" {
|
||
resourceID = val
|
||
} else if val, ok := req.Context["resource_id"].(string); ok && val != "" {
|
||
resourceID = val
|
||
} else if val, ok := req.Context["guest_id"].(string); ok && val != "" {
|
||
resourceID = val
|
||
}
|
||
}
|
||
}
|
||
|
||
if incidentContext := s.buildIncidentContext(resourceID, alertIdentifier); incidentContext != "" {
|
||
prompt += incidentContext
|
||
}
|
||
|
||
// Add all saved knowledge when no specific target is selected
|
||
// This gives the AI context about everything learned from previous sessions
|
||
if req.TargetType == "" && s.knowledgeStore != nil {
|
||
prompt += s.knowledgeStore.FormatAllForContext()
|
||
}
|
||
|
||
// Add target context if provided
|
||
if req.TargetType != "" {
|
||
guestName := ""
|
||
if name, ok := req.Context["guestName"].(string); ok {
|
||
guestName = name
|
||
} else if name, ok := req.Context["name"].(string); ok {
|
||
guestName = name
|
||
}
|
||
|
||
if guestName != "" {
|
||
// Include the node in the focus header so AI can't miss it for routing
|
||
nodeName := ""
|
||
if node, ok := req.Context["node"].(string); ok && node != "" {
|
||
nodeName = node
|
||
} else if node, ok := req.Context["guest_node"].(string); ok && node != "" {
|
||
nodeName = node
|
||
}
|
||
if nodeName != "" {
|
||
prompt += fmt.Sprintf("\n\n## Current Focus\nYou are analyzing **%s** (%s on node **%s**)\n**ROUTING: When using run_on_host=true, set target_host=\"%s\"**",
|
||
guestName, req.TargetType, nodeName, nodeName)
|
||
} else {
|
||
prompt += fmt.Sprintf("\n\n## Current Focus\nYou are analyzing **%s** (%s)", guestName, req.TargetType)
|
||
}
|
||
} else if req.TargetID != "" {
|
||
prompt += fmt.Sprintf("\n\n## Current Focus\nYou are analyzing %s '%s'", req.TargetType, req.TargetID)
|
||
}
|
||
|
||
// Add prior action history for this resource as neutral context.
|
||
prompt += s.buildRemediationContext(req.TargetID)
|
||
|
||
}
|
||
|
||
// If we're helping with a patrol finding, provide the finding ID and the
|
||
// available lifecycle tools without forcing a tool choice.
|
||
if req.FindingID != "" {
|
||
prompt += fmt.Sprintf("\n\n## Patrol Finding Context\n"+
|
||
"You are helping with patrol finding **%s**.\n\n"+
|
||
"Lifecycle tools are available when current evidence supports them:\n"+
|
||
"- `%s` records that the underlying issue has actually been fixed.\n"+
|
||
"- `%s` records a false positive or expected behavior.\n"+
|
||
"Only call a lifecycle tool when that is the model's conclusion from the current evidence.\n",
|
||
req.FindingID,
|
||
agentcapabilities.ResolveFindingCapabilityName,
|
||
agentcapabilities.DismissFindingCapabilityName)
|
||
}
|
||
|
||
// Add any provided context in a structured way
|
||
if len(req.Context) > 0 {
|
||
prompt += "\n\n## Current Metrics and State"
|
||
|
||
// Group metrics by category for better readability
|
||
categories := map[string][]string{
|
||
"Identity": {"name", "guestName", "type", "vmid", "node", "guest_node", "status", "uptime"},
|
||
"CPU": {"cpu_usage", "cpu_cores"},
|
||
"Memory": {"memory_used", "memory_total", "memory_usage", "memory_balloon", "swap_used", "swap_total"},
|
||
"Disk": {"disk_used", "disk_total", "disk_usage"},
|
||
"I/O Rates": {"disk_read_rate", "disk_write_rate", "network_in_rate", "network_out_rate"},
|
||
"Backup": {"backup_status", "last_backup", "days_since_backup"},
|
||
"System Info": {"os_name", "os_version", "guest_agent", "ip_addresses", "tags"},
|
||
"User Context": {"user_notes", "user_annotations"},
|
||
}
|
||
|
||
categoryOrder := []string{"Identity", "User Context", "Backup", "CPU", "Memory", "Disk", "I/O Rates", "System Info"}
|
||
|
||
for _, category := range categoryOrder {
|
||
keys := categories[category]
|
||
hasValues := false
|
||
categoryContent := ""
|
||
|
||
for _, k := range keys {
|
||
if v, ok := req.Context[k]; ok && v != nil && v != "" {
|
||
if !hasValues {
|
||
categoryContent = fmt.Sprintf("\n### %s", category)
|
||
hasValues = true
|
||
}
|
||
categoryContent += fmt.Sprintf("\n- %s: %v", formatContextKey(k), v)
|
||
}
|
||
}
|
||
|
||
if hasValues {
|
||
prompt += categoryContent
|
||
}
|
||
}
|
||
|
||
// Add any remaining context that wasn't categorized
|
||
for k, v := range req.Context {
|
||
found := false
|
||
for _, keys := range categories {
|
||
for _, key := range keys {
|
||
if k == key {
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
if found {
|
||
break
|
||
}
|
||
}
|
||
if !found && v != nil && v != "" {
|
||
prompt += fmt.Sprintf("\n- %s: %v", formatContextKey(k), v)
|
||
}
|
||
}
|
||
|
||
// Add enriched historical context (baselines, trends, predictions)
|
||
// This is the Pulse Pro value-add that Claude Code can't replicate
|
||
prompt += s.buildEnrichedResourceContext(req.TargetID, req.TargetType, req.Context)
|
||
}
|
||
|
||
return prompt
|
||
}
|
||
|
||
// formatContextKey converts snake_case keys to readable labels
|
||
func formatContextKey(key string) string {
|
||
replacements := map[string]string{
|
||
"guestName": "Guest Name",
|
||
"name": "Name",
|
||
"type": "Type",
|
||
"vmid": "VMID",
|
||
"node": "Host Node",
|
||
"guest_node": "Host Node",
|
||
"status": "Status",
|
||
"uptime": "Uptime",
|
||
"cpu_usage": "CPU Usage",
|
||
"cpu_cores": "CPU Cores",
|
||
"memory_used": "Memory Used",
|
||
"memory_total": "Memory Total",
|
||
"memory_usage": "Memory Usage",
|
||
"memory_balloon": "Memory Balloon",
|
||
"swap_used": "Swap Used",
|
||
"swap_total": "Swap Total",
|
||
"disk_used": "Disk Used",
|
||
"disk_total": "Disk Total",
|
||
"disk_usage": "Disk Usage",
|
||
"disk_read_rate": "Disk Read Rate",
|
||
"disk_write_rate": "Disk Write Rate",
|
||
"network_in_rate": "Network In Rate",
|
||
"network_out_rate": "Network Out Rate",
|
||
"backup_status": "Backup Status",
|
||
"last_backup": "Last Backup",
|
||
"days_since_backup": "Days Since Backup",
|
||
"os_name": "OS Name",
|
||
"os_version": "OS Version",
|
||
"guest_agent": "Guest Agent",
|
||
"ip_addresses": "IP Addresses",
|
||
"tags": "Tags",
|
||
"user_notes": "User Notes",
|
||
"user_annotations": "User Annotations",
|
||
}
|
||
|
||
if label, ok := replacements[key]; ok {
|
||
return label
|
||
}
|
||
return key
|
||
}
|
||
|
||
// buildUserAnnotationsContext gathers all user annotations from guests and docker containers
|
||
// These provide infrastructure context that the AI should know about for any query
|
||
func (s *Service) buildUserAnnotationsContext() string {
|
||
// Return empty if persistence is not available
|
||
if s.persistence == nil {
|
||
return ""
|
||
}
|
||
|
||
var annotations []string
|
||
|
||
// Load guest metadata
|
||
guestStore, err := s.persistence.LoadGuestMetadata()
|
||
if err != nil {
|
||
log.Warn().Err(err).Msg("failed to load guest metadata for AI context")
|
||
} else {
|
||
guestMeta := guestStore.GetAll()
|
||
log.Debug().Int("count", len(guestMeta)).Msg("loaded guest metadata for AI context")
|
||
for id, meta := range guestMeta {
|
||
if meta != nil && len(meta.Notes) > 0 {
|
||
// Use LastKnownName if available, otherwise use ID
|
||
name := meta.LastKnownName
|
||
if name == "" {
|
||
name = id
|
||
}
|
||
for _, note := range meta.Notes {
|
||
annotations = append(annotations, fmt.Sprintf("- Guest '%s': %s", name, note))
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Load docker metadata - include host info for context
|
||
dockerStore, err := s.persistence.LoadDockerMetadata()
|
||
if err != nil {
|
||
log.Warn().Err(err).Msg("failed to load docker metadata for AI context")
|
||
} else {
|
||
dockerMeta := dockerStore.GetAll()
|
||
log.Debug().Int("count", len(dockerMeta)).Msg("loaded docker metadata for AI context")
|
||
for id, meta := range dockerMeta {
|
||
if meta != nil && len(meta.Notes) > 0 {
|
||
// Extract host and container info from ID (format: hostid:container:containerid)
|
||
name := id
|
||
hostInfo := ""
|
||
parts := strings.Split(id, ":")
|
||
if len(parts) >= 3 {
|
||
hostInfo = parts[0] // First part is the host identifier
|
||
containerID := parts[2]
|
||
if len(containerID) > 12 {
|
||
containerID = containerID[:12]
|
||
}
|
||
name = fmt.Sprintf("Docker container %s", containerID)
|
||
}
|
||
log.Debug().Str("name", name).Str("host", hostInfo).Int("notes", len(meta.Notes)).Msg("found docker container with annotations")
|
||
for _, note := range meta.Notes {
|
||
if hostInfo != "" {
|
||
annotations = append(annotations, fmt.Sprintf("- %s (on host '%s'): %s", name, hostInfo, note))
|
||
} else {
|
||
annotations = append(annotations, fmt.Sprintf("- %s: %s", name, note))
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
log.Debug().Int("total_annotations", len(annotations)).Msg("built user annotations context")
|
||
|
||
if len(annotations) == 0 {
|
||
return ""
|
||
}
|
||
|
||
return "\n\n## User Infrastructure Notes\nThe user has added these annotations to describe their infrastructure. USE THESE to understand relationships between systems:\n" + strings.Join(annotations, "\n")
|
||
}
|
||
|
||
// TestConnection tests the AI provider connection
|
||
// Tests the provider for the currently configured default model
|
||
func (s *Service) TestConnection(ctx context.Context) error {
|
||
s.mu.RLock()
|
||
cfg := s.cfg
|
||
defaultProvider := s.provider
|
||
persistence := s.persistence
|
||
s.mu.RUnlock()
|
||
|
||
// Load config if not available
|
||
if cfg == nil {
|
||
if persistence == nil {
|
||
return fmt.Errorf("Pulse Assistant config persistence unavailable")
|
||
}
|
||
var err error
|
||
cfg, err = persistence.LoadAIConfig()
|
||
if err != nil {
|
||
return fmt.Errorf("failed to load Pulse Assistant config: %w", err)
|
||
}
|
||
}
|
||
|
||
if cfg == nil || !cfg.IsConfigured() {
|
||
return fmt.Errorf("no provider configured")
|
||
}
|
||
|
||
// Try to create a provider for the current default model
|
||
selectedModel, err := ResolveConfiguredModel(ctx, cfg)
|
||
if err != nil {
|
||
if defaultProvider != nil {
|
||
return defaultProvider.TestConnection(ctx)
|
||
}
|
||
return err
|
||
}
|
||
provider, err := providers.NewForModel(cfg, selectedModel)
|
||
if err != nil {
|
||
// Fall back to default provider or NewFromConfig
|
||
log.Debug().Err(err).Str("model", selectedModel).Msg("could not create provider for model, using fallback")
|
||
if defaultProvider != nil {
|
||
provider = defaultProvider
|
||
} else {
|
||
return fmt.Errorf("failed to create fallback AI provider: %w", err)
|
||
}
|
||
}
|
||
|
||
return provider.TestConnection(ctx)
|
||
}
|
||
|
||
// ListModels fetches available models from ALL configured AI providers
|
||
// Returns a unified list with models prefixed by provider name
|
||
func (s *Service) ListModels(ctx context.Context) ([]providers.ModelInfo, error) {
|
||
models, _, err := s.ListModelsWithCache(ctx)
|
||
return models, err
|
||
}
|
||
|
||
func (s *Service) ListModelsWithCache(ctx context.Context) ([]providers.ModelInfo, bool, error) {
|
||
s.mu.RLock()
|
||
persistence := s.persistence
|
||
s.mu.RUnlock()
|
||
if persistence == nil {
|
||
return nil, false, fmt.Errorf("Pulse Assistant config persistence unavailable")
|
||
}
|
||
|
||
cfg, err := persistence.LoadAIConfig()
|
||
if err != nil {
|
||
return nil, false, fmt.Errorf("failed to load Pulse Assistant config: %w", err)
|
||
}
|
||
if cfg == nil {
|
||
return nil, false, fmt.Errorf("Pulse Assistant not configured")
|
||
}
|
||
|
||
cacheKey := buildModelsCacheKey(cfg)
|
||
|
||
// If config changed, clear all cached provider entries
|
||
s.modelsCache.mu.Lock()
|
||
if s.modelsCache.key != cacheKey {
|
||
s.modelsCache.key = cacheKey
|
||
s.modelsCache.providers = make(map[string]providerModelsEntry)
|
||
}
|
||
s.modelsCache.mu.Unlock()
|
||
|
||
providersList := cfg.GetConfiguredProviders()
|
||
|
||
allCached := true
|
||
|
||
for _, providerName := range providersList {
|
||
if !cfg.HasProvider(providerName) {
|
||
continue
|
||
}
|
||
|
||
// Check if this provider's cache is still valid
|
||
s.modelsCache.mu.RLock()
|
||
entry, hasEntry := s.modelsCache.providers[providerName]
|
||
valid := hasEntry && s.modelsCache.ttl > 0 && time.Since(entry.at) < s.modelsCache.ttl && len(entry.models) > 0
|
||
s.modelsCache.mu.RUnlock()
|
||
|
||
if valid {
|
||
continue
|
||
}
|
||
|
||
allCached = false
|
||
|
||
// Create provider
|
||
provider, err := providers.NewForProvider(cfg, providerName, "")
|
||
if err != nil {
|
||
log.Debug().Err(err).Str("provider", providerName).Msg("skipping provider - not configured")
|
||
continue
|
||
}
|
||
|
||
// Fetch models from this provider
|
||
models, err := provider.ListModels(ctx)
|
||
if err != nil {
|
||
log.Warn().Err(err).Str("provider", providerName).Msg("failed to fetch models from provider")
|
||
// Keep stale entry (don't overwrite or delete) — the provider's
|
||
// previous models remain visible until a successful fetch replaces them.
|
||
if !hasEntry || len(entry.models) == 0 {
|
||
if fallbackModels := providerCatalogFallbackModels(providerName); len(fallbackModels) > 0 {
|
||
s.modelsCache.mu.Lock()
|
||
s.modelsCache.providers[providerName] = providerModelsEntry{
|
||
at: time.Now(),
|
||
models: prefixProviderModels(providerName, fallbackModels),
|
||
}
|
||
s.modelsCache.mu.Unlock()
|
||
}
|
||
}
|
||
continue
|
||
}
|
||
models = mergeProviderCatalogFallbackModels(providerName, models)
|
||
|
||
// Build prefixed model list for this provider
|
||
prefixed := prefixProviderModels(providerName, models)
|
||
|
||
s.modelsCache.mu.Lock()
|
||
s.modelsCache.providers[providerName] = providerModelsEntry{
|
||
at: time.Now(),
|
||
models: prefixed,
|
||
}
|
||
s.modelsCache.mu.Unlock()
|
||
}
|
||
|
||
// Aggregate results in stable provider order
|
||
var allModels []providers.ModelInfo
|
||
s.modelsCache.mu.RLock()
|
||
for _, providerName := range providersList {
|
||
if entry, ok := s.modelsCache.providers[providerName]; ok {
|
||
allModels = append(allModels, entry.models...)
|
||
}
|
||
}
|
||
s.modelsCache.mu.RUnlock()
|
||
|
||
return allModels, allCached, nil
|
||
}
|
||
|
||
func prefixProviderModels(providerName string, models []providers.ModelInfo) []providers.ModelInfo {
|
||
prefixed := make([]providers.ModelInfo, 0, len(models))
|
||
for _, m := range models {
|
||
if strings.TrimSpace(m.ID) == "" {
|
||
continue
|
||
}
|
||
description := strings.TrimSpace(m.Description)
|
||
if description == "" {
|
||
description = providerDisplayName(providerName) + ": " + m.ID
|
||
}
|
||
prefixed = append(prefixed, providers.ModelInfo{
|
||
ID: config.FormatModelString(providerName, m.ID),
|
||
Name: m.Name,
|
||
Description: description,
|
||
CreatedAt: m.CreatedAt,
|
||
Notable: m.Notable,
|
||
Provider: providerName,
|
||
})
|
||
}
|
||
return prefixed
|
||
}
|
||
|
||
func mergeProviderCatalogFallbackModels(providerName string, models []providers.ModelInfo) []providers.ModelInfo {
|
||
fallbacks := providerCatalogFallbackModels(providerName)
|
||
if len(fallbacks) == 0 {
|
||
return models
|
||
}
|
||
seen := make(map[string]struct{}, len(models)+len(fallbacks))
|
||
merged := make([]providers.ModelInfo, 0, len(models)+len(fallbacks))
|
||
for _, model := range models {
|
||
id := strings.ToLower(strings.TrimSpace(model.ID))
|
||
if id == "" {
|
||
continue
|
||
}
|
||
seen[id] = struct{}{}
|
||
merged = append(merged, model)
|
||
}
|
||
for _, fallback := range fallbacks {
|
||
id := strings.ToLower(strings.TrimSpace(fallback.ID))
|
||
if id == "" {
|
||
continue
|
||
}
|
||
if _, ok := seen[id]; ok {
|
||
continue
|
||
}
|
||
merged = append(merged, fallback)
|
||
}
|
||
return merged
|
||
}
|
||
|
||
func providerCatalogFallbackModels(providerName string) []providers.ModelInfo {
|
||
fallbacks := config.AIProviderFallbackModels(providerName)
|
||
if len(fallbacks) == 0 {
|
||
return nil
|
||
}
|
||
models := make([]providers.ModelInfo, 0, len(fallbacks))
|
||
for _, fallback := range fallbacks {
|
||
models = append(models, providers.ModelInfo{
|
||
ID: fallback.ID,
|
||
Name: fallback.Name,
|
||
Description: fallback.Description,
|
||
Notable: fallback.Notable,
|
||
})
|
||
}
|
||
return models
|
||
}
|
||
|
||
func buildModelsCacheKey(cfg *config.AIConfig) string {
|
||
if cfg == nil {
|
||
return "nil"
|
||
}
|
||
|
||
var b strings.Builder
|
||
b.WriteString("providers=")
|
||
b.WriteString(strings.Join(cfg.GetConfiguredProviders(), ","))
|
||
b.WriteString("|auth=")
|
||
b.WriteString(string(cfg.AuthMethod))
|
||
b.WriteString("|keys=")
|
||
b.WriteString(aiCredentialCacheSignature(cfg))
|
||
|
||
b.WriteString("|openai_base=")
|
||
b.WriteString(cfg.OpenAIBaseURL)
|
||
b.WriteString("|ollama_base=")
|
||
b.WriteString(cfg.OllamaBaseURL)
|
||
b.WriteString("|ollama_auth=")
|
||
if cfg.OllamaUsername != "" || cfg.OllamaPassword != "" {
|
||
b.WriteString(credentialCacheDigest(cfg.OllamaUsername, cfg.OllamaPassword))
|
||
}
|
||
|
||
return b.String()
|
||
}
|
||
|
||
func aiCredentialCacheSignature(cfg *config.AIConfig) string {
|
||
if cfg == nil {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
for _, providerName := range cfg.GetConfiguredProviders() {
|
||
b.WriteString(providerName)
|
||
b.WriteString("=")
|
||
if key := cfg.GetAPIKeyForProvider(providerName); key != "" {
|
||
b.WriteString(credentialCacheDigest(providerName, key))
|
||
}
|
||
b.WriteString(";")
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
// providerDisplayName returns a user-friendly name for a provider
|
||
func providerDisplayName(provider string) string {
|
||
return config.AIProviderDisplayName(provider)
|
||
}
|
||
|
||
// Reload reloads the AI configuration (call after settings change)
|
||
func (s *Service) Reload() error {
|
||
if err := s.LoadConfig(); err != nil {
|
||
return fmt.Errorf("Reload: %w", err)
|
||
}
|
||
|
||
// Also reload the chat service so patrol picks up model/provider changes
|
||
s.mu.RLock()
|
||
cs := s.chatService
|
||
cfg := s.cfg
|
||
s.mu.RUnlock()
|
||
|
||
if cs != nil && cfg != nil {
|
||
if err := cs.ReloadConfig(context.Background(), cfg); err != nil {
|
||
log.Warn().Err(err).Msg("failed to reload chat service config")
|
||
}
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// buildRemediationContext adds resource-scoped remediation history as neutral context.
|
||
// Pulse must not keyword-match the current problem to old fixes or recommend a
|
||
// prior command; the selected model owns diagnosis and remediation reasoning.
|
||
func (s *Service) buildRemediationContext(resourceID string) string {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol == nil {
|
||
return ""
|
||
}
|
||
|
||
remLog := patrol.GetRemediationLog()
|
||
if remLog == nil {
|
||
return ""
|
||
}
|
||
|
||
if resourceID != "" {
|
||
history := remLog.GetForResource(resourceID, 5)
|
||
if len(history) > 0 {
|
||
var context string
|
||
context += "\n\n## Prior Actions Recorded For This Resource\n"
|
||
context += "These are historical actions only. Decide from current evidence whether they are relevant.\n"
|
||
for _, rec := range history {
|
||
ago := time.Since(rec.Timestamp)
|
||
agoStr := formatDuration(ago)
|
||
context += fmt.Sprintf("- %s ago: %s → `%s` (%s)\n",
|
||
agoStr,
|
||
truncateString(rec.Problem, 50),
|
||
truncateString(rec.Action, 60),
|
||
rec.Outcome)
|
||
}
|
||
return context
|
||
}
|
||
}
|
||
|
||
return ""
|
||
}
|
||
|
||
// buildIncidentContext adds alert-scoped incident memory first, then the
|
||
// canonical resource timeline and relationship context when a resource is known.
|
||
// The incident store is a derived investigation view, not the source of truth
|
||
// for durable backend history.
|
||
func (s *Service) buildIncidentContext(resourceID, alertIdentifier string) string {
|
||
s.mu.RLock()
|
||
store := s.incidentStore
|
||
s.mu.RUnlock()
|
||
|
||
var sections []string
|
||
|
||
if store != nil {
|
||
if alertIdentifier != "" {
|
||
if text := store.FormatForAlert(alertIdentifier, 8); text != "" {
|
||
sections = append(sections, text)
|
||
}
|
||
} else if resourceID != "" {
|
||
if text := store.FormatForResource(resourceID, 4); text != "" {
|
||
sections = append(sections, text)
|
||
}
|
||
}
|
||
}
|
||
|
||
if resourceID != "" {
|
||
if recentChanges := s.buildRecentResourceChangesContext(resourceID); recentChanges != "" {
|
||
sections = append(sections, recentChanges)
|
||
}
|
||
if relationshipContext := s.buildResourceRelationshipContext(resourceID); relationshipContext != "" {
|
||
sections = append(sections, relationshipContext)
|
||
}
|
||
}
|
||
|
||
if len(sections) == 0 {
|
||
return ""
|
||
}
|
||
return strings.Join(sections, "")
|
||
}
|
||
|
||
// RecordIncidentAnalysis stores an AI analysis event for an alert.
|
||
func (s *Service) RecordIncidentAnalysis(alertIdentifier, summary string, details map[string]interface{}) {
|
||
if alertIdentifier == "" {
|
||
return
|
||
}
|
||
s.mu.RLock()
|
||
store := s.incidentStore
|
||
s.mu.RUnlock()
|
||
if store == nil {
|
||
return
|
||
}
|
||
store.RecordAnalysis(alertIdentifier, summary, details)
|
||
}
|
||
|
||
// RecordIncidentRunbook stores a runbook execution event for an alert.
|
||
func (s *Service) RecordIncidentRunbook(resourceID, alertIdentifier, runbookID, title string, outcome memory.Outcome, automatic bool, message string) {
|
||
if alertIdentifier == "" || runbookID == "" {
|
||
return
|
||
}
|
||
s.mu.RLock()
|
||
store := s.incidentStore
|
||
s.mu.RUnlock()
|
||
if store != nil {
|
||
store.RecordRunbook(alertIdentifier, runbookID, title, string(outcome), automatic, message)
|
||
}
|
||
if resourceID == "" {
|
||
resourceID = s.lookupIncidentResourceIdentifier(alertIdentifier)
|
||
}
|
||
actor := ""
|
||
if automatic {
|
||
actor = "agent:pulse-patrol"
|
||
}
|
||
s.recordCanonicalResourceChange(unifiedresources.BuildRunbookExecutionChange(
|
||
resourceID,
|
||
alertIdentifier,
|
||
actor,
|
||
runbookID,
|
||
title,
|
||
string(outcome),
|
||
automatic,
|
||
message,
|
||
nil,
|
||
))
|
||
}
|
||
|
||
func extractAlertIdentifier(ctx map[string]interface{}) string {
|
||
if ctx == nil {
|
||
return ""
|
||
}
|
||
if alertIdentifier, ok := ctx["alertIdentifier"].(string); ok && alertIdentifier != "" {
|
||
return alertIdentifier
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func extractIncidentResourceIdentifier(targetID string, ctx map[string]interface{}) string {
|
||
targetID = strings.TrimSpace(targetID)
|
||
if targetID != "" {
|
||
return targetID
|
||
}
|
||
if ctx == nil {
|
||
return ""
|
||
}
|
||
for _, key := range []string{"resourceID", "resource_id", "resourceId"} {
|
||
if value, ok := ctx[key].(string); ok && strings.TrimSpace(value) != "" {
|
||
return strings.TrimSpace(value)
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func (s *Service) lookupIncidentResourceIdentifier(alertIdentifier string) string {
|
||
if strings.TrimSpace(alertIdentifier) == "" {
|
||
return ""
|
||
}
|
||
s.mu.RLock()
|
||
store := s.incidentStore
|
||
s.mu.RUnlock()
|
||
if store == nil {
|
||
return ""
|
||
}
|
||
page, err := store.QueryIncidents(memory.IncidentQuery{AlertIdentifier: alertIdentifier, Limit: 1})
|
||
if err != nil {
|
||
log.Warn().Err(err).Msg("Canonical incident resource lookup unavailable")
|
||
return ""
|
||
}
|
||
if len(page.Incidents) == 0 {
|
||
return ""
|
||
}
|
||
return strings.TrimSpace(page.Incidents[0].ResourceID)
|
||
}
|
||
|
||
func (s *Service) recordCanonicalResourceChange(change *unifiedresources.ResourceChange) {
|
||
if s == nil || change == nil {
|
||
return
|
||
}
|
||
|
||
s.mu.RLock()
|
||
store := s.resourceExportStore
|
||
orgID := strings.TrimSpace(s.orgID)
|
||
storeOrgID := strings.TrimSpace(s.resourceExportStoreOrgID)
|
||
s.mu.RUnlock()
|
||
|
||
if store == nil {
|
||
return
|
||
}
|
||
if storeOrgID != "" && orgID != "" && storeOrgID != orgID {
|
||
return
|
||
}
|
||
if err := store.RecordChange(*change); err != nil {
|
||
log.Warn().
|
||
Err(err).
|
||
Str("resource_id", change.ResourceID).
|
||
Str("kind", string(change.Kind)).
|
||
Msg("failed to record canonical incident timeline change")
|
||
}
|
||
}
|
||
|
||
func (s *Service) buildRecentResourceChangesContext(resourceID string) string {
|
||
resourceID = strings.TrimSpace(resourceID)
|
||
if resourceID == "" {
|
||
return ""
|
||
}
|
||
|
||
s.mu.RLock()
|
||
store := s.resourceExportStore
|
||
orgID := strings.TrimSpace(s.orgID)
|
||
storeOrgID := strings.TrimSpace(s.resourceExportStoreOrgID)
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if store != nil && (storeOrgID == "" || storeOrgID == orgID) {
|
||
changes, err := store.GetRecentChanges(resourceID, time.Now().Add(-24*time.Hour), 5)
|
||
if err != nil {
|
||
log.Warn().
|
||
Err(err).
|
||
Str("resource_id", resourceID).
|
||
Msg("failed to load canonical resource timeline context")
|
||
} else if len(changes) > 0 {
|
||
if len(changes) > 3 {
|
||
changes = changes[:3]
|
||
}
|
||
if section := unifiedresources.FormatResourceRecentChangesContext(changes, false, "###"); section != "" {
|
||
return section
|
||
}
|
||
}
|
||
}
|
||
|
||
if patrol == nil {
|
||
return ""
|
||
}
|
||
|
||
changeDetector := patrol.GetChangeDetector()
|
||
if changeDetector == nil {
|
||
return ""
|
||
}
|
||
|
||
changes := changeDetector.GetChangesForResource(resourceID, 5)
|
||
if len(changes) == 0 {
|
||
return ""
|
||
}
|
||
if len(changes) > 3 {
|
||
changes = changes[:3]
|
||
}
|
||
return memory.FormatRecentChangesContext(changes, false, "###")
|
||
}
|
||
|
||
type canonicalResourceGetter interface {
|
||
Get(id string) (*unifiedresources.Resource, bool)
|
||
}
|
||
|
||
func (s *Service) buildResourceRelationshipContext(resourceID string) string {
|
||
resourceID = strings.TrimSpace(resourceID)
|
||
if resourceID == "" {
|
||
return ""
|
||
}
|
||
|
||
s.mu.RLock()
|
||
rs := s.readState
|
||
s.mu.RUnlock()
|
||
|
||
if rs == nil {
|
||
return ""
|
||
}
|
||
|
||
getter, ok := rs.(canonicalResourceGetter)
|
||
if !ok {
|
||
return ""
|
||
}
|
||
|
||
resource, ok := getter.Get(resourceID)
|
||
if !ok || resource == nil {
|
||
return ""
|
||
}
|
||
return unifiedresources.FormatResourceRelationshipContext(resource, 3)
|
||
}
|
||
|
||
// truncateString truncates a string to maxLen characters.
|
||
func truncateString(s string, maxLen int) string {
|
||
if len(s) <= maxLen {
|
||
return s
|
||
}
|
||
return s[:maxLen-3] + "..."
|
||
}
|
||
|
||
// buildEnrichedResourceContext adds historical intelligence to regular AI chat
|
||
// This is what makes Pulse Pro valuable - context that Claude Code can't have:
|
||
// - Baseline comparisons ("this is 2x normal")
|
||
// - Trend analysis ("memory growing 5%/day")
|
||
// - Predictions ("disk full in 12 days")
|
||
// - Recent changes ("config changed 2 hours ago")
|
||
func (s *Service) buildEnrichedResourceContext(resourceID, _ string, currentMetrics map[string]interface{}) string {
|
||
s.mu.RLock()
|
||
patrol := s.patrolService
|
||
s.mu.RUnlock()
|
||
|
||
if patrol == nil {
|
||
return ""
|
||
}
|
||
|
||
var sections []string
|
||
|
||
// Get baseline store for comparisons
|
||
baselineStore := patrol.GetBaselineStore()
|
||
if baselineStore != nil {
|
||
var baselineInfo []string
|
||
|
||
// Helper to get raw numeric value from context
|
||
// Prefers _raw fields, falls back to regular fields
|
||
getRawValue := func(rawKey, formattedKey string) (float64, bool) {
|
||
// Try raw value first
|
||
if val, ok := currentMetrics[rawKey].(float64); ok && val > 0 {
|
||
return val, true
|
||
}
|
||
// Fallback to formatted value if it's already a float
|
||
if val, ok := currentMetrics[formattedKey].(float64); ok && val > 0 {
|
||
return val, true
|
||
}
|
||
return 0, false
|
||
}
|
||
|
||
// Helper to format baseline comparison with status
|
||
// ALWAYS shows baseline info when available - this is Pulse Pro's value-add
|
||
formatBaselineComparison := func(metric string, currentVal float64, bl *baseline.MetricBaseline) string {
|
||
if bl.SampleCount < 10 {
|
||
return fmt.Sprintf("- %s: %.1f%% (baseline learning: %d/10 samples)", metric, currentVal, bl.SampleCount)
|
||
}
|
||
|
||
ratio := currentVal / bl.Mean
|
||
status := "✓ normal"
|
||
if ratio > 2.0 {
|
||
status = "🔴 **ANOMALY**"
|
||
} else if ratio > 1.5 {
|
||
status = "🟡 elevated"
|
||
} else if ratio < 0.3 {
|
||
status = "🔵 unusually low"
|
||
} else if ratio < 0.5 {
|
||
status = "low"
|
||
}
|
||
|
||
return fmt.Sprintf("- %s: %.1f%% vs baseline %.1f%% (σ=%.1f) → %s", metric, currentVal, bl.Mean, bl.StdDev, status)
|
||
}
|
||
|
||
// Check CPU baseline - always show if we have data
|
||
if cpuVal, ok := getRawValue("cpu_usage_raw", "cpu_usage"); ok {
|
||
if bl, exists := baselineStore.GetBaseline(resourceID, "cpu"); exists {
|
||
baselineInfo = append(baselineInfo, formatBaselineComparison("CPU", cpuVal, bl))
|
||
}
|
||
}
|
||
|
||
// Check memory baseline - always show if we have data
|
||
if memVal, ok := getRawValue("memory_usage_raw", "memory_usage"); ok {
|
||
if bl, exists := baselineStore.GetBaseline(resourceID, "memory"); exists {
|
||
baselineInfo = append(baselineInfo, formatBaselineComparison("Memory", memVal, bl))
|
||
}
|
||
}
|
||
|
||
// Check disk baseline - always show if we have data
|
||
if diskVal, ok := getRawValue("disk_usage_raw", "disk_usage"); ok {
|
||
if bl, exists := baselineStore.GetBaseline(resourceID, "disk"); exists {
|
||
baselineInfo = append(baselineInfo, formatBaselineComparison("Disk", diskVal, bl))
|
||
}
|
||
}
|
||
|
||
if len(baselineInfo) > 0 {
|
||
sections = append(sections, "### Learned Baselines (7-day patterns)\n"+strings.Join(baselineInfo, "\n"))
|
||
}
|
||
}
|
||
|
||
// Compute trends from metrics history (this works immediately, no learning period)
|
||
metricsHistory := patrol.GetMetricsHistoryProvider()
|
||
if metricsHistory != nil {
|
||
var trendInfo []string
|
||
duration := 24 * time.Hour // Last 24 hours
|
||
|
||
// Get metrics for this resource
|
||
// Try guest metrics first, then node metrics
|
||
allMetrics := metricsHistory.GetAllGuestMetrics(resourceID, duration)
|
||
if len(allMetrics) == 0 {
|
||
// Try as node
|
||
cpuPoints := metricsHistory.GetNodeMetrics(resourceID, "cpu", duration)
|
||
memPoints := metricsHistory.GetNodeMetrics(resourceID, "memory", duration)
|
||
if len(cpuPoints) > 0 {
|
||
allMetrics["cpu"] = cpuPoints
|
||
}
|
||
if len(memPoints) > 0 {
|
||
allMetrics["memory"] = memPoints
|
||
}
|
||
}
|
||
|
||
// Analyze trends for each metric
|
||
for metric, points := range allMetrics {
|
||
if len(points) < 3 { // Need at least 3 points for trends
|
||
continue
|
||
}
|
||
|
||
// Get first and last values
|
||
first := points[0].Value
|
||
last := points[len(points)-1].Value
|
||
duration := points[len(points)-1].Timestamp.Sub(points[0].Timestamp)
|
||
|
||
if duration < 30*time.Minute { // Need at least 30min of data
|
||
continue
|
||
}
|
||
|
||
// Calculate rate of change per day
|
||
change := last - first
|
||
hoursSpan := duration.Hours()
|
||
ratePerDay := change * (24 / hoursSpan)
|
||
|
||
// Only report significant trends
|
||
if metric == "disk" && ratePerDay > 0.5 { // Growing by >0.5GB/day
|
||
trendInfo = append(trendInfo, fmt.Sprintf("**Disk** growing %.1f GB/day over last %.0fh",
|
||
ratePerDay, hoursSpan))
|
||
} else if metric == "memory" && absFloat(ratePerDay) > 2 { // >2% change per day
|
||
direction := "growing"
|
||
if ratePerDay < 0 {
|
||
direction = "declining"
|
||
}
|
||
trendInfo = append(trendInfo, fmt.Sprintf("**Memory** %s %.1f%%/day over last %.0fh",
|
||
direction, absFloat(ratePerDay), hoursSpan))
|
||
} else if metric == "cpu" && absFloat(ratePerDay) > 5 { // >5% change per day
|
||
direction := "increasing"
|
||
if ratePerDay < 0 {
|
||
direction = "decreasing"
|
||
}
|
||
trendInfo = append(trendInfo, fmt.Sprintf("**CPU** %s %.1f%%/day over last %.0fh",
|
||
direction, absFloat(ratePerDay), hoursSpan))
|
||
}
|
||
}
|
||
|
||
if len(trendInfo) > 0 {
|
||
sections = append(sections, "### Trends (24h)\n"+strings.Join(trendInfo, "\n"))
|
||
}
|
||
}
|
||
|
||
// Get pattern detector for predictions
|
||
patternDetector := patrol.GetPatternDetector()
|
||
if patternDetector != nil {
|
||
predictions := patternDetector.GetPredictionsForResource(resourceID)
|
||
if len(predictions) > 0 {
|
||
var predInfo []string
|
||
for _, pred := range predictions {
|
||
if pred.DaysUntil < 30 { // Only show predictions within 30 days
|
||
predInfo = append(predInfo, fmt.Sprintf("**%s** predicted in %.0f days (%.0f%% confidence)",
|
||
pred.EventType, pred.DaysUntil, pred.Confidence*100))
|
||
}
|
||
}
|
||
if len(predInfo) > 0 {
|
||
sections = append(sections, "### Predictions\n"+strings.Join(predInfo, "\n"))
|
||
}
|
||
}
|
||
}
|
||
|
||
// Get change detector for recent changes
|
||
if recentChanges := s.buildRecentResourceChangesContext(resourceID); recentChanges != "" {
|
||
sections = append(sections, recentChanges)
|
||
}
|
||
|
||
// Get canonical relationship context from unified resources.
|
||
if relationshipContext := s.buildResourceRelationshipContext(resourceID); relationshipContext != "" {
|
||
sections = append(sections, relationshipContext)
|
||
}
|
||
|
||
// Get shared correlation context for learned related resources.
|
||
if intel := patrol.GetIntelligence(); intel != nil {
|
||
if ctx := intel.FormatCorrelationsContext(resourceID); ctx != "" {
|
||
sections = append(sections, ctx)
|
||
}
|
||
}
|
||
|
||
// Get alert history for this resource (this works immediately, uses existing alert data)
|
||
s.mu.RLock()
|
||
alertProvider := s.alertProvider
|
||
s.mu.RUnlock()
|
||
|
||
if alertProvider != nil {
|
||
// Get active alerts for this resource
|
||
activeAlerts := alertProvider.GetAlertsByResource(resourceID)
|
||
|
||
// Get historical alerts (last 20)
|
||
historicalAlerts := alertProvider.GetAlertHistory(resourceID, 20)
|
||
|
||
if len(activeAlerts) > 0 || len(historicalAlerts) > 0 {
|
||
var alertInfo []string
|
||
|
||
if len(activeAlerts) > 0 {
|
||
alertInfo = append(alertInfo, fmt.Sprintf("**%d active alert(s)** right now", len(activeAlerts)))
|
||
for _, a := range activeAlerts {
|
||
alertInfo = append(alertInfo, fmt.Sprintf("- %s %s: %s (active %s)",
|
||
strings.ToUpper(a.Level), a.Type, a.Message, a.Duration))
|
||
}
|
||
}
|
||
|
||
if len(historicalAlerts) > 0 {
|
||
// Count alerts by type in the last 30 days
|
||
alertsByType := make(map[string]int)
|
||
cutoff := time.Now().Add(-30 * 24 * time.Hour)
|
||
for _, a := range historicalAlerts {
|
||
if a.ResolvedTime.After(cutoff) {
|
||
alertsByType[a.Type]++
|
||
}
|
||
}
|
||
|
||
if len(alertsByType) > 0 {
|
||
var typeCounts []string
|
||
for alertType, count := range alertsByType {
|
||
typeCounts = append(typeCounts, fmt.Sprintf("%d %s", count, alertType))
|
||
}
|
||
alertInfo = append(alertInfo, fmt.Sprintf("**Past 30 days:** %s alerts", strings.Join(typeCounts, ", ")))
|
||
}
|
||
}
|
||
|
||
if len(alertInfo) > 0 {
|
||
sections = append(sections, "### Alert History\n"+strings.Join(alertInfo, "\n"))
|
||
}
|
||
}
|
||
}
|
||
|
||
// If no sections but baseline store exists, show learning status
|
||
if len(sections) == 0 {
|
||
baselineStore := patrol.GetBaselineStore()
|
||
if baselineStore != nil {
|
||
// Check if we're still learning baselines for this resource
|
||
if rb, exists := baselineStore.GetResourceBaseline(resourceID); exists {
|
||
// Count metrics with enough samples
|
||
ready := 0
|
||
learning := 0
|
||
for _, mb := range rb.Metrics {
|
||
if mb.SampleCount >= 10 {
|
||
ready++
|
||
} else if mb.SampleCount > 0 {
|
||
learning++
|
||
}
|
||
}
|
||
|
||
if learning > 0 && ready == 0 {
|
||
return "\n\n## Historical Intelligence (Pulse Pro)\n*Learning baselines for this resource... Trends and anomaly detection will be available after more data is collected.*"
|
||
}
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
return "\n\n## Historical Intelligence (Pulse Pro)\n" + strings.Join(sections, "\n\n")
|
||
}
|
||
|
||
// absFloat returns the absolute value of a float64
|
||
func absFloat(x float64) float64 {
|
||
if x < 0 {
|
||
return -x
|
||
}
|
||
return x
|
||
}
|