mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-23 11:46:28 +00:00
chore: AI patrol and baseline improvements
- Enhanced patrol finding display in Alerts.tsx - Improved baseline store with better error handling - Added clean thinking test coverage - Updated patrol logic for better finding management
This commit is contained in:
rcourtman
@@ -2188,7 +2188,7 @@ function OverviewTab(props: {
|
||||
// Map of all findings by ID (including resolved) for displaying patrol run details
|
||||
const [allFindingsMap, setAllFindingsMap] = createSignal<Map<string, Finding>>(new Map());
|
||||
const [lastKnownPatrolAt, setLastKnownPatrolAt] = createSignal<string | null>(null);
|
||||
const [showRunHistory, setShowRunHistory] = createSignal(false);
|
||||
const [showRunHistory, setShowRunHistory] = createSignal(true); // Expanded by default - users on AI Insights want to see this
|
||||
const [forcePatrolLoading, setForcePatrolLoading] = createSignal(false);
|
||||
const [expandedRunId, setExpandedRunId] = createSignal<string | null>(null);
|
||||
const [historyTimeFilter, setHistoryTimeFilter] = createSignal<'24h' | '7d' | 'all'>('all');
|
||||
@@ -2432,11 +2432,7 @@ function OverviewTab(props: {
|
||||
}
|
||||
});
|
||||
setAllFindingsMap(findingsMap);
|
||||
|
||||
// Auto-expand history if most recent run found issues
|
||||
if (runHistory && runHistory.length > 0 && runHistory[0].status !== 'healthy') {
|
||||
setShowRunHistory(true);
|
||||
}
|
||||
// History is expanded by default now, no need for auto-expand logic
|
||||
} catch (_e) {
|
||||
// AI patrol may not be enabled - silently fail
|
||||
}
|
||||
@@ -3513,72 +3509,142 @@ function OverviewTab(props: {
|
||||
{/* Expanded Details Row */}
|
||||
<Show when={expandedRunId() === run.id}>
|
||||
<tr class="bg-gray-50 dark:bg-gray-800/50">
|
||||
<td colspan="5" class="p-3">
|
||||
{/* Show findings from this run */}
|
||||
<td colspan="5" class="p-4">
|
||||
{(() => {
|
||||
const findingsMap = allFindingsMap();
|
||||
const activeFindings: Finding[] = [];
|
||||
const resolvedFindings: Finding[] = [];
|
||||
|
||||
// Only include warning+ severity findings (info/watch are filtered out)
|
||||
const isDisplayableSeverity = (severity: string) =>
|
||||
severity === 'warning' || severity === 'critical';
|
||||
// Categorize ALL findings from this run (not just warning+)
|
||||
const criticalFindings: Finding[] = [];
|
||||
const warningFindings: Finding[] = [];
|
||||
const infoFindings: Finding[] = [];
|
||||
const resolvedFindings: Finding[] = [];
|
||||
|
||||
(run.finding_ids || []).forEach(id => {
|
||||
const finding = findingsMap.get(id);
|
||||
if (finding && isDisplayableSeverity(finding.severity)) {
|
||||
if (finding) {
|
||||
if (finding.resolved_at) {
|
||||
resolvedFindings.push(finding);
|
||||
} else {
|
||||
activeFindings.push(finding);
|
||||
switch (finding.severity) {
|
||||
case 'critical':
|
||||
criticalFindings.push(finding);
|
||||
break;
|
||||
case 'warning':
|
||||
warningFindings.push(finding);
|
||||
break;
|
||||
default:
|
||||
infoFindings.push(finding);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (activeFindings.length === 0 && resolvedFindings.length === 0) return null;
|
||||
const hasActionableFindings = criticalFindings.length > 0 || warningFindings.length > 0;
|
||||
const hasAnyFindings = hasActionableFindings || infoFindings.length > 0 || resolvedFindings.length > 0;
|
||||
|
||||
return (
|
||||
<div>
|
||||
{/* Active findings */}
|
||||
<Show when={activeFindings.length > 0}>
|
||||
<span class="text-[10px] text-gray-500 dark:text-gray-400 uppercase tracking-wider">
|
||||
Active findings:
|
||||
</span>
|
||||
<div class="flex flex-col gap-1 mt-1">
|
||||
<For each={activeFindings}>
|
||||
{(finding) => (
|
||||
<div class="flex items-center gap-2 text-xs">
|
||||
<span class={`px-1.5 py-0.5 rounded text-[10px] ${finding.severity === 'critical' ? 'bg-red-100 dark:bg-red-900/50 text-red-700 dark:text-red-300' :
|
||||
finding.severity === 'warning' ? 'bg-yellow-100 dark:bg-yellow-900/50 text-yellow-700 dark:text-yellow-300' :
|
||||
'bg-blue-100 dark:bg-blue-900/50 text-blue-700 dark:text-blue-300'
|
||||
}`}>
|
||||
{finding.severity}
|
||||
</span>
|
||||
<span class="font-medium text-gray-700 dark:text-gray-300">{finding.title}</span>
|
||||
<span class="text-gray-500 dark:text-gray-400">on {finding.resource_name}</span>
|
||||
</div>
|
||||
)}
|
||||
</For>
|
||||
<div class="space-y-4">
|
||||
{/* Resource Breakdown & Stats Row */}
|
||||
<div class="flex flex-wrap gap-4 text-xs text-gray-600 dark:text-gray-400">
|
||||
<div class="flex items-center gap-1.5">
|
||||
<svg class="w-3.5 h-3.5 text-purple-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 12h14M5 12a2 2 0 01-2-2V6a2 2 0 012-2h14a2 2 0 012 2v4a2 2 0 01-2 2M5 12a2 2 0 00-2 2v4a2 2 0 002 2h14a2 2 0 002-2v-4a2 2 0 00-2-2m-2-4h.01M17 16h.01" />
|
||||
</svg>
|
||||
<span class="font-medium">{run.nodes_checked || 0}</span> nodes
|
||||
</div>
|
||||
<Show when={run.guests_checked > 0}>
|
||||
<div class="flex items-center gap-1.5">
|
||||
<svg class="w-3.5 h-3.5 text-blue-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 3v2m6-2v2M9 19v2m6-2v2M5 9H3m2 6H3m18-6h-2m2 6h-2M7 19h10a2 2 0 002-2V7a2 2 0 00-2-2H7a2 2 0 00-2 2v10a2 2 0 002 2zM9 9h6v6H9V9z" />
|
||||
</svg>
|
||||
<span class="font-medium">{run.guests_checked}</span> VMs/CTs
|
||||
</div>
|
||||
</Show>
|
||||
<Show when={run.docker_checked > 0}>
|
||||
<div class="flex items-center gap-1.5">
|
||||
<svg class="w-3.5 h-3.5 text-cyan-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M20 7l-8-4-8 4m16 0l-8 4m8-4v10l-8 4m0-10L4 7m8 4v10M4 7v10l8 4" />
|
||||
</svg>
|
||||
<span class="font-medium">{run.docker_checked}</span> Docker
|
||||
</div>
|
||||
</Show>
|
||||
<Show when={run.storage_checked > 0}>
|
||||
<div class="flex items-center gap-1.5">
|
||||
<svg class="w-3.5 h-3.5 text-amber-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 7v10c0 2.21 3.582 4 8 4s8-1.79 8-4V7M4 7c0 2.21 3.582 4 8 4s8-1.79 8-4M4 7c0-2.21 3.582-4 8-4s8 1.79 8 4" />
|
||||
</svg>
|
||||
<span class="font-medium">{run.storage_checked}</span> storage
|
||||
</div>
|
||||
</Show>
|
||||
<Show when={run.hosts_checked > 0}>
|
||||
<div class="flex items-center gap-1.5">
|
||||
<svg class="w-3.5 h-3.5 text-green-500" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9.75 17L9 20l-1 1h8l-1-1-.75-3M3 13h18M5 17h14a2 2 0 002-2V5a2 2 0 00-2-2H5a2 2 0 00-2 2v10a2 2 0 002 2z" />
|
||||
</svg>
|
||||
<span class="font-medium">{run.hosts_checked}</span> hosts
|
||||
</div>
|
||||
</Show>
|
||||
{/* Token usage for transparency */}
|
||||
<Show when={run.output_tokens}>
|
||||
<div class="flex items-center gap-1.5 ml-auto text-gray-400 dark:text-gray-500">
|
||||
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z" />
|
||||
</svg>
|
||||
<span>{((run.input_tokens || 0) / 1000).toFixed(0)}k in / {((run.output_tokens || 0) / 1000).toFixed(1)}k out tokens</span>
|
||||
</div>
|
||||
</Show>
|
||||
</div>
|
||||
|
||||
{/* Critical & Warning Findings - Always show prominently */}
|
||||
<Show when={criticalFindings.length > 0 || warningFindings.length > 0}>
|
||||
<div class="space-y-2">
|
||||
<span class="text-[10px] text-gray-500 dark:text-gray-400 uppercase tracking-wider font-medium">
|
||||
Findings Requiring Attention
|
||||
</span>
|
||||
<div class="space-y-1.5">
|
||||
<For each={[...criticalFindings, ...warningFindings]}>
|
||||
{(finding) => (
|
||||
<div class="flex items-start gap-2 text-xs p-2 rounded-lg bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700">
|
||||
<span class={`flex-shrink-0 px-1.5 py-0.5 rounded text-[10px] font-medium ${finding.severity === 'critical'
|
||||
? 'bg-red-100 dark:bg-red-900/50 text-red-700 dark:text-red-300'
|
||||
: 'bg-yellow-100 dark:bg-yellow-900/50 text-yellow-700 dark:text-yellow-300'
|
||||
}`}>
|
||||
{finding.severity}
|
||||
</span>
|
||||
<div class="flex-1 min-w-0">
|
||||
<div class="font-medium text-gray-800 dark:text-gray-200">{finding.title}</div>
|
||||
<div class="text-gray-500 dark:text-gray-400 mt-0.5">
|
||||
{finding.resource_name}
|
||||
<Show when={finding.description}>
|
||||
<span class="mx-1">·</span>
|
||||
<span class="text-gray-400 dark:text-gray-500">{finding.description.substring(0, 100)}{finding.description.length > 100 ? '...' : ''}</span>
|
||||
</Show>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</For>
|
||||
</div>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* Resolved findings - show value delivered */}
|
||||
<Show when={resolvedFindings.length > 0}>
|
||||
<div class={activeFindings.length > 0 ? "mt-3 pt-2 border-t border-gray-200 dark:border-gray-700" : ""}>
|
||||
<span class="text-[10px] text-green-600 dark:text-green-400 uppercase tracking-wider flex items-center gap-1">
|
||||
{/* Info/Watch Findings - Show with lower emphasis */}
|
||||
<Show when={infoFindings.length > 0}>
|
||||
<div class="space-y-2">
|
||||
<span class="text-[10px] text-blue-600 dark:text-blue-400 uppercase tracking-wider font-medium flex items-center gap-1">
|
||||
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 13l4 4L19 7" />
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
|
||||
</svg>
|
||||
Resolved ({resolvedFindings.length})
|
||||
Observations ({infoFindings.length})
|
||||
</span>
|
||||
<div class="flex flex-col gap-1 mt-1">
|
||||
<For each={resolvedFindings}>
|
||||
<div class="space-y-1">
|
||||
<For each={infoFindings}>
|
||||
{(finding) => (
|
||||
<div class="flex items-center gap-2 text-xs opacity-70">
|
||||
<span class="px-1.5 py-0.5 rounded text-[10px] bg-green-100 dark:bg-green-900/50 text-green-700 dark:text-green-300">
|
||||
✓ resolved
|
||||
<div class="flex items-center gap-2 text-xs text-gray-600 dark:text-gray-400 py-1">
|
||||
<span class="px-1.5 py-0.5 rounded text-[10px] bg-blue-100 dark:bg-blue-900/40 text-blue-600 dark:text-blue-400">
|
||||
{finding.severity}
|
||||
</span>
|
||||
<span class="text-gray-600 dark:text-gray-400">{finding.title}</span>
|
||||
<span>{finding.title}</span>
|
||||
<span class="text-gray-400 dark:text-gray-500">on {finding.resource_name}</span>
|
||||
</div>
|
||||
)}
|
||||
@@ -3586,10 +3652,87 @@ function OverviewTab(props: {
|
||||
</div>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* Resolved findings */}
|
||||
<Show when={resolvedFindings.length > 0}>
|
||||
<div class="space-y-2">
|
||||
<span class="text-[10px] text-green-600 dark:text-green-400 uppercase tracking-wider font-medium flex items-center gap-1">
|
||||
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 13l4 4L19 7" />
|
||||
</svg>
|
||||
Resolved Since Last Patrol ({resolvedFindings.length})
|
||||
</span>
|
||||
<div class="space-y-1">
|
||||
<For each={resolvedFindings}>
|
||||
{(finding) => (
|
||||
<div class="flex items-center gap-2 text-xs text-gray-500 dark:text-gray-500 py-1">
|
||||
<span class="px-1.5 py-0.5 rounded text-[10px] bg-green-100 dark:bg-green-900/40 text-green-600 dark:text-green-400">
|
||||
✓ fixed
|
||||
</span>
|
||||
<span class="line-through opacity-70">{finding.title}</span>
|
||||
<span class="opacity-50">on {finding.resource_name}</span>
|
||||
</div>
|
||||
)}
|
||||
</For>
|
||||
</div>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* No findings case - show AI summary */}
|
||||
<Show when={!hasAnyFindings}>
|
||||
<div class="flex items-center gap-2 text-sm text-green-600 dark:text-green-400">
|
||||
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 12l2 2 4-4m6 2a9 9 0 11-18 0 9 9 0 0118 0z" />
|
||||
</svg>
|
||||
<span class="font-medium">All clear!</span>
|
||||
<span class="text-gray-500 dark:text-gray-400">No issues detected in this patrol run.</span>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* AI Analysis Summary - Collapsible for long content */}
|
||||
<Show when={run.ai_analysis}>
|
||||
{(() => {
|
||||
const [showFullAnalysis, setShowFullAnalysis] = createSignal(false);
|
||||
const analysis = run.ai_analysis || '';
|
||||
// Extract summary (first paragraph or first 300 chars)
|
||||
const summaryMatch = analysis.match(/^([^]*?)(?:\n\n|\[FINDING\]|$)/);
|
||||
const summary = summaryMatch ? summaryMatch[1].trim() : analysis.substring(0, 300);
|
||||
const hasMore = analysis.length > summary.length + 50;
|
||||
|
||||
return (
|
||||
<div class="border-t border-gray-200 dark:border-gray-700 pt-3 mt-3">
|
||||
<div class="flex items-center justify-between mb-2">
|
||||
<span class="text-[10px] text-purple-600 dark:text-purple-400 uppercase tracking-wider font-medium flex items-center gap-1">
|
||||
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9.663 17h4.673M12 3v1m6.364 1.636l-.707.707M21 12h-1M4 12H3m3.343-5.657l-.707-.707m2.828 9.9a5 5 0 117.072 0l-.548.547A3.374 3.374 0 0014 18.469V19a2 2 0 11-4 0v-.531c0-.895-.356-1.754-.988-2.386l-.548-.547z" />
|
||||
</svg>
|
||||
AI Analysis
|
||||
</span>
|
||||
<Show when={hasMore}>
|
||||
<button
|
||||
class="text-[10px] text-purple-600 dark:text-purple-400 hover:underline"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setShowFullAnalysis(!showFullAnalysis());
|
||||
}}
|
||||
>
|
||||
{showFullAnalysis() ? 'Show less' : 'Show full analysis'}
|
||||
</button>
|
||||
</Show>
|
||||
</div>
|
||||
<div class={`text-xs text-gray-600 dark:text-gray-400 whitespace-pre-wrap ${showFullAnalysis() ? 'max-h-96 overflow-y-auto' : ''}`}>
|
||||
{showFullAnalysis() ? analysis.replace(/\[FINDING\][\s\S]*?\[\/FINDING\]/g, '').trim() : summary}
|
||||
<Show when={hasMore && !showFullAnalysis()}>
|
||||
<span class="text-gray-400">...</span>
|
||||
</Show>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
</Show>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</Show>
|
||||
|
||||
@@ -210,15 +210,29 @@ func (s *Store) IsAnomaly(resourceID, metric string, value float64) (bool, float
|
||||
return false, 0 // Not enough data to determine
|
||||
}
|
||||
|
||||
// Calculate absolute difference
|
||||
absDiff := math.Abs(value - baseline.Mean)
|
||||
|
||||
// Don't flag small absolute changes as anomalies
|
||||
if absDiff < 3.0 {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
if baseline.StdDev == 0 {
|
||||
// No variance - any different value is anomalous
|
||||
if value != baseline.Mean {
|
||||
return true, math.Inf(1)
|
||||
// No variance - only flag if change is significant (> 5 percentage points)
|
||||
if absDiff > 5.0 {
|
||||
return true, 0 // No valid z-score when stddev is 0
|
||||
}
|
||||
return false, 0
|
||||
}
|
||||
|
||||
zScore := (value - baseline.Mean) / baseline.StdDev
|
||||
// Apply minimum stddev floor
|
||||
effectiveStdDev := baseline.StdDev
|
||||
if effectiveStdDev < 1.0 {
|
||||
effectiveStdDev = 1.0
|
||||
}
|
||||
|
||||
zScore := (value - baseline.Mean) / effectiveStdDev
|
||||
|
||||
// Consider anything > 2 standard deviations as anomalous
|
||||
// (covers ~95% of normal distribution)
|
||||
@@ -245,16 +259,39 @@ func (s *Store) CheckAnomaly(resourceID, metric string, value float64) (AnomalyS
|
||||
return AnomalyNone, 0, nil
|
||||
}
|
||||
|
||||
// Calculate absolute difference for threshold checks
|
||||
absDiff := math.Abs(value - baseline.Mean)
|
||||
|
||||
// Handle zero stddev case more intelligently
|
||||
// When values have been completely stable, small variations aren't anomalies
|
||||
if baseline.StdDev == 0 {
|
||||
if value != baseline.Mean {
|
||||
return AnomalyCritical, math.Inf(1), baseline
|
||||
// Only flag as anomaly if the absolute difference is significant
|
||||
// For percentage metrics (cpu, memory, disk), require > 5 percentage point change
|
||||
// This prevents false positives from small floating point variations
|
||||
if absDiff < 5.0 {
|
||||
return AnomalyNone, 0, baseline
|
||||
}
|
||||
return AnomalyNone, 0, baseline
|
||||
// Even with large difference, just flag as warning, not critical
|
||||
// since we don't have historical variance data to judge severity
|
||||
return AnomalyMedium, 0, baseline
|
||||
}
|
||||
|
||||
zScore := (value - baseline.Mean) / baseline.StdDev
|
||||
// Apply minimum stddev floor to prevent tiny variations from appearing extreme
|
||||
// If historical stddev is < 1%, use 1% as the floor for z-score calculation
|
||||
effectiveStdDev := baseline.StdDev
|
||||
if effectiveStdDev < 1.0 {
|
||||
effectiveStdDev = 1.0
|
||||
}
|
||||
|
||||
zScore := (value - baseline.Mean) / effectiveStdDev
|
||||
absZ := math.Abs(zScore)
|
||||
|
||||
// Also require a minimum absolute difference for practical significance
|
||||
// Don't flag anomalies for changes < 3 percentage points regardless of z-score
|
||||
if absDiff < 3.0 {
|
||||
return AnomalyNone, zScore, baseline
|
||||
}
|
||||
|
||||
var severity AnomalySeverity
|
||||
switch {
|
||||
case absZ < 2.0:
|
||||
|
||||
@@ -105,35 +105,76 @@ func TestIsAnomaly(t *testing.T) {
|
||||
func TestCheckAnomaly_Severity(t *testing.T) {
|
||||
store := NewStore(StoreConfig{MinSamples: 10})
|
||||
|
||||
// Create very stable data with known statistics
|
||||
// Mean = 50, StdDev = 1
|
||||
// Create data with larger stddev to allow meaningful tests
|
||||
// Mean = 50, and we'll create values with wider variance
|
||||
points := make([]MetricPoint, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
// Alternate between 49, 50, 51 for stddev ~1
|
||||
points[i] = MetricPoint{Value: 50 + float64(i%3) - 1}
|
||||
// Values from 45 to 55 to give stddev ~3
|
||||
points[i] = MetricPoint{Value: 50 + float64(i%11) - 5}
|
||||
}
|
||||
|
||||
store.Learn("test-vm", "vm", "cpu", points)
|
||||
baseline, _ := store.GetBaseline("test-vm", "cpu")
|
||||
|
||||
// The stddev should be around 3.0
|
||||
if baseline.StdDev < 2 || baseline.StdDev > 4 {
|
||||
t.Logf("Stddev is %f (expected ~3)", baseline.StdDev)
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
value float64
|
||||
expectedSeverity AnomalySeverity
|
||||
description string
|
||||
}{
|
||||
{50, AnomalyNone}, // Mean
|
||||
{50 + baseline.StdDev*1.5, AnomalyNone}, // 1.5 std devs - normal
|
||||
{50 + baseline.StdDev*2.2, AnomalyLow}, // 2.2 std devs
|
||||
{50 + baseline.StdDev*2.7, AnomalyMedium}, // 2.7 std devs
|
||||
{50 + baseline.StdDev*3.5, AnomalyHigh}, // 3.5 std devs
|
||||
{50 + baseline.StdDev*4.5, AnomalyCritical}, // 4.5 std devs
|
||||
// Values at or near mean - no anomaly
|
||||
{50, AnomalyNone, "Mean value"},
|
||||
{52, AnomalyNone, "Within 1 std dev and <3 point diff"},
|
||||
|
||||
// Small statistical deviations but below minimum absolute threshold (3 points)
|
||||
// should be filtered out
|
||||
{52.5, AnomalyNone, "Small absolute difference"},
|
||||
|
||||
// Larger deviations that meet both thresholds
|
||||
// With stddev ~3.2: z-score = diff/stddev
|
||||
// 44: 6pts/3.2 = 1.87z - below 2.0 threshold = None
|
||||
{44, AnomalyNone, "~2 std devs - below anomaly threshold"},
|
||||
// 42: 8pts/3.2 = 2.5z - low/medium range
|
||||
{42, AnomalyLow, "2.5 std devs with 8 point diff"},
|
||||
// 40: 10pts/3.2 = 3.1z - high range (3.0-4.0)
|
||||
{40, AnomalyHigh, "3.1 std devs with 10 point diff"},
|
||||
// 38: 12pts/3.2 = 3.75z - high range
|
||||
{38, AnomalyHigh, "3.75 std devs with 12 point diff"},
|
||||
// 35: 15pts/3.2 = 4.7z - critical range (>4.0)
|
||||
{35, AnomalyCritical, ">4 std devs with 15 point diff"},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
severity, _, _ := store.CheckAnomaly("test-vm", "cpu", tc.value)
|
||||
severity, zScore, _ := store.CheckAnomaly("test-vm", "cpu", tc.value)
|
||||
if severity != tc.expectedSeverity {
|
||||
t.Errorf("Value %f: expected severity %s, got %s", tc.value, tc.expectedSeverity, severity)
|
||||
t.Errorf("%s: Value %f (z=%.2f): expected severity %s, got %s",
|
||||
tc.description, tc.value, zScore, tc.expectedSeverity, severity)
|
||||
}
|
||||
}
|
||||
|
||||
// Test that small differences are filtered even with high z-scores
|
||||
// Create very stable data (stddev near 0)
|
||||
stablePoints := make([]MetricPoint, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
stablePoints[i] = MetricPoint{Value: 50}
|
||||
}
|
||||
store.Learn("stable-vm", "vm", "disk", stablePoints)
|
||||
|
||||
// Small change from perfectly stable baseline should NOT be anomaly
|
||||
severity, _, _ := store.CheckAnomaly("stable-vm", "disk", 51)
|
||||
if severity != AnomalyNone {
|
||||
t.Error("1% change from stable baseline should not be anomaly")
|
||||
}
|
||||
|
||||
// Larger change from stable baseline SHOULD be anomaly (medium severity)
|
||||
severity, _, _ = store.CheckAnomaly("stable-vm", "disk", 56)
|
||||
if severity == AnomalyNone {
|
||||
t.Error("6% change from stable baseline should be anomaly")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetResourceBaseline(t *testing.T) {
|
||||
|
||||
@@ -33,11 +33,13 @@ func TestBaselineStoreAdapter(t *testing.T) {
|
||||
t.Fatalf("unexpected baseline: mean=%v stddev=%v samples=%d", mean, stddev, samples)
|
||||
}
|
||||
|
||||
severity, z, gotMean, gotStd, ok := adapter.CheckAnomaly("node:pve1", "cpu", 11)
|
||||
severity, z, gotMean, gotStd, ok := adapter.CheckAnomaly("node:pve1", "cpu", 16)
|
||||
if !ok {
|
||||
t.Fatalf("expected anomaly check ok")
|
||||
}
|
||||
if severity != "critical" || z <= 0 || gotMean != 10 || gotStd != 0 {
|
||||
// With new practical thresholds: 6 point difference from stable baseline (stddev=0)
|
||||
// should be flagged as medium severity (not critical, since we don't have variance data)
|
||||
if severity != "medium" || gotMean != 10 || gotStd != 0 {
|
||||
t.Fatalf("unexpected anomaly: severity=%q z=%v mean=%v stddev=%v", severity, z, gotMean, gotStd)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCleanThinkingTokens_DeepSeekMarkers(t *testing.T) {
|
||||
input := `## Analysis Summary
|
||||
|
||||
<｜end▁of▁thinking｜>
|
||||
|
||||
Now, also consider the duplicate PBS services. Let's add an info finding.<｜end▁of▁thinking｜>
|
||||
|
||||
|
||||
|
||||
Now, also check for any storage growth concerns: frigate-storage rising slow.
|
||||
|
||||
After comprehensive analysis of your infrastructure, I identified several issues.
|
||||
|
||||
### Key Findings:
|
||||
|
||||
1. **Critical CPU overload on Tower host**`
|
||||
|
||||
result := cleanThinkingTokens(input)
|
||||
|
||||
// Should NOT contain thinking markers
|
||||
if contains(result, "<｜end▁of▁thinking｜>") {
|
||||
t.Errorf("cleanThinkingTokens() should have removed DeepSeek thinking markers")
|
||||
}
|
||||
|
||||
// Should NOT contain internal reasoning
|
||||
if contains(result, "Now, also consider") || contains(result, "Let's add an info") {
|
||||
t.Errorf("cleanThinkingTokens() should have removed internal reasoning")
|
||||
}
|
||||
|
||||
// Should still contain the actual content
|
||||
if !contains(result, "## Analysis Summary") {
|
||||
t.Errorf("cleanThinkingTokens() removed header")
|
||||
}
|
||||
if !contains(result, "### Key Findings") {
|
||||
t.Errorf("cleanThinkingTokens() removed findings section")
|
||||
}
|
||||
if !contains(result, "Critical CPU overload") {
|
||||
t.Errorf("cleanThinkingTokens() removed actual finding")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanThinkingTokens_ASCIIVariant(t *testing.T) {
|
||||
input := `Some content<|end_of_thinking|>
|
||||
|
||||
Now, let's check something.
|
||||
|
||||
## Real Content`
|
||||
|
||||
result := cleanThinkingTokens(input)
|
||||
|
||||
if result != "## Real Content" {
|
||||
t.Errorf("cleanThinkingTokens() failed for ASCII variant: got %q", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanThinkingTokens_ReasoningPatterns(t *testing.T) {
|
||||
input := `## Analysis
|
||||
|
||||
Let's check the CPU usage.
|
||||
|
||||
Now, I need to look at memory.
|
||||
|
||||
### Findings
|
||||
|
||||
- Issue 1`
|
||||
|
||||
result := cleanThinkingTokens(input)
|
||||
|
||||
// Should remove the reasoning lines but keep the findings
|
||||
if !contains(result, "## Analysis") || !contains(result, "### Findings") || !contains(result, "- Issue 1") {
|
||||
t.Errorf("cleanThinkingTokens() removed too much: got %q", result)
|
||||
}
|
||||
|
||||
if contains(result, "Let's check") || contains(result, "Now, I need") {
|
||||
t.Errorf("cleanThinkingTokens() should have removed reasoning: got %q", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanThinkingTokens_EmptyString(t *testing.T) {
|
||||
result := cleanThinkingTokens("")
|
||||
if result != "" {
|
||||
t.Errorf("cleanThinkingTokens(\"\") should return empty string, got %q", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanThinkingTokens_NoMarkers(t *testing.T) {
|
||||
input := `## Clean Analysis
|
||||
|
||||
This is a normal response without any thinking tokens.
|
||||
|
||||
### Findings
|
||||
|
||||
1. Issue one
|
||||
2. Issue two`
|
||||
|
||||
result := cleanThinkingTokens(input)
|
||||
|
||||
// Should be mostly unchanged (just trimmed)
|
||||
if result != input {
|
||||
t.Errorf("cleanThinkingTokens() modified clean content:\nGot: %q\nExpected: %q", result, input)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, substr string) bool {
|
||||
return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsHelper(s, substr))
|
||||
}
|
||||
|
||||
func containsHelper(s, substr string) bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
+100
-3
@@ -2110,6 +2110,98 @@ type AIAnalysisResult struct {
|
||||
OutputTokens int
|
||||
}
|
||||
|
||||
// cleanThinkingTokens removes model-specific thinking markers from AI responses.
|
||||
// Different AI models use different markers for their internal reasoning:
|
||||
// - DeepSeek: <｜end▁of▁thinking｜> or similar unicode variants
|
||||
// - Other models may use <think></think> or similar markers
|
||||
func cleanThinkingTokens(content string) string {
|
||||
if content == "" {
|
||||
return content
|
||||
}
|
||||
|
||||
// Remove DeepSeek thinking markers and everything before them on the same line
|
||||
// These appear as: <｜end▁of▁thinking｜> or <|end_of_thinking|>
|
||||
thinkingMarkers := []string{
|
||||
"<｜end▁of▁thinking｜>", // DeepSeek Unicode variant
|
||||
"<|end_of_thinking|>", // ASCII variant
|
||||
"<|end▁of▁thinking|>", // Mixed variant
|
||||
"</think>", // Generic thinking block end
|
||||
}
|
||||
|
||||
for _, marker := range thinkingMarkers {
|
||||
for strings.Contains(content, marker) {
|
||||
idx := strings.Index(content, marker)
|
||||
if idx >= 0 {
|
||||
// Find start of the line containing the marker
|
||||
lineStart := strings.LastIndex(content[:idx], "\n")
|
||||
if lineStart == -1 {
|
||||
lineStart = 0
|
||||
}
|
||||
// Find end of the line containing the marker
|
||||
markerEnd := idx + len(marker)
|
||||
lineEnd := strings.Index(content[markerEnd:], "\n")
|
||||
if lineEnd == -1 {
|
||||
lineEnd = len(content)
|
||||
} else {
|
||||
lineEnd = markerEnd + lineEnd
|
||||
}
|
||||
// Remove the entire line containing the marker
|
||||
content = content[:lineStart] + content[lineEnd:]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also remove any lines that look like internal reasoning
|
||||
// These typically start with patterns like "Now, " or "Let's " after a blank line
|
||||
lines := strings.Split(content, "\n")
|
||||
var cleanedLines []string
|
||||
skipUntilContent := false
|
||||
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
|
||||
// Skip lines that look like internal reasoning
|
||||
if skipUntilContent {
|
||||
// Resume when we hit actual content (markdown headers, findings, etc.)
|
||||
if strings.HasPrefix(trimmed, "#") ||
|
||||
strings.HasPrefix(trimmed, "[FINDING]") ||
|
||||
strings.HasPrefix(trimmed, "**") ||
|
||||
strings.HasPrefix(trimmed, "-") ||
|
||||
strings.HasPrefix(trimmed, "1.") {
|
||||
skipUntilContent = false
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Detect reasoning patterns (typically after empty lines)
|
||||
if trimmed == "" && i+1 < len(lines) {
|
||||
nextTrimmed := strings.TrimSpace(lines[i+1])
|
||||
if strings.HasPrefix(nextTrimmed, "Now, ") ||
|
||||
strings.HasPrefix(nextTrimmed, "Let's ") ||
|
||||
strings.HasPrefix(nextTrimmed, "Let me ") ||
|
||||
strings.HasPrefix(nextTrimmed, "I should ") ||
|
||||
strings.HasPrefix(nextTrimmed, "I'll ") ||
|
||||
strings.HasPrefix(nextTrimmed, "I need to ") ||
|
||||
strings.HasPrefix(nextTrimmed, "Checking ") ||
|
||||
strings.HasPrefix(nextTrimmed, "Looking at ") {
|
||||
skipUntilContent = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
cleanedLines = append(cleanedLines, line)
|
||||
}
|
||||
|
||||
// Clean up excessive blank lines
|
||||
content = strings.Join(cleanedLines, "\n")
|
||||
for strings.Contains(content, "\n\n\n") {
|
||||
content = strings.ReplaceAll(content, "\n\n\n", "\n\n")
|
||||
}
|
||||
|
||||
return strings.TrimSpace(content)
|
||||
}
|
||||
|
||||
|
||||
// runAIAnalysis uses the LLM to analyze infrastructure and identify issues
|
||||
func (p *PatrolService) runAIAnalysis(ctx context.Context, state models.StateSnapshot) (*AIAnalysisResult, error) {
|
||||
@@ -2148,10 +2240,11 @@ func (p *PatrolService) runAIAnalysis(ctx context.Context, state models.StateSna
|
||||
p.appendStreamContent(content)
|
||||
}
|
||||
case "thinking":
|
||||
// Thinking chunks become separate blocks (like AI chat)
|
||||
// Thinking chunks are for live streaming only - don't persist them
|
||||
// They allow users to see the AI's reasoning in real-time, but the
|
||||
// final stored analysis should only contain the actual findings
|
||||
if thinking, ok := event.Data.(string); ok && thinking != "" {
|
||||
contentBuffer.WriteString(thinking)
|
||||
// Send as a "thinking" event type so frontend can style it differently
|
||||
// Broadcast for live viewing ONLY - don't add to contentBuffer
|
||||
p.broadcast(PatrolStreamEvent{
|
||||
Type: "thinking",
|
||||
Content: thinking,
|
||||
@@ -2171,6 +2264,10 @@ func (p *PatrolService) runAIAnalysis(ctx context.Context, state models.StateSna
|
||||
if finalContent == "" {
|
||||
finalContent = contentBuffer.String()
|
||||
}
|
||||
|
||||
// Clean any thinking tokens that might have leaked through from the provider
|
||||
finalContent = cleanThinkingTokens(finalContent)
|
||||
|
||||
inputTokens = resp.InputTokens
|
||||
outputTokens = resp.OutputTokens
|
||||
|
||||
|
||||
Reference in New Issue
Block a user