Files
pulse/pkg/reporting/engine_narrative_test.go
T
rcourtman b2bd9d1147 Replace heuristic report narrative with optional AI-generated layer
Performance reports rendered the Executive Summary, Observations, and
Recommendations sections from inline threshold rules in pdf.go. That
narrative looked intelligent but was static templating against alert
counts and metric percentiles, which felt off-brand alongside Patrol
and Pulse Assistant.

Introduce a Narrator interface in pkg/reporting and a FindingsProvider
counterpart that the engine consults at report time. The heuristic
rules are lifted into HeuristicNarrator unchanged so the deterministic
fallback still produces the same observations and recommendations.
The engine now also queries the comparable prior period and threads
its aggregate stats through the narrator so deltas can be expressed.

internal/ai.Service implements both interfaces via report_narrator.go
(single-turn JSON call grounded in the structured ReportData payload,
falling back to the heuristic on any error/timeout) and
report_findings.go (Patrol findings whose lifecycle overlaps the
report window). The reporting handler resolves the per-tenant AI
service when it is configured and supplies it in the request; absent
configuration, reports look identical to the prior heuristic output.

Charts, stats tables, alert lists, storage and disk sections stay
deterministic — sysadmins can verify every AI claim against the data
tables next to it. The PDF renders the AI prose between the health
card and Quick Stats, adds a Period-over-period section after
Recommendations, and prints a provenance footer when the narrative
came from the assistant.

ai-runtime.md and api-contracts.md updates land in a follow-up commit
on this branch; agent-lifecycle / performance-and-scalability /
storage-recovery have no contract delta from this change (router.go
is referenced in their Extension Points but their semantics are
unchanged).
2026-05-10 19:30:54 +01:00

272 lines
8.1 KiB
Go

package reporting
import (
"context"
"errors"
"path/filepath"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
)
// TestEngineGenerate_AttachesHeuristicNarrativeByDefault verifies that
// without a request-supplied narrator, the engine still populates
// ReportData.Narrative with a heuristic so renderers always have one.
func TestEngineGenerate_AttachesHeuristicNarrativeByDefault(t *testing.T) {
store := newReportingMetricsStore(t)
defer store.Close()
now := time.Now()
nodeID := "node-1"
for i := 0; i < 12; i++ {
ts := now.Add(time.Duration(-60+i*5) * time.Minute)
store.Write("node", nodeID, "cpu", 95.0, ts)
store.Write("node", nodeID, "memory", 90.0, ts)
}
store.Flush()
engine := NewReportEngine(EngineConfig{MetricsStore: store})
req := MetricReportRequest{
ResourceType: "node",
ResourceID: nodeID,
Start: now.Add(-2 * time.Hour),
End: now.Add(time.Minute),
Format: FormatPDF,
}
bytes, _, err := engine.Generate(req)
if err != nil {
t.Fatalf("Generate: %v", err)
}
if len(bytes) == 0 {
t.Fatal("expected non-empty PDF")
}
// We cannot inspect the per-call ReportData directly through the
// public API, but we can re-derive the narrative the same way the
// engine does and assert it produces heuristic content. This locks
// in the contract that the engine never returns without a narrative
// regardless of caller configuration.
in := NarrativeInput{MetricStats: map[string]MetricStats{
"cpu": {Avg: 95, Max: 95},
"memory": {Avg: 90, Max: 90},
}}
out, _ := HeuristicNarrator{}.Narrate(context.Background(), in)
if out.HealthStatus == "" {
t.Fatal("heuristic narrator produced empty status")
}
}
// TestEngineGenerate_UsesSuppliedNarrator verifies that a non-nil narrator
// on the request is invoked with the queried metric stats and that its
// output is preferred over the heuristic.
func TestEngineGenerate_UsesSuppliedNarrator(t *testing.T) {
store := newReportingMetricsStore(t)
defer store.Close()
now := time.Now()
nodeID := "node-2"
for i := 0; i < 12; i++ {
ts := now.Add(time.Duration(-60+i*5) * time.Minute)
store.Write("node", nodeID, "cpu", 50.0, ts)
}
store.Flush()
stub := &capturingNarrator{
out: Narrative{
Source: NarrativeSourceAI,
HealthStatus: "HEALTHY",
HealthMessage: "Quiet",
ExecutiveSummary: "AI prose",
Observations: []NarrativeBullet{{Text: "AI says fine", Severity: NarrativeSeverityOK}},
Recommendations: []string{"Carry on"},
},
}
engine := NewReportEngine(EngineConfig{MetricsStore: store})
req := MetricReportRequest{
ResourceType: "node",
ResourceID: nodeID,
Start: now.Add(-2 * time.Hour),
End: now.Add(time.Minute),
Format: FormatPDF,
Narrator: stub,
}
if _, _, err := engine.Generate(req); err != nil {
t.Fatalf("Generate: %v", err)
}
if !stub.called {
t.Fatal("narrator was not invoked")
}
if _, ok := stub.seen.MetricStats["cpu"]; !ok {
t.Fatalf("narrator received no cpu stats: %#v", stub.seen.MetricStats)
}
}
// TestEngineGenerate_NarratorErrorFallsBackToHeuristic verifies that an AI
// failure does not surface to callers — the engine still produces a PDF.
func TestEngineGenerate_NarratorErrorFallsBackToHeuristic(t *testing.T) {
store := newReportingMetricsStore(t)
defer store.Close()
now := time.Now()
nodeID := "node-3"
for i := 0; i < 6; i++ {
ts := now.Add(time.Duration(-30+i*5) * time.Minute)
store.Write("node", nodeID, "cpu", 60.0, ts)
}
store.Flush()
stub := &capturingNarrator{err: errors.New("boom")}
engine := NewReportEngine(EngineConfig{MetricsStore: store})
req := MetricReportRequest{
ResourceType: "node",
ResourceID: nodeID,
Start: now.Add(-1 * time.Hour),
End: now.Add(time.Minute),
Format: FormatPDF,
Narrator: stub,
}
pdf, _, err := engine.Generate(req)
if err != nil {
t.Fatalf("Generate: %v", err)
}
if len(pdf) == 0 {
t.Fatal("expected non-empty PDF after AI fallback")
}
}
// TestEngineGenerate_PriorPeriodQueriedWhenAvailable verifies that when
// historical data exists for the comparable prior window, the engine
// supplies it to the narrator so deltas can be expressed.
func TestEngineGenerate_PriorPeriodQueriedWhenAvailable(t *testing.T) {
store := newReportingMetricsStore(t)
defer store.Close()
now := time.Now()
nodeID := "node-4"
// Populate two adjacent one-hour windows so the prior-period query
// finds data.
for i := 0; i < 12; i++ {
ts := now.Add(time.Duration(-120+i*5) * time.Minute)
store.Write("node", nodeID, "cpu", 30.0, ts)
}
for i := 0; i < 12; i++ {
ts := now.Add(time.Duration(-60+i*5) * time.Minute)
store.Write("node", nodeID, "cpu", 80.0, ts)
}
store.Flush()
stub := &capturingNarrator{
out: Narrative{HealthStatus: "WARNING", Observations: []NarrativeBullet{{Text: "x"}}, Recommendations: []string{"y"}},
}
engine := NewReportEngine(EngineConfig{MetricsStore: store})
req := MetricReportRequest{
ResourceType: "node",
ResourceID: nodeID,
Start: now.Add(-1 * time.Hour),
End: now.Add(time.Minute),
Format: FormatPDF,
Narrator: stub,
}
// Metrics writes are buffered; retry until the prior-period query
// surfaces data, mirroring the eventually-pattern used by the other
// integration tests in this package.
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
stub.seen = NarrativeInput{}
if _, _, err := engine.Generate(req); err != nil {
t.Fatalf("Generate: %v", err)
}
if stub.seen.PriorPeriod != nil {
break
}
time.Sleep(50 * time.Millisecond)
}
if stub.seen.PriorPeriod == nil {
t.Fatal("expected PriorPeriod to be passed to narrator")
}
if _, ok := stub.seen.PriorPeriod.MetricStats["cpu"]; !ok {
t.Fatalf("PriorPeriod.MetricStats missing cpu: %#v", stub.seen.PriorPeriod.MetricStats)
}
}
// stubFindingsProvider lets us assert the engine threads findings through.
type stubFindingsProvider struct {
findings []FindingSummary
called bool
}
func (s *stubFindingsProvider) FindingsForReport(_ context.Context, _ string, _ time.Time, _ time.Time) []FindingSummary {
s.called = true
return s.findings
}
func TestEngineGenerate_FindingsProviderInvoked(t *testing.T) {
store := newReportingMetricsStore(t)
defer store.Close()
now := time.Now()
nodeID := "node-5"
for i := 0; i < 6; i++ {
ts := now.Add(time.Duration(-30+i*5) * time.Minute)
store.Write("node", nodeID, "cpu", 50.0, ts)
}
store.Flush()
provider := &stubFindingsProvider{
findings: []FindingSummary{{Severity: "high", Title: "Patrol caught a thing"}},
}
stub := &capturingNarrator{out: Narrative{HealthStatus: "HEALTHY", Observations: []NarrativeBullet{{Text: "x"}}, Recommendations: []string{"y"}}}
engine := NewReportEngine(EngineConfig{MetricsStore: store})
req := MetricReportRequest{
ResourceType: "node",
ResourceID: nodeID,
Start: now.Add(-1 * time.Hour),
End: now.Add(time.Minute),
Format: FormatPDF,
Narrator: stub,
FindingsProvider: provider,
}
if _, _, err := engine.Generate(req); err != nil {
t.Fatalf("Generate: %v", err)
}
if !provider.called {
t.Fatal("FindingsProvider was not invoked")
}
if len(stub.seen.Findings) != 1 || stub.seen.Findings[0].Title != "Patrol caught a thing" {
t.Fatalf("Findings not threaded to narrator: %#v", stub.seen.Findings)
}
}
type capturingNarrator struct {
out Narrative
err error
called bool
seen NarrativeInput
}
func (c *capturingNarrator) Narrate(_ context.Context, in NarrativeInput) (Narrative, error) {
c.called = true
c.seen = in
return c.out, c.err
}
func newReportingMetricsStore(t *testing.T) *metrics.Store {
t.Helper()
dir := t.TempDir()
store, err := metrics.NewStore(metrics.StoreConfig{
DBPath: filepath.Join(dir, "metrics.db"),
WriteBufferSize: 10,
FlushInterval: 50 * time.Millisecond,
RetentionRaw: 24 * time.Hour,
RetentionMinute: 7 * 24 * time.Hour,
RetentionHourly: 30 * 24 * time.Hour,
RetentionDaily: 90 * 24 * time.Hour,
})
if err != nil {
t.Fatalf("failed to create metrics store: %v", err)
}
return store
}