Stop reports and resource history calling a moved alert recovered

A node alert that moves to its linked Pulse agent closes with a
moved_to_agent resolution (#2580), and #2591 taught Patrol, alert
history and the assistant to say so. Two readers still showed the
close as a recovery.

Performance reports dropped the resolution when copying recently
resolved alerts into reporting.AlertInfo. A node report generated in
the five-minute window after a handover read HEALTHY, "All systems
operating normally", counted the alert as triggered and resolved, and
told the AI narrator resolved: true, although nothing had seen the
condition clear. Report rows now carry the resolution and the alert's
resource. pkg/reporting treats an alert that closed without recovering
as still live unless the report also shows its successor's reading for
that metric (an open alert, or a recovery since the handover), so a
handover never reads as healthy and one condition is counted once;
fleet reports match handovers across all their resources. The PDF
card, Quick Stats, active list and heuristic narrative say where the
alert went and to check the agent, the alerts table shows "Moved" with
the engine's summary, fleets count it as active and name the agent,
and the narrator gets resolved: false, the summary and whether the
successor's alert is listed, with prompt rules for both.

Resource change readers labelled every alert_resolved change "Alert
resolved" in green and prefixed the headline with it, so a moved close
read "Alert resolved: Alert moved to pve1 (Host Agent). This is not a
recovery...". getResourceChangePresentation now reads the
alert_resolution metadata: such a change is "Alert moved" in the
neutral blue of the history badge, and its summary is the whole
headline in the drawer, ResourceChangeSummary and Patrol's assessment
handoff.

Reports requested as agent, which is how the Reports page addresses
every Proxmox node, still carry no alert rows at all, and node alert
history is stored under legacy node IDs that the drawer never queries;
both are separate fixes.
This commit is contained in:
rcourtman
2026-10-06 16:55:10 +01:00
parent d26a483731
commit 112ce030d7
25 changed files with 1159 additions and 205 deletions
+229
View File
@@ -0,0 +1,229 @@
package reporting
import (
"fmt"
"strings"
)
// AlertResolutionMovedToAgent is the reason code of an alert that closed
// because a linked Pulse agent took its metric over
// (alerts.AlertResolutionMovedToAgent).
const AlertResolutionMovedToAgent = "moved_to_agent"
// Recovered reports whether the alert closed because its condition cleared.
// An open alert, or one that closed without recovering, is not recovered.
func (a AlertInfo) Recovered() bool {
return a.ResolvedTime != nil && a.Resolution == nil
}
// ClosedWithoutRecovery reports whether the alert closed while its condition
// may still hold, such as a node alert handed to its Pulse agent.
func (a AlertInfo) ClosedWithoutRecovery() bool {
return a.ResolvedTime != nil && a.Resolution != nil
}
// ResolutionOutcome is the short form of a close that was not a recovery,
// such as "moved to pve1 (Host Agent)". It is empty for an open alert and for
// an ordinary recovery.
func (a AlertInfo) ResolutionOutcome() string {
if !a.ClosedWithoutRecovery() {
return ""
}
if a.Resolution.Reason == AlertResolutionMovedToAgent {
return "moved to " + a.successorLabel()
}
return "closed without recovering"
}
// ResolutionSummary is the alert engine's account of a close that was not a
// recovery, falling back to the short form when the engine supplied none. It
// is empty for an open alert and for an ordinary recovery.
func (a AlertInfo) ResolutionSummary() string {
if !a.ClosedWithoutRecovery() {
return ""
}
if summary := strings.TrimSpace(a.Resolution.Summary); summary != "" {
return summary
}
return fmt.Sprintf("Alert %s. This is not a recovery.", a.ResolutionOutcome())
}
func (a AlertInfo) successorLabel() string {
if a.Resolution != nil {
if name := strings.TrimSpace(a.Resolution.SuccessorName); name != "" {
return name
}
}
return "its Pulse agent"
}
// unresolvedAlerts returns the alerts a report must treat as live: every open
// alert, and every alert that closed without recovering unless the report
// also shows its successor's own reading for that metric. That alert carries
// the condition's current state, so one condition is never counted twice and
// a handover never reads as a recovery.
func unresolvedAlerts(alerts []AlertInfo) []AlertInfo {
return unresolvedAlertsAmong(alerts, alerts)
}
// unresolvedAlertsAmong is unresolvedAlerts with successors looked up in
// pool, so a fleet report matches a handover against every resource's alerts.
func unresolvedAlertsAmong(alerts, pool []AlertInfo) []AlertInfo {
out := make([]AlertInfo, 0, len(alerts))
for _, alert := range alerts {
switch {
case alert.ResolvedTime == nil:
out = append(out, alert)
case alert.ClosedWithoutRecovery() && successorAlertFor(alert, pool) == nil:
out = append(out, alert)
}
}
return out
}
// successorAlertFor returns the alert in pool that carries a moved alert's
// condition: one of the same type on its successor or one of the successor's
// children (an agent's filesystem alerts sit under "<agent>/disk:<mount>"),
// still open or recovered at or after the handover. A successor recovery from
// before the handover says nothing about the reading since, so it does not
// count.
func successorAlertFor(moved AlertInfo, pool []AlertInfo) *AlertInfo {
if !moved.ClosedWithoutRecovery() {
return nil
}
successor := strings.TrimSpace(moved.Resolution.SuccessorResourceID)
if successor == "" {
return nil
}
for i := range pool {
candidate := &pool[i]
if candidate.Type != moved.Type || candidate.ClosedWithoutRecovery() {
continue
}
if candidate.ResolvedTime != nil && candidate.ResolvedTime.Before(*moved.ResolvedTime) {
continue
}
resourceID := strings.TrimSpace(candidate.ResourceID)
if resourceID == successor || strings.HasPrefix(resourceID, successor+"/") {
return candidate
}
}
return nil
}
// SuccessorAlertListed reports whether alerts shows the reading of the
// resource this alert moved to: an alert of the same type on the successor,
// open or recovered since the handover. The report then counts the condition
// once, through that alert. It is false for an open alert or a recovery.
func (a AlertInfo) SuccessorAlertListed(alerts []AlertInfo) bool {
return successorAlertFor(a, alerts) != nil
}
// fleetAlerts is every alert listed across a fleet report's resources.
func (d *MultiReportData) fleetAlerts() []AlertInfo {
if d == nil {
return nil
}
var pool []AlertInfo
for _, rd := range d.Resources {
if rd != nil {
pool = append(pool, rd.Alerts...)
}
}
return pool
}
// alertHealth is the health verdict a report's alerts support. The PDF health
// card and the heuristic narrator both read it, so they never disagree.
type alertHealth struct {
Status string // HEALTHY, WARNING or CRITICAL
Message string
Critical int
Warning int
// Unresolved is every alert the report treats as live (unresolvedAlerts).
Unresolved []AlertInfo
}
func assessAlertHealth(alerts []AlertInfo) alertHealth {
health := alertHealth{Unresolved: unresolvedAlerts(alerts)}
open := 0
var moved []AlertInfo
for _, alert := range health.Unresolved {
if alert.Level == "critical" {
health.Critical++
} else {
health.Warning++
}
if alert.ClosedWithoutRecovery() {
moved = append(moved, alert)
} else {
open++
}
}
switch {
case health.Critical > 0:
health.Status = "CRITICAL"
case health.Warning > 0:
health.Status = "WARNING"
default:
health.Status = "HEALTHY"
}
switch {
case open == 0 && len(moved) > 0:
health.Message = movedAlertsHealthMessage(moved)
case health.Critical == 1:
health.Message = "1 critical issue requires immediate attention"
case health.Critical > 1:
health.Message = fmt.Sprintf("%d critical issues require immediate attention", health.Critical)
case health.Warning == 1:
health.Message = "1 warning detected - review recommended"
case health.Warning > 1:
health.Message = fmt.Sprintf("%d warnings detected - review recommended", health.Warning)
default:
health.Message = "All systems operating normally"
}
return health
}
// movedAlertsHealthMessage says where alerts that closed without recovering
// went, so the report points at the resource that now holds the reading
// instead of calling this one healthy.
func movedAlertsHealthMessage(moved []AlertInfo) string {
first := moved[0]
for _, alert := range moved[1:] {
if alert.ResolutionOutcome() != first.ResolutionOutcome() {
return fmt.Sprintf("%d alerts closed without recovering - review recommended", len(moved))
}
}
if first.Resolution.Reason != AlertResolutionMovedToAgent {
if len(moved) == 1 {
return "1 alert closed without recovering - review recommended"
}
return fmt.Sprintf("%d alerts closed without recovering - review recommended", len(moved))
}
if len(moved) == 1 {
return fmt.Sprintf("Alert %s - check the agent for the current reading", first.ResolutionOutcome())
}
return fmt.Sprintf("%d alerts %s - check the agent for the current reading", len(moved), first.ResolutionOutcome())
}
// movedAlertSuccessors names, once each and in report order, the Pulse agents
// that unresolved alerts moved to.
func movedAlertSuccessors(unresolved []AlertInfo) []string {
var successors []string
seen := make(map[string]struct{})
for _, alert := range unresolved {
if !alert.ClosedWithoutRecovery() || alert.Resolution.Reason != AlertResolutionMovedToAgent {
continue
}
successor := alert.successorLabel()
if _, ok := seen[successor]; ok {
continue
}
seen[successor] = struct{}{}
successors = append(successors, successor)
}
return successors
}
+273
View File
@@ -0,0 +1,273 @@
package reporting
import (
"context"
"strings"
"testing"
"time"
"github.com/go-pdf/fpdf"
)
const movedAlertSummary = "Alert moved to pve1 (Host Agent). This is not a recovery: check the agent for the current reading."
// movedNodeAlert is a node memory alert that closed because the linked Pulse
// agent took the metric over, as GetRecentlyResolved reports it.
func movedNodeAlert(level string) AlertInfo {
resolved := time.Now().Add(-2 * time.Minute)
return AlertInfo{
Type: "memory",
Level: level,
Message: "Node memory at 95%",
StartTime: time.Now().Add(-time.Hour),
ResolvedTime: &resolved,
ResourceID: "pve1-node",
Resolution: &AlertResolution{
Reason: AlertResolutionMovedToAgent,
SuccessorResourceID: "agent:host-1",
SuccessorName: "pve1 (Host Agent)",
Summary: movedAlertSummary,
},
}
}
// A node alert that moved to its Pulse agent within the recently-resolved
// window must not turn the report HEALTHY: nothing saw the condition clear.
func TestHeuristicNarrator_MovedAlertIsNotARecovery(t *testing.T) {
out, err := HeuristicNarrator{}.Narrate(context.Background(), NarrativeInput{
MetricStats: map[string]MetricStats{"memory": {Avg: 60, Max: 96}},
Alerts: []AlertInfo{movedNodeAlert("warning")},
})
if err != nil {
t.Fatalf("Narrate: %v", err)
}
if out.HealthStatus != "WARNING" {
t.Fatalf("HealthStatus = %q, want WARNING", out.HealthStatus)
}
if want := "Alert moved to pve1 (Host Agent) - check the agent for the current reading"; out.HealthMessage != want {
t.Fatalf("HealthMessage = %q, want %q", out.HealthMessage, want)
}
var movedBullet *NarrativeBullet
for i, bullet := range out.Observations {
if strings.Contains(bullet.Text, "triggered and resolved") {
t.Fatalf("moved alert counted as triggered and resolved: %+v", out.Observations)
}
if bullet.Text == movedAlertSummary {
movedBullet = &out.Observations[i]
}
}
if movedBullet == nil || movedBullet.Severity != NarrativeSeverityInfo {
t.Fatalf("expected an observation recording the handover, got %+v", out.Observations)
}
recs := strings.Join(out.Recommendations, "\n")
if !strings.Contains(recs, "Check the current reading on pve1 (Host Agent)") {
t.Fatalf("expected a recommendation to check the agent, got %q", recs)
}
if strings.Contains(recs, "No immediate action required") {
t.Fatalf("a moved alert is not a reason to stand down, got %q", recs)
}
critical := assessAlertHealth([]AlertInfo{movedNodeAlert("critical")})
if critical.Status != "CRITICAL" || critical.Critical != 1 {
t.Fatalf("a moved critical alert keeps its level, got %+v", critical)
}
// An ordinary recovery is still a recovery.
recovered := movedNodeAlert("warning")
recovered.Resolution = nil
plain, _ := HeuristicNarrator{}.Narrate(context.Background(), NarrativeInput{
MetricStats: map[string]MetricStats{"memory": {Avg: 60, Max: 70}},
Alerts: []AlertInfo{recovered},
})
if plain.HealthStatus != "HEALTHY" || plain.HealthMessage != "All systems operating normally" {
t.Fatalf("recovered alert should leave the report healthy, got %q / %q", plain.HealthStatus, plain.HealthMessage)
}
}
// When the report also lists the alert the successor raised for the same
// metric, that alert carries the condition: the moved alert must not count
// it a second time.
func TestUnresolvedAlerts_SuccessorAlertCarriesTheCondition(t *testing.T) {
agentMemory := AlertInfo{Type: "memory", Level: "warning", Message: "Agent memory at 94%", ResourceID: "agent:host-1"}
health := assessAlertHealth([]AlertInfo{movedNodeAlert("warning"), agentMemory})
if health.Warning != 1 || len(health.Unresolved) != 1 || health.Unresolved[0] != agentMemory {
t.Fatalf("expected only the agent's own alert to count, got %+v", health)
}
if health.Message != "1 warning detected - review recommended" {
t.Fatalf("HealthMessage = %q", health.Message)
}
out, _ := HeuristicNarrator{}.Narrate(context.Background(), NarrativeInput{
MetricStats: map[string]MetricStats{"memory": {Avg: 60, Max: 96}},
Alerts: []AlertInfo{movedNodeAlert("warning"), agentMemory},
})
if strings.Contains(strings.Join(out.Recommendations, "\n"), "Check the current reading") {
t.Fatalf("covered handover should not ask to check the agent: %+v", out.Recommendations)
}
// The agent's filesystem alerts sit under the agent's resource ID.
movedDisk := movedNodeAlert("warning")
movedDisk.Type = "disk"
agentRoot := AlertInfo{Type: "disk", Level: "warning", ResourceID: "agent:host-1/disk:root"}
if got := unresolvedAlerts([]AlertInfo{movedDisk, agentRoot}); len(got) != 1 || got[0] != agentRoot {
t.Fatalf("expected the agent filesystem alert to cover the moved disk alert, got %+v", got)
}
// A different metric, or an agent whose ID merely shares a prefix, does
// not carry the moved alert's condition.
agentCPU := AlertInfo{Type: "cpu", Level: "warning", ResourceID: "agent:host-1"}
otherAgent := AlertInfo{Type: "memory", Level: "warning", ResourceID: "agent:host-10"}
if got := unresolvedAlerts([]AlertInfo{movedNodeAlert("warning"), agentCPU, otherAgent}); len(got) != 3 {
t.Fatalf("expected the moved alert to stay unresolved beside unrelated alerts, got %+v", got)
}
}
func TestExecutiveSummary_MovedAlertIsNotHealthy(t *testing.T) {
data := &ReportData{
Title: "pve1",
ResourceType: "node",
ResourceID: "pve1-node",
Start: time.Now().Add(-time.Hour),
End: time.Now(),
GeneratedAt: time.Now(),
Summary: MetricSummary{ByMetric: map[string]MetricStats{
"cpu": {Avg: 5, Max: 12, Count: 60},
"memory": {Avg: 60, Max: 96, Count: 60},
}},
TotalPoints: 120,
Alerts: []AlertInfo{movedNodeAlert("warning")},
}
// fpdf escapes parentheses inside PDF string literals.
text := strings.NewReplacer(`\(`, "(", `\)`, ")").Replace(renderExecutiveSummaryText(t, data))
for _, want := range []string{
"WARNING",
"Alert moved to pve1 (Host Agent) - check the agent for the current reading",
"Node memory at 95% (moved to pve1 (Host Agent))",
"Moved",
"Moved: " + movedAlertSummary,
} {
if !strings.Contains(text, want) {
t.Errorf("expected %q in executive summary, got:\n%s", want, text)
}
}
if strings.Contains(text, "HEALTHY") || strings.Contains(text, "All systems operating normally") {
t.Errorf("a handover must not render the healthy card, got:\n%s", text)
}
// The health card does not wrap, so the handover message must fit it.
pdf := fpdf.New("P", "mm", "A4", "")
pdf.SetFont("Arial", "", 11)
pageWidth, _ := pdf.GetPageSize()
if width := pdf.GetStringWidth(assessAlertHealth(data.Alerts).Message); width > pageWidth-40 {
t.Fatalf("handover health message is %.1fmm wide, card is %.1fmm", width, pageWidth-40)
}
}
// A successor's reading only counts from the handover on: an agent recovery
// from before the node alert moved says nothing about the condition since.
func TestUnresolvedAlerts_SuccessorReadingMustFollowTheHandover(t *testing.T) {
moved := movedNodeAlert("warning")
before := moved.ResolvedTime.Add(-time.Minute)
after := moved.ResolvedTime.Add(time.Minute)
agentRecoveredBefore := AlertInfo{Type: "memory", Level: "warning", ResourceID: "agent:host-1", ResolvedTime: &before}
agentRecoveredAfter := AlertInfo{Type: "memory", Level: "warning", ResourceID: "agent:host-1", ResolvedTime: &after}
stale := assessAlertHealth([]AlertInfo{agentRecoveredBefore, moved})
if stale.Status != "WARNING" || len(stale.Unresolved) != 1 || stale.Unresolved[0] != moved {
t.Fatalf("an earlier agent recovery must not cover a later handover, got %+v", stale)
}
if moved.SuccessorAlertListed([]AlertInfo{agentRecoveredBefore, moved}) {
t.Fatal("SuccessorAlertListed must ignore a recovery from before the handover")
}
cleared := assessAlertHealth([]AlertInfo{agentRecoveredAfter, moved})
if cleared.Status != "HEALTHY" || len(cleared.Unresolved) != 0 {
t.Fatalf("an agent recovery after the handover shows the condition cleared, got %+v", cleared)
}
if !moved.SuccessorAlertListed([]AlertInfo{agentRecoveredAfter, moved}) {
t.Fatal("SuccessorAlertListed must accept a recovery after the handover")
}
}
func TestBuildFleetNarrativeInput_MovedAlertStaysActive(t *testing.T) {
now := time.Now()
movedElsewhere := movedNodeAlert("critical")
movedElsewhere.Resolution = &AlertResolution{
Reason: AlertResolutionMovedToAgent,
SuccessorResourceID: "agent:host-3",
SuccessorName: "pve3 (Host Agent)",
Summary: "Alert moved to pve3 (Host Agent). This is not a recovery: check the agent for the current reading.",
}
multi := &MultiReportData{
Title: "Fleet",
Start: now.Add(-time.Hour),
End: now,
Resources: []*ReportData{
{
ResourceID: "pve1-node",
ResourceType: "node",
Resource: &ResourceInfo{Name: "pve1", Status: "online"},
Alerts: []AlertInfo{movedNodeAlert("warning")},
},
{
// The agent pve1's alert moved to is its own fleet entry.
ResourceID: "agent:host-1",
ResourceType: "agent",
Resource: &ResourceInfo{Name: "pve1 (Host Agent)", Status: "online"},
Alerts: []AlertInfo{{Type: "memory", Level: "warning", ResourceID: "agent:host-1"}},
},
{
ResourceID: "pve3-node",
ResourceType: "node",
Resource: &ResourceInfo{Name: "pve3", Status: "online"},
Alerts: []AlertInfo{movedElsewhere},
},
},
}
in := buildFleetNarrativeInput(multi)
if in.Aggregate.TotalResolvedAlerts != 0 {
t.Fatalf("moved alerts are not resolved, got %d", in.Aggregate.TotalResolvedAlerts)
}
// pve1's handover is carried by the agent's own alert in another fleet
// entry; pve3's has no successor reading, so it stays live at its level.
if in.Aggregate.TotalActiveAlerts != 2 || in.Aggregate.TotalCriticalAlerts != 1 {
t.Fatalf("expected the agent alert and pve3's moved alert active, got active=%d critical=%d",
in.Aggregate.TotalActiveAlerts, in.Aggregate.TotalCriticalAlerts)
}
if got := in.Resources[0]; got.ActiveAlerts != 0 || len(got.MovedTo) != 0 {
t.Fatalf("pve1's covered handover must not count again, got %+v", got)
}
if got := in.Resources[2]; got.ActiveAlerts != 1 || len(got.MovedTo) != 1 || got.MovedTo[0] != "pve3 (Host Agent)" {
t.Fatalf("pve3 should keep its moved alert and name the agent, got %+v", got)
}
out, _ := HeuristicFleetNarrator{}.NarrateFleet(context.Background(), in)
if out.HealthStatus != "CRITICAL" {
t.Fatalf("fleet HealthStatus = %q, want CRITICAL", out.HealthStatus)
}
recs := strings.Join(out.Recommendations, "\n")
if !strings.Contains(recs, "Check the current reading on pve3 (Host Agent)") {
t.Fatalf("expected a recommendation naming pve3's agent, got %q", recs)
}
if strings.Contains(recs, "pve1 (Host Agent)") || strings.Contains(recs, "No fleet-wide action required") {
t.Fatalf("unexpected recommendation, got %q", recs)
}
// A covered handover leaves only the agent's open warning: still no
// all-clear in the patterns or the recommendations.
multi.Resources = multi.Resources[:2]
covered, _ := HeuristicFleetNarrator{}.NarrateFleet(context.Background(), buildFleetNarrativeInput(multi))
if covered.HealthStatus != "WARNING" {
t.Fatalf("fleet HealthStatus = %q, want WARNING", covered.HealthStatus)
}
for _, pattern := range covered.Patterns {
if strings.Contains(pattern.Text, "nominal thresholds") {
t.Fatalf("an open alert rules out the all-clear pattern, got %+v", covered.Patterns)
}
}
if recs := strings.Join(covered.Recommendations, "\n"); strings.Contains(recs, "No fleet-wide action required") ||
!strings.Contains(recs, "Review the active alerts") {
t.Fatalf("an open alert needs a review recommendation, got %q", recs)
}
}
+27 -3
View File
@@ -38,6 +38,9 @@ type FleetResourceSummary struct {
UnhealthyDisks int
StoragePoolsHigh int
Findings int
// MovedTo names the Pulse agents this resource's unresolved alerts
// moved to without recovering; those alerts also count as active.
MovedTo []string
}
// FleetAggregate is the fleet-wide rollup. Means are means-of-means:
@@ -313,7 +316,9 @@ func fleetHeuristicPatterns(in FleetNarrativeInput) []NarrativeBullet {
Severity: NarrativeSeverityInfo,
})
}
if len(out) == 0 {
// Open alerts, including ones that moved to an agent without
// recovering, rule out the all-clear even when no metric pattern fired.
if len(out) == 0 && in.Aggregate.TotalActiveAlerts == 0 {
out = append(out, NarrativeBullet{
Text: fmt.Sprintf("Fleet of %d resources operating within nominal thresholds", in.Aggregate.ResourceCount),
Severity: NarrativeSeverityOK,
@@ -373,6 +378,19 @@ func fleetHeuristicRecommendations(in FleetNarrativeInput) []string {
if disksHigh > 0 {
recs = append(recs, "Replace failing or end-of-life disks before they cause outage")
}
seenSuccessors := make(map[string]struct{})
for _, r := range in.Resources {
for _, successor := range r.MovedTo {
if _, ok := seenSuccessors[successor]; ok {
continue
}
seenSuccessors[successor] = struct{}{}
recs = append(recs, fmt.Sprintf("Check the current reading on %s - an alert moved there without recovering", successor))
}
}
if len(recs) == 0 && a.TotalActiveAlerts > 0 {
recs = append(recs, "Review the active alerts on the resources listed above")
}
if len(recs) == 0 {
recs = append(recs, "No fleet-wide action required - continue routine monitoring")
}
@@ -402,6 +420,7 @@ func buildFleetNarrativeInput(data *MultiReportData) FleetNarrativeInput {
var sumCPU, sumMem, sumDisk float64
var countCPU, countMem, countDisk int
fleetAlerts := data.fleetAlerts()
for _, rd := range data.Resources {
if rd == nil {
@@ -450,11 +469,16 @@ func buildFleetNarrativeInput(data *MultiReportData) FleetNarrativeInput {
}
for _, alert := range rd.Alerts {
if alert.ResolvedTime != nil {
if alert.Recovered() {
summary.ResolvedAlerts++
in.Aggregate.TotalResolvedAlerts++
continue
}
}
// An alert that moved without recovering stays active until the
// fleet report lists its successor's own alert.
unresolved := unresolvedAlertsAmong(rd.Alerts, fleetAlerts)
summary.MovedTo = movedAlertSuccessors(unresolved)
for _, alert := range unresolved {
summary.ActiveAlerts++
in.Aggregate.TotalActiveAlerts++
if alert.Level == "critical" {
+19 -58
View File
@@ -145,53 +145,11 @@ func (HeuristicNarrator) Narrate(_ context.Context, in NarrativeInput) (Narrativ
}
func heuristicHealthStatus(in NarrativeInput) string {
criticalAlerts := 0
warningAlerts := 0
for _, alert := range in.Alerts {
if alert.ResolvedTime != nil {
continue
}
if alert.Level == "critical" {
criticalAlerts++
} else {
warningAlerts++
}
}
switch {
case criticalAlerts > 0:
return "CRITICAL"
case warningAlerts > 0:
return "WARNING"
default:
return "HEALTHY"
}
return assessAlertHealth(in.Alerts).Status
}
func heuristicHealthMessage(in NarrativeInput) string {
criticalAlerts := 0
warningAlerts := 0
for _, alert := range in.Alerts {
if alert.ResolvedTime != nil {
continue
}
if alert.Level == "critical" {
criticalAlerts++
} else {
warningAlerts++
}
}
switch {
case criticalAlerts == 1:
return "1 critical issue requires immediate attention"
case criticalAlerts > 1:
return fmt.Sprintf("%d critical issues require immediate attention", criticalAlerts)
case warningAlerts == 1:
return "1 warning detected - review recommended"
case warningAlerts > 1:
return fmt.Sprintf("%d warnings detected - review recommended", warningAlerts)
default:
return "All systems operating normally"
}
return assessAlertHealth(in.Alerts).Message
}
// heuristicObservations mirrors the original generateObservations rules.
@@ -273,7 +231,7 @@ func heuristicObservations(in NarrativeInput) []NarrativeBullet {
resolved := 0
for _, alert := range in.Alerts {
if alert.ResolvedTime != nil {
if alert.Recovered() {
resolved++
}
}
@@ -283,6 +241,17 @@ func heuristicObservations(in NarrativeInput) []NarrativeBullet {
Severity: NarrativeSeverityInfo,
})
}
// A close that was not a recovery is recorded as what happened. The
// health verdict, the active list and the recommendations carry the
// warning while the successor's reading is not in this report.
for _, alert := range in.Alerts {
if alert.ClosedWithoutRecovery() {
obs = append(obs, NarrativeBullet{
Text: alert.ResolutionSummary(),
Severity: NarrativeSeverityInfo,
})
}
}
for _, disk := range in.Disks {
if disk.WearLevel > 0 && disk.WearLevel <= 10 {
@@ -314,19 +283,8 @@ func heuristicObservations(in NarrativeInput) []NarrativeBullet {
// heuristicRecommendations mirrors the original generateRecommendations rules.
func heuristicRecommendations(in NarrativeInput) []string {
criticalAlerts := 0
warningAlerts := 0
for _, alert := range in.Alerts {
if alert.ResolvedTime != nil {
continue
}
if alert.Level == "critical" {
criticalAlerts++
} else {
warningAlerts++
}
}
_ = warningAlerts // retained for future shaping; matches original signature
health := assessAlertHealth(in.Alerts)
criticalAlerts := health.Critical
var recs []string
@@ -344,6 +302,9 @@ func heuristicRecommendations(in NarrativeInput) []string {
if criticalAlerts > 0 {
recs = append(recs, "Investigate and resolve critical alerts immediately")
}
for _, successor := range movedAlertSuccessors(health.Unresolved) {
recs = append(recs, fmt.Sprintf("Check the current reading on %s - an alert moved there without recovering", successor))
}
if stats, ok := in.MetricStats["memory"]; ok {
if stats.Avg > 85 {
+92 -73
View File
@@ -464,41 +464,23 @@ func (g *PDFGenerator) writeCoverPage(pdf *fpdf.Fpdf, data *ReportData) {
func (g *PDFGenerator) writeExecutiveSummary(pdf *fpdf.Fpdf, data *ReportData) {
pageWidth, _ := pdf.GetPageSize()
// Determine overall health status
healthStatus := "HEALTHY"
// Determine overall health status. An alert that closed without
// recovering (moved to its Pulse agent) still counts until the report
// lists the successor's own alert for that metric.
health := assessAlertHealth(data.Alerts)
healthStatus := health.Status
healthColor := colorAccent // Green
healthMessage := "All systems operating normally"
healthMessage := health.Message
activeAlerts := 0
criticalAlerts := 0
warningAlerts := 0
for _, alert := range data.Alerts {
if alert.ResolvedTime == nil {
activeAlerts++
if alert.Level == "critical" {
criticalAlerts++
} else {
warningAlerts++
}
}
}
unresolved := health.Unresolved
activeAlerts := len(unresolved)
criticalAlerts := health.Critical
warningAlerts := health.Warning
if criticalAlerts > 0 {
healthStatus = "CRITICAL"
healthColor = colorDanger
if criticalAlerts == 1 {
healthMessage = "1 critical issue requires immediate attention"
} else {
healthMessage = fmt.Sprintf("%d critical issues require immediate attention", criticalAlerts)
}
} else if warningAlerts > 0 {
healthStatus = "WARNING"
healthColor = colorWarning
if warningAlerts == 1 {
healthMessage = "1 warning detected - review recommended"
} else {
healthMessage = fmt.Sprintf("%d warnings detected - review recommended", warningAlerts)
}
} else if data.TotalPoints == 0 && len(data.Summary.ByMetric) == 0 {
// No metrics arrived for the requested window. Reporting
// HEALTHY would green-light an empty report and mislead the
@@ -633,8 +615,8 @@ func (g *PDFGenerator) writeExecutiveSummary(pdf *fpdf.Fpdf, data *ReportData) {
pdf.SetFont("Arial", "", 9)
alertCount := 0
for _, alert := range data.Alerts {
if alert.ResolvedTime == nil && alertCount < 5 {
for _, alert := range unresolved {
if alertCount < 5 {
if alert.Level == "critical" {
pdf.SetTextColor(colorDanger[0], colorDanger[1], colorDanger[2])
pdf.CellFormat(8, 5, "!", "", 0, "C", false, 0, "")
@@ -643,11 +625,7 @@ func (g *PDFGenerator) writeExecutiveSummary(pdf *fpdf.Fpdf, data *ReportData) {
pdf.CellFormat(8, 5, "!", "", 0, "C", false, 0, "")
}
pdf.SetTextColor(colorTextDark[0], colorTextDark[1], colorTextDark[2])
msg := alert.Message
if len(msg) > 70 {
msg = msg[:67] + "..."
}
pdf.CellFormat(0, 5, msg, "", 1, "L", false, 0, "")
pdf.CellFormat(0, 5, unresolvedAlertLine(alert, 70), "", 1, "L", false, 0, "")
alertCount++
}
}
@@ -1509,8 +1487,12 @@ func (g *PDFGenerator) writeAlertsSection(pdf *fpdf.Fpdf, data *ReportData) {
// Started
pdf.CellFormat(colWidths[3], 6, alert.StartTime.Format("Jan 02 15:04"), "1", 0, "C", fill, 0, "")
// Resolved
if alert.ResolvedTime != nil {
// Resolved. A close that was not a recovery reads "Moved" in
// neutral text, never as a green resolve time.
if alert.ClosedWithoutRecovery() {
pdf.SetTextColor(colorTextMuted[0], colorTextMuted[1], colorTextMuted[2])
pdf.CellFormat(colWidths[4], 6, closedWithoutRecoveryLabel(alert), "1", 0, "C", fill, 0, "")
} else if alert.ResolvedTime != nil {
pdf.SetTextColor(colorAccent[0], colorAccent[1], colorAccent[2])
pdf.CellFormat(colWidths[4], 6, alert.ResolvedTime.Format("Jan 02 15:04"), "1", 0, "C", fill, 0, "")
} else {
@@ -1523,9 +1505,62 @@ func (g *PDFGenerator) writeAlertsSection(pdf *fpdf.Fpdf, data *ReportData) {
fill = !fill
}
// Say once what each "Moved" row means, in the alert engine's words.
var notes []string
seenNotes := make(map[string]struct{})
for _, alert := range data.Alerts {
summary := alert.ResolutionSummary()
if summary == "" {
continue
}
note := closedWithoutRecoveryLabel(alert) + ": " + summary
if _, ok := seenNotes[note]; ok {
continue
}
seenNotes[note] = struct{}{}
notes = append(notes, note)
}
if len(notes) > 0 {
pdf.Ln(2)
pdf.SetFont("Arial", "I", 8)
pdf.SetTextColor(colorTextMuted[0], colorTextMuted[1], colorTextMuted[2])
for _, note := range notes {
pdf.MultiCell(0, 4, note, "", "L", false)
}
pdf.SetTextColor(colorTextDark[0], colorTextDark[1], colorTextDark[2])
}
pdf.Ln(10)
}
// closedWithoutRecoveryLabel is the alerts table's Resolved cell for a close
// that was not a recovery.
func closedWithoutRecoveryLabel(alert AlertInfo) string {
if alert.Resolution != nil && alert.Resolution.Reason == AlertResolutionMovedToAgent {
return "Moved"
}
return "Closed"
}
// unresolvedAlertLine is one alert in an "Active Alerts" list, cut to max
// characters. An alert that closed without recovering keeps its outcome,
// such as "(moved to pve1 (Host Agent))", and gives up message text instead.
func unresolvedAlertLine(alert AlertInfo, max int) string {
suffix := ""
if outcome := alert.ResolutionOutcome(); outcome != "" {
suffix = " (" + outcome + ")"
}
room := max - len(suffix)
if room < 10 {
room = 10
}
msg := alert.Message
if len(msg) > room {
msg = msg[:room-3] + "..."
}
return msg + suffix
}
// writeStorageSection writes storage pools table
func (g *PDFGenerator) writeStorageSection(pdf *fpdf.Fpdf, data *ReportData) {
if len(data.Storage) == 0 {
@@ -1831,8 +1866,10 @@ func (g *PDFGenerator) GenerateMulti(data *MultiReportData) ([]byte, error) {
_, pageHeight := pdf.GetPageSize()
bottomLimit := pageHeight - 30
pageOpen := false
fleetAlerts := data.fleetAlerts()
for _, rd := range data.Resources {
startNewPage := !pageOpen || pdf.GetY()+condensedResourceBlockHeight(rd) > bottomLimit
unresolved := unresolvedAlertsAmong(rd.Alerts, fleetAlerts)
startNewPage := !pageOpen || pdf.GetY()+condensedResourceBlockHeight(rd, unresolved) > bottomLimit
if startNewPage {
pdf.AddPage()
g.addMultiPageHeader(pdf, data, "Resource Details")
@@ -1843,7 +1880,7 @@ func (g *PDFGenerator) GenerateMulti(data *MultiReportData) ([]byte, error) {
pdf.Line(20, pdf.GetY(), 190, pdf.GetY())
pdf.Ln(5)
}
g.writeCondensedResourcePage(pdf, rd)
g.writeCondensedResourcePage(pdf, rd, unresolved)
}
// Add page numbers to all pages except cover
@@ -2043,15 +2080,14 @@ func (g *PDFGenerator) writeFleetSummary(pdf *fpdf.Fpdf, data *MultiReportData)
totalActive := 0
totalCritical := 0
totalWarning := 0
fleetAlerts := data.fleetAlerts()
for _, rd := range data.Resources {
for _, alert := range rd.Alerts {
if alert.ResolvedTime == nil {
totalActive++
if alert.Level == "critical" {
totalCritical++
} else {
totalWarning++
}
for _, alert := range unresolvedAlertsAmong(rd.Alerts, fleetAlerts) {
totalActive++
if alert.Level == "critical" {
totalCritical++
} else {
totalWarning++
}
}
}
@@ -2215,12 +2251,7 @@ func (g *PDFGenerator) writeFleetSummary(pdf *fpdf.Fpdf, data *MultiReportData)
pdf.CellFormat(colWidths[6], 6, fmt.Sprintf("%.1f%%", avgDisk), "1", 0, "C", fill, 0, "")
// Alerts count
alertCount := 0
for _, alert := range rd.Alerts {
if alert.ResolvedTime == nil {
alertCount++
}
}
alertCount := len(unresolvedAlertsAmong(rd.Alerts, fleetAlerts))
pdf.SetTextColor(getAlertCountColor(alertCount)[0], getAlertCountColor(alertCount)[1], getAlertCountColor(alertCount)[2])
pdf.CellFormat(colWidths[7], 6, fmt.Sprintf("%d", alertCount), "1", 0, "C", fill, 0, "")
@@ -2395,17 +2426,12 @@ func writeFleetNarrativeSection(pdf *fpdf.Fpdf, fn *FleetNarrative) {
// resource block needs so the fleet layout can decide whether it fits on
// the current page. Estimates err slightly high; a block that still
// overflows is carried across pages by fpdf's auto page break.
func condensedResourceBlockHeight(rd *ReportData) float64 {
func condensedResourceBlockHeight(rd *ReportData, unresolved []AlertInfo) float64 {
height := 11.0 + 6.0 + 27.0 + 8.0 // name header + availability line + stats bar + padding
if len(rd.Metrics["cpu"]) >= 2 || len(rd.Metrics["memory"]) >= 2 {
height += 58 // chart title + canvas + legend
}
active := 0
for _, alert := range rd.Alerts {
if alert.ResolvedTime == nil {
active++
}
}
active := len(unresolved)
if active > 0 {
shown := active
if shown > 3 {
@@ -2422,7 +2448,9 @@ func condensedResourceBlockHeight(rd *ReportData) float64 {
return height
}
func (g *PDFGenerator) writeCondensedResourcePage(pdf *fpdf.Fpdf, rd *ReportData) {
// writeCondensedResourcePage writes one resource's block of a fleet report.
// unresolved is the resource's live alerts matched against the whole fleet.
func (g *PDFGenerator) writeCondensedResourcePage(pdf *fpdf.Fpdf, rd *ReportData, unresolved []AlertInfo) {
// Resource header
resourceName := rd.ResourceID
if rd.Resource != nil && rd.Resource.Name != "" {
@@ -2586,13 +2614,8 @@ func (g *PDFGenerator) writeCondensedResourcePage(pdf *fpdf.Fpdf, rd *ReportData
}
}
// Active alerts (up to 3)
activeAlerts := make([]AlertInfo, 0)
for _, alert := range rd.Alerts {
if alert.ResolvedTime == nil {
activeAlerts = append(activeAlerts, alert)
}
}
// Active alerts (up to 3), including any that moved without recovering
activeAlerts := unresolved
if len(activeAlerts) > 0 {
pdf.Ln(3)
@@ -2615,11 +2638,7 @@ func (g *PDFGenerator) writeCondensedResourcePage(pdf *fpdf.Fpdf, rd *ReportData
}
pdf.CellFormat(6, 5, "!", "", 0, "C", false, 0, "")
pdf.SetTextColor(colorTextDark[0], colorTextDark[1], colorTextDark[2])
msg := alert.Message
if len(msg) > 80 {
msg = msg[:77] + "..."
}
pdf.CellFormat(0, 5, msg, "", 1, "L", false, 0, "")
pdf.CellFormat(0, 5, unresolvedAlertLine(alert, 80), "", 1, "L", false, 0, "")
}
if len(activeAlerts) > 3 {
pdf.SetTextColor(colorTextMuted[0], colorTextMuted[1], colorTextMuted[2])
+22
View File
@@ -115,6 +115,28 @@ type AlertInfo struct {
StartTime time.Time
ResolvedTime *time.Time // nil if still active
Acknowledged bool
// ResourceID is the resource the alert was raised on, so an alert that
// moved can find the alert its successor raised for the same metric.
ResourceID string
// Resolution is set only on a resolved alert whose close was not a
// recovery, such as a node alert handed to its Pulse agent. A report
// never counts such an alert as resolved.
Resolution *AlertResolution
}
// AlertResolution says why an alert closed without its condition clearing.
// It mirrors models.AlertResolution so this package stays free of internal
// dependencies.
type AlertResolution struct {
// Reason is the alert engine's reason code, such as
// AlertResolutionMovedToAgent.
Reason string
SuccessorResourceID string
SuccessorName string
// Summary is the alert engine's one-line account, such as "Alert moved
// to pve1 (Host Agent). This is not a recovery: check the agent for the
// current reading."
Summary string
}
// BackupInfo contains backup information for VMs/containers