Files
pulse/internal/unifiedresources/incident_changes.go
T
rcourtman 1f2ff618b6 Own PVE disk alert history rows by recorded hardware
PVE disk health and wearout alerts reference the disk by instance, node
and device path, so a physical disk's history never showed its own SMART
and wearout alerts. Binding the path reference failed review: a binding
belongs to the reference string, so any disk that takes over the path
inherits every row and every read of it.

Own each lifecycle row instead. The alert now records the disk's WWN
next to its serial, and the history writer puts the row under the disk
the registry knows by that hardware identity, wherever it sits now. The
registry only translates identity into a canonical ID, so a generation
that still places another disk at the path cannot claim the row, and no
check precedes a later write. A disk not yet in inventory, or removed,
gets the ID the registry mints for its identity. Rows without usable or
unambiguous identity stay under the reference, as do rows journaled
before this change; journal rows are never rewritten.

Reads by the reference still return every row raised under it, through
the alert identifier index, and the incident projection keeps owned rows
under their alert reference, so the Alerts history Resource action and
incident timelines show the same occurrences as before. A read by the
disk returns its own occurrences with their saved shells and notes.
2026-10-07 11:18:45 +01:00

255 lines
8.1 KiB
Go

package unifiedresources
import (
"fmt"
"strings"
"time"
"github.com/google/uuid"
)
const (
MetadataAlertIdentifier = "alert_identifier"
MetadataAlertStartedAt = "alert_started_at"
MetadataAlertType = "alert_type"
MetadataAlertLevel = "alert_level"
MetadataAlertMessage = "alert_message"
MetadataAlertValue = "alert_value"
MetadataAlertThreshold = "alert_threshold"
// MetadataAlertResolution is the reason code of a resolve that was not a
// recovery, such as "moved_to_agent". Ordinary recoveries omit it.
MetadataAlertResolution = "alert_resolution"
// MetadataAlertResourceID is the alert's own resource reference, recorded
// on a lifecycle row written under another resource because the row's
// hardware identity names it (see OwnedAlertReference).
MetadataAlertResourceID = "alert_resource_id"
MetadataCommand = "command"
MetadataSuccess = "success"
MetadataOutputExcerpt = "output_excerpt"
MetadataRunbookID = "runbook_id"
MetadataOutcome = "outcome"
MetadataAutomatic = "automatic"
MetadataMessage = "message"
)
const resourceChangeOutputExcerptLimit = 500
// AlertTimelineChange captures the alert-lifecycle details that should be
// durable in the canonical resource history rather than only incident memory.
type AlertTimelineChange struct {
AlertIdentifier string
AlertStartedAt time.Time
AlertType string
AlertLevel string
AlertMessage string
AlertValue float64
AlertThreshold float64
AlertMetadata map[string]any
// ResolutionReason and ResolutionSummary describe a resolve that was not
// a recovery. They are empty for an ordinary recovery and ignored on
// every other lifecycle kind.
ResolutionReason string
ResolutionSummary string
}
// BuildAlertTimelineChange constructs a canonical resource change for alert
// lifecycle events tied to a specific resource.
func BuildAlertTimelineChange(resourceID string, kind ChangeKind, occurredAt time.Time, actor string, alert AlertTimelineChange) *ResourceChange {
resourceID = strings.TrimSpace(resourceID)
if resourceID == "" {
return nil
}
observedAt := occurredAt.UTC()
if observedAt.IsZero() {
observedAt = time.Now().UTC()
}
change := &ResourceChange{
ID: uuid.NewString(),
ObservedAt: observedAt,
ResourceID: resourceID,
Kind: kind,
SourceType: SourceHeuristic,
Confidence: ConfidenceHigh,
Actor: strings.TrimSpace(actor),
Reason: alertChangeReason(kind, alert),
Metadata: map[string]any{
MetadataAlertIdentifier: strings.TrimSpace(alert.AlertIdentifier),
MetadataAlertType: strings.TrimSpace(alert.AlertType),
MetadataAlertLevel: strings.TrimSpace(alert.AlertLevel),
MetadataAlertMessage: strings.TrimSpace(alert.AlertMessage),
MetadataAlertValue: alert.AlertValue,
MetadataAlertThreshold: alert.AlertThreshold,
},
}
if !alert.AlertStartedAt.IsZero() {
change.Metadata[MetadataAlertStartedAt] = alert.AlertStartedAt.UTC().Format(time.RFC3339Nano)
}
if reason := strings.TrimSpace(alert.ResolutionReason); kind == ChangeAlertResolved && reason != "" {
change.Metadata[MetadataAlertResolution] = reason
}
for key, value := range cloneChangeMetadata(alert.AlertMetadata) {
if _, exists := change.Metadata[key]; exists {
continue
}
change.Metadata[key] = value
}
if !occurredAt.IsZero() {
change.OccurredAt = cloneTimePtr(&occurredAt)
}
return change
}
// BuildCommandExecutionChange constructs a canonical resource change for a
// command that was executed against a resource in response to an incident.
func BuildCommandExecutionChange(resourceID, alertIdentifier, actor, command string, success bool, output string, details map[string]any) *ResourceChange {
resourceID = strings.TrimSpace(resourceID)
command = strings.TrimSpace(command)
if resourceID == "" || command == "" {
return nil
}
observedAt := time.Now().UTC()
change := &ResourceChange{
ID: uuid.NewString(),
ObservedAt: observedAt,
OccurredAt: cloneTimePtr(&observedAt),
ResourceID: resourceID,
Kind: ChangeCommandExecuted,
SourceType: SourceAgentAction,
Confidence: ConfidenceHigh,
Actor: strings.TrimSpace(actor),
Reason: commandExecutionReason(command, success),
Metadata: cloneChangeMetadata(details),
}
change.Metadata[MetadataAlertIdentifier] = strings.TrimSpace(alertIdentifier)
change.Metadata[MetadataCommand] = command
change.Metadata[MetadataSuccess] = success
if excerpt := truncateResourceChangeOutput(output, resourceChangeOutputExcerptLimit); excerpt != "" {
change.Metadata[MetadataOutputExcerpt] = excerpt
}
return change
}
// BuildRunbookExecutionChange constructs a canonical resource change for a
// runbook that was executed against a resource in response to an incident.
func BuildRunbookExecutionChange(resourceID, alertIdentifier, actor, runbookID, title, outcome string, automatic bool, message string, details map[string]any) *ResourceChange {
resourceID = strings.TrimSpace(resourceID)
runbookID = strings.TrimSpace(runbookID)
if resourceID == "" || runbookID == "" {
return nil
}
observedAt := time.Now().UTC()
change := &ResourceChange{
ID: uuid.NewString(),
ObservedAt: observedAt,
OccurredAt: cloneTimePtr(&observedAt),
ResourceID: resourceID,
Kind: ChangeRunbookExecuted,
SourceType: SourceUserAction,
Confidence: ConfidenceHigh,
Actor: strings.TrimSpace(actor),
Reason: runbookExecutionReason(title, outcome),
Metadata: cloneChangeMetadata(details),
}
if automatic {
change.SourceType = SourceAgentAction
}
change.Metadata[MetadataAlertIdentifier] = strings.TrimSpace(alertIdentifier)
change.Metadata[MetadataRunbookID] = runbookID
change.Metadata[MetadataOutcome] = strings.TrimSpace(outcome)
change.Metadata[MetadataAutomatic] = automatic
if trimmedTitle := strings.TrimSpace(title); trimmedTitle != "" {
change.Metadata["title"] = trimmedTitle
}
if trimmedMessage := strings.TrimSpace(message); trimmedMessage != "" {
change.Metadata[MetadataMessage] = trimmedMessage
}
return change
}
func alertChangeReason(kind ChangeKind, alert AlertTimelineChange) string {
message := strings.TrimSpace(alert.AlertMessage)
switch kind {
case ChangeAlertFired:
if message != "" {
return message
}
return "Alert fired"
case ChangeAlertAcknowledged:
if message != "" {
return fmt.Sprintf("Alert acknowledged: %s", message)
}
return "Alert acknowledged"
case ChangeAlertUnacknowledged:
if message != "" {
return fmt.Sprintf("Alert unacknowledged: %s", message)
}
return "Alert unacknowledged"
case ChangeAlertSnoozed:
return "Alert snoozed"
case ChangeAlertUnsnoozed:
return "Alert notifications resumed"
case ChangeAlertResolved:
// A close that was not a recovery must not read as one: the alert's
// own message describes the condition that is still unresolved.
if summary := strings.TrimSpace(alert.ResolutionSummary); summary != "" {
return summary
}
if message != "" {
return fmt.Sprintf("Alert resolved: %s", message)
}
return "Alert resolved"
default:
if message != "" {
return message
}
return "Alert event recorded"
}
}
func commandExecutionReason(command string, success bool) string {
status := "failed"
if success {
status = "succeeded"
}
return fmt.Sprintf("Command %s: %s", status, command)
}
func runbookExecutionReason(title, outcome string) string {
title = strings.TrimSpace(title)
if title == "" {
title = "runbook"
}
outcome = strings.TrimSpace(outcome)
if outcome == "" {
return fmt.Sprintf("Runbook %s executed", title)
}
return fmt.Sprintf("Runbook %s (%s)", title, outcome)
}
func cloneChangeMetadata(details map[string]any) map[string]any {
if len(details) == 0 {
return make(map[string]any)
}
cloned := make(map[string]any, len(details))
for key, value := range details {
cloned[key] = value
}
return cloned
}
func truncateResourceChangeOutput(output string, limit int) string {
output = strings.TrimSpace(output)
if output == "" || limit <= 0 {
return ""
}
runes := []rune(output)
if len(runes) <= limit {
return output
}
return string(runes[:limit]) + "..."
}