mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-11-01 08:20:01 +00:00
pulse-triage[bot]
b0f7a63181
The sender already carries moved_to_agent reasons, but the public PSA template discarded them and described every resolved occurrence as recovered. Preserve each member's reason and successor, explain mixed and unknown outcomes, and exercise copied templates through the real sender plus the shipped Markdown renderer. Contract-Neutral: Existing webhook help and copied-template regression coverage only; no runtime, endpoint, schema or delivery policy change. Change-source: pulse-maintainer
328 lines
15 KiB
Go
328 lines
15 KiB
Go
package notifications
|
|
|
|
import (
|
|
"crypto/hmac"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// Exercise the copyable templates through the real sender, not a second
|
|
// renderer or a model receiver. The input override permits an exact-parent
|
|
// documentation negative control without modifying production source.
|
|
func webhookGuide(t *testing.T) string {
|
|
t.Helper()
|
|
path := os.Getenv("PULSE_WEBHOOK_GUIDE_TEST_INPUT")
|
|
if path == "" {
|
|
path = "../../docs/WEBHOOKS.md"
|
|
}
|
|
data, err := os.ReadFile(path)
|
|
require.NoError(t, err)
|
|
return string(data)
|
|
}
|
|
|
|
func webhookGuidePSATemplate(t *testing.T) string {
|
|
t.Helper()
|
|
sections := strings.SplitN(webhookGuide(t), "### Sample PSA payloads", 2)
|
|
require.Len(t, sections, 2)
|
|
block := regexp.MustCompile("(?s)```json\\n(.*?)\\n```").FindStringSubmatch(sections[1])
|
|
require.Len(t, block, 2)
|
|
return block[1]
|
|
}
|
|
|
|
type guideMember struct {
|
|
ID string `json:"alertId"`
|
|
StartedAt string `json:"startedAt"`
|
|
Severity string `json:"severity"`
|
|
ResourceID string `json:"resourceId"`
|
|
Resource string `json:"resource"`
|
|
Summary string `json:"summary"`
|
|
ResolutionReason string `json:"resolutionReason"`
|
|
SuccessorResourceID string `json:"successorResourceId"`
|
|
SuccessorName string `json:"successorName"`
|
|
}
|
|
|
|
type guidePayload struct {
|
|
Event string `json:"event"`
|
|
TenantID string `json:"tenantId"`
|
|
StartedAt string `json:"startedAt"`
|
|
AlertCount int `json:"alertCount"`
|
|
Members []guideMember `json:"alerts"`
|
|
NotRecovered *bool `json:"notRecovered"`
|
|
}
|
|
|
|
type guideRequest struct {
|
|
body, eventID, timestamp, signature string
|
|
}
|
|
|
|
func webhookGuideSender(t *testing.T, payloadTemplate string) (*NotificationManager, WebhookConfig, <-chan guideRequest) {
|
|
t.Helper()
|
|
manager := NewNotificationManager("https://pulse.example.test")
|
|
require.NoError(t, manager.UpdateAllowedPrivateCIDRs("127.0.0.1"))
|
|
manager.SetTenantIdentityResolver(func() (string, string) { return "synthetic-tenant", "Synthetic tenant" })
|
|
requests := make(chan guideRequest, 16)
|
|
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
body, err := io.ReadAll(r.Body)
|
|
if err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
requests <- guideRequest{string(body), r.Header.Get("X-Pulse-Event-ID"),
|
|
r.Header.Get("X-Pulse-Timestamp"), r.Header.Get("X-Pulse-Signature")}
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
webhook := WebhookConfig{ID: "synthetic-guide", Name: "Synthetic guide receiver", URL: server.URL,
|
|
Enabled: true, Service: "generic", SigningSecret: "synthetic-guide-test-secret", Template: payloadTemplate}
|
|
return manager, webhook, requests
|
|
}
|
|
|
|
func webhookGuideRead(t *testing.T, requests <-chan guideRequest) (guideRequest, guidePayload) {
|
|
t.Helper()
|
|
select {
|
|
case request := <-requests:
|
|
var payload guidePayload
|
|
require.NoError(t, json.Unmarshal([]byte(request.body), &payload), "copied template must render valid JSON")
|
|
mac := hmac.New(sha256.New, []byte("synthetic-guide-test-secret"))
|
|
_, err := mac.Write([]byte(request.timestamp + "." + request.body))
|
|
require.NoError(t, err)
|
|
require.Equal(t, "v1="+hex.EncodeToString(mac.Sum(nil)), request.signature)
|
|
return request, payload
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("no local sender request received")
|
|
return guideRequest{}, guidePayload{}
|
|
}
|
|
}
|
|
|
|
func TestWebhookGuidePSALifecycle(t *testing.T) {
|
|
manager, webhook, requests := webhookGuideSender(t, webhookGuidePSATemplate(t))
|
|
start := time.Date(2026, 10, 4, 7, 0, 0, 123456789, time.UTC)
|
|
alert := &alerts.Alert{ID: "host::metric-threshold:cpu", StartTime: start,
|
|
Level: alerts.AlertLevelWarning, Type: "cpu", ResourceID: "synthetic-host",
|
|
ResourceName: "quoted \"host\"\nwith é", Message: "synthetic \\ message\n\"quoted\"", Value: 95, Threshold: 90}
|
|
|
|
require.NoError(t, manager.sendGroupedWebhook(webhook, []*alerts.Alert{alert}))
|
|
firstRequest, first := webhookGuideRead(t, requests)
|
|
require.Len(t, first.Members, 1)
|
|
require.Equal(t, "alert", first.Event)
|
|
require.Equal(t, "synthetic-tenant", first.TenantID)
|
|
require.Equal(t, 1, first.AlertCount)
|
|
require.Equal(t, guideMember{ID: alert.ID, StartedAt: start.Format(time.RFC3339Nano), Severity: "warning",
|
|
ResourceID: alert.ResourceID, Resource: alert.ResourceName, Summary: alert.Message}, first.Members[0])
|
|
|
|
// A synthetic retry retains the member identity even when rendering anew.
|
|
require.NoError(t, manager.sendGroupedWebhook(webhook, []*alerts.Alert{alert}))
|
|
retryRequest, retry := webhookGuideRead(t, requests)
|
|
require.Equal(t, first.Members, retry.Members)
|
|
require.Equal(t, firstRequest.eventID, retryRequest.eventID)
|
|
|
|
critical := alert.Clone()
|
|
critical.Level = alerts.AlertLevelCritical
|
|
require.NoError(t, manager.sendGroupedWebhook(webhook, []*alerts.Alert{critical}))
|
|
criticalRequest, escalated := webhookGuideRead(t, requests)
|
|
require.Equal(t, firstRequest.eventID, criticalRequest.eventID, "header cannot identify a severity change")
|
|
require.Equal(t, first.Members[0].StartedAt, escalated.Members[0].StartedAt)
|
|
require.Equal(t, "critical", escalated.Members[0].Severity)
|
|
|
|
require.NoError(t, manager.sendResolvedWebhook(webhook, []*alerts.Alert{critical}, start.Add(time.Minute)))
|
|
_, resolved := webhookGuideRead(t, requests)
|
|
require.Equal(t, "resolved", resolved.Event)
|
|
require.Equal(t, escalated.Members, resolved.Members, "recovery must correlate to the original occurrence")
|
|
|
|
// A second occurrence in the SAME SECOND defeats both header-only and
|
|
// whole-second primary-start deduplication. Member precision preserves it.
|
|
recurrent := critical.Clone()
|
|
recurrent.StartTime = start.Add(100 * time.Nanosecond)
|
|
require.NoError(t, manager.sendGroupedWebhook(webhook, []*alerts.Alert{recurrent}))
|
|
recurrentRequest, recurrence := webhookGuideRead(t, requests)
|
|
require.Equal(t, firstRequest.eventID, recurrentRequest.eventID)
|
|
require.Equal(t, first.StartedAt, recurrence.StartedAt)
|
|
require.NotEqual(t, first.Members[0].StartedAt, recurrence.Members[0].StartedAt)
|
|
require.Equal(t, recurrent.StartTime.Format(time.RFC3339Nano), recurrence.Members[0].StartedAt)
|
|
}
|
|
|
|
func TestWebhookGuidePSAGroupMembershipAndInfo(t *testing.T) {
|
|
manager, webhook, requests := webhookGuideSender(t, webhookGuidePSATemplate(t))
|
|
start := time.Date(2026, 10, 4, 9, 0, 0, 987654321, time.FixedZone("synthetic", 2*60*60))
|
|
primary := &alerts.Alert{ID: "primary", StartTime: start, Level: alerts.AlertLevelWarning}
|
|
info := &alerts.Alert{ID: "info-member", StartTime: start.Add(time.Second), Level: alerts.AlertLevelInfo}
|
|
critical := &alerts.Alert{ID: "critical-member", StartTime: start.Add(2 * time.Second), Level: alerts.AlertLevelCritical}
|
|
for _, members := range [][]*alerts.Alert{{primary, info}, {primary, critical}, {primary, critical, info}} {
|
|
require.NoError(t, manager.sendGroupedWebhook(webhook, members))
|
|
request, payload := webhookGuideRead(t, requests)
|
|
require.Equal(t, "primary:alert", request.eventID, "changed groups can have the same correlation header")
|
|
require.Equal(t, len(members), payload.AlertCount)
|
|
require.Len(t, payload.Members, len(members))
|
|
for index, alert := range members {
|
|
require.Equal(t, alert.ID, payload.Members[index].ID)
|
|
require.Equal(t, string(alert.Level), payload.Members[index].Severity)
|
|
require.Equal(t, alert.StartTime.UTC().Format(time.RFC3339Nano), payload.Members[index].StartedAt)
|
|
}
|
|
}
|
|
require.NoError(t, manager.sendResolvedWebhook(webhook, []*alerts.Alert{primary, critical, info}, start.Add(time.Minute)))
|
|
_, resolved := webhookGuideRead(t, requests)
|
|
require.Equal(t, "resolved", resolved.Event)
|
|
require.Len(t, resolved.Members, 3)
|
|
require.Equal(t, "info", resolved.Members[2].Severity)
|
|
}
|
|
|
|
func TestWebhookGuidePSACloseReasons(t *testing.T) {
|
|
start := time.Date(2026, 10, 7, 3, 0, 0, 123456789, time.UTC)
|
|
recovered := &alerts.Alert{ID: "node::metric-threshold:cpu", StartTime: start,
|
|
ResourceID: "node", ResourceName: "synthetic-node", Node: "synthetic-node",
|
|
Level: alerts.AlertLevelWarning, Type: "cpu", Value: 95, Threshold: 90,
|
|
Message: "Original high CPU reading"}
|
|
moved := recovered.Clone()
|
|
moved.ID = "node::metric-threshold:memory"
|
|
moved.Type = "memory"
|
|
moved.Message = "Original high memory reading"
|
|
moved.Resolution = &alerts.AlertResolution{Reason: alerts.AlertResolutionMovedToAgent,
|
|
SuccessorResourceID: "agent-\"id\"\\path", SuccessorName: "agent \"name\"\né — 警告"}
|
|
missingSuccessor := moved.Clone()
|
|
missingSuccessor.Resolution.SuccessorResourceID = ""
|
|
missingSuccessor.Resolution.SuccessorName = ""
|
|
unknown := moved.Clone()
|
|
unknown.Resolution.Reason = alerts.AlertResolutionReason("future-\"reason\"\n")
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
members []*alerts.Alert
|
|
firing bool
|
|
notRecovered bool
|
|
}{
|
|
{"firing-is-not-recovery", []*alerts.Alert{recovered}, true, false},
|
|
{"ordinary-recovery", []*alerts.Alert{recovered}, false, false},
|
|
{"moved-still-above-threshold", []*alerts.Alert{moved}, false, true},
|
|
{"recovered-primary-moved-member", []*alerts.Alert{recovered, moved}, false, true},
|
|
{"moved-primary-recovered-member", []*alerts.Alert{moved, recovered}, false, true},
|
|
{"missing-successor", []*alerts.Alert{missingSuccessor}, false, true},
|
|
// The batch flag recognises current reasons only. Preserve an unknown
|
|
// member reason even when the flag is false; it is not a health verdict.
|
|
{"unknown-reason-preserved", []*alerts.Alert{unknown}, false, false},
|
|
{"nil-member-skipped", []*alerts.Alert{nil, recovered, moved}, false, true},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
manager, webhook, requests := webhookGuideSender(t, webhookGuidePSATemplate(t))
|
|
if tc.firing {
|
|
require.NoError(t, manager.sendGroupedWebhook(webhook, tc.members))
|
|
} else {
|
|
require.NoError(t, manager.sendResolvedWebhook(webhook, tc.members, start.Add(time.Minute)))
|
|
}
|
|
request, payload := webhookGuideRead(t, requests)
|
|
require.NotNil(t, payload.NotRecovered, "copied payload must distinguish missing from false")
|
|
require.Equal(t, tc.notRecovered, *payload.NotRecovered)
|
|
require.Equal(t, map[bool]string{true: "alert", false: "resolved"}[tc.firing], payload.Event)
|
|
|
|
var raw struct {
|
|
Members []map[string]json.RawMessage `json:"alerts"`
|
|
}
|
|
require.NoError(t, json.Unmarshal([]byte(request.body), &raw))
|
|
index := 0
|
|
for _, member := range tc.members {
|
|
if member == nil {
|
|
continue
|
|
}
|
|
require.Less(t, index, len(payload.Members))
|
|
got := payload.Members[index]
|
|
require.Equal(t, member.ID, got.ID)
|
|
require.Equal(t, member.StartTime.UTC().Format(time.RFC3339Nano), got.StartedAt)
|
|
require.Equal(t, member.Message, got.Summary, "member message is not a recovery summary")
|
|
for _, key := range []string{"resolutionReason", "successorResourceId", "successorName"} {
|
|
require.Contains(t, raw.Members[index], key, "an empty known field must differ from an older missing field")
|
|
}
|
|
wantReason, wantID, wantName := "", "", ""
|
|
if member.Resolution != nil {
|
|
wantReason = string(member.Resolution.Reason)
|
|
wantID = member.Resolution.SuccessorResourceID
|
|
wantName = member.Resolution.SuccessorName
|
|
}
|
|
require.Equal(t, wantReason, got.ResolutionReason)
|
|
require.Equal(t, wantID, got.SuccessorResourceID)
|
|
require.Equal(t, wantName, got.SuccessorName)
|
|
index++
|
|
}
|
|
require.Len(t, payload.Members, index)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestWebhookGuideShortTemplateEscapesStrings(t *testing.T) {
|
|
section := strings.SplitN(webhookGuide(t), "### Sample PSA payloads", 2)[0]
|
|
block := regexp.MustCompile("(?s)```http\\nPOST /api/notifications/webhooks\\nX-Pulse-Org-ID:.*?\\n\\n(.*?)\\n```").FindStringSubmatch(section)
|
|
require.Len(t, block, 2)
|
|
var config struct {
|
|
Template string `json:"template"`
|
|
}
|
|
require.NoError(t, json.Unmarshal([]byte(block[1]), &config))
|
|
manager, webhook, requests := webhookGuideSender(t, config.Template)
|
|
alert := &alerts.Alert{ID: "quoted\"id", ResourceName: "host\"\\\nname", Message: "message\"\\\n",
|
|
StartTime: time.Date(2026, 10, 4, 7, 0, 0, 123, time.UTC), Level: alerts.AlertLevelWarning}
|
|
require.NoError(t, manager.sendGroupedWebhook(webhook, []*alerts.Alert{alert}))
|
|
request, _ := webhookGuideRead(t, requests)
|
|
var payload map[string]interface{}
|
|
require.NoError(t, json.Unmarshal([]byte(request.body), &payload))
|
|
require.Equal(t, alert.ID, payload["alertId"])
|
|
require.Equal(t, "warning: "+alert.ResourceName+" "+alert.Message, payload["summary"])
|
|
require.Equal(t, alert.StartTime.Format(time.RFC3339), payload["startedAt"])
|
|
}
|
|
|
|
func webhookGuideExamplePayload(t *testing.T) string {
|
|
t.Helper()
|
|
sections := strings.SplitN(webhookGuide(t), "**Example Payload:**", 2)
|
|
require.Len(t, sections, 2)
|
|
block := regexp.MustCompile("(?s)```json\\n(.*?)\\n```").FindStringSubmatch(sections[1])
|
|
require.Len(t, block, 2)
|
|
return block[1]
|
|
}
|
|
|
|
func TestWebhookGuideExamplePayloadEscapesAlertText(t *testing.T) {
|
|
for _, message := range []struct {
|
|
name, text string
|
|
value, wantValue float64
|
|
}{
|
|
{"simple-test", "A simple test notification", 0, 0},
|
|
{"quoted-name", `Resource "synthetic-host" is unavailable`, 95.25, 95.3},
|
|
{"backslash-path", `Synthetic path C:\backup\reports`, 12.5, 12.5},
|
|
{"multiline", "Synthetic first line\nsecond line\r\nthird\tcolumn", 1, 1},
|
|
{"unicode", "Synthetic é host — 警告", 0.0286, 0},
|
|
{"member-injection", `Synthetic text", "injected": true, "other": "value`, 99, 99},
|
|
} {
|
|
t.Run(message.name, func(t *testing.T) {
|
|
manager, webhook, requests := webhookGuideSender(t, webhookGuideExamplePayload(t))
|
|
alert := &alerts.Alert{ID: "synthetic-example", ResourceName: "synthetic-host",
|
|
Level: alerts.AlertLevelWarning, Message: message.text, Value: message.value}
|
|
require.NoError(t, manager.sendGroupedWebhook(webhook, []*alerts.Alert{alert}))
|
|
request, _ := webhookGuideRead(t, requests)
|
|
var payload map[string]interface{}
|
|
require.NoError(t, json.Unmarshal([]byte(request.body), &payload))
|
|
require.Len(t, payload, 2, "alert text must not add JSON members")
|
|
require.Equal(t, "Alert: warning - "+message.text, payload["text"])
|
|
require.Equal(t, message.wantValue, payload["value"], "preserve the sender's one-decimal numeric value")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestWebhookGuideExamplePayloadEscapesRecoveryText(t *testing.T) {
|
|
manager, webhook, requests := webhookGuideSender(t, webhookGuideExamplePayload(t))
|
|
alert := &alerts.Alert{ID: "synthetic-example", ResourceName: "synthetic \"host\"\nwith \\ path",
|
|
Node: "synthetic-node", Level: alerts.AlertLevelWarning, Value: 0,
|
|
StartTime: time.Date(2026, 10, 6, 4, 0, 0, 0, time.UTC)}
|
|
require.NoError(t, manager.sendResolvedWebhook(webhook, []*alerts.Alert{alert}, alert.StartTime.Add(time.Minute)))
|
|
request, _ := webhookGuideRead(t, requests)
|
|
var payload map[string]interface{}
|
|
require.NoError(t, json.Unmarshal([]byte(request.body), &payload))
|
|
require.Len(t, payload, 2)
|
|
require.Equal(t, "Alert: warning - "+alert.ResourceName+" on "+alert.Node+" is now healthy", payload["text"])
|
|
require.Equal(t, float64(0), payload["value"])
|
|
}
|