Files
pulse/internal/api/alerting/notifications.go
T
rcourtman f279dc0b07 Resolve masked webhook values per key and keep responses masked
The webhook editor sends back the ***REDACTED*** values the list returns
for every header, custom field and signing secret the user did not
retype. Three paths mishandled them:

- Update restored the whole saved header or custom-field map whenever any
  value was masked, silently dropping a header or field added in the same
  edit.
- Update echoed the restored saved custom fields, so a Pushover token and
  user key came back in the response even though the list masks them, in
  breach of the api-contracts webhook response clause. Create echoed the
  request, and could store a masked value verbatim.
- The editor's Test button posts the form, so testing an edit after a
  reload sent the literal mask as the token or header.

Resolve each masked value to the saved value of the same key on update and
on a form test that carries a saved webhook's id, keep values typed in the
same request, and drop masks with no saved value. Build list, create and
update responses from one masked shape. Normalization now drops blank
headers, so clearing a header removes it instead of sending an empty
Content-Type that generic receivers cannot parse.
2026-10-06 09:21:41 +01:00

1217 lines
42 KiB
Go

package alerting
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext"
"github.com/rcourtman/pulse-go-rewrite/internal/api/apihttp"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
internalauth "github.com/rcourtman/pulse-go-rewrite/pkg/auth"
"github.com/rs/zerolog/log"
)
const (
notificationTestRequestBodyLimit = 64 * 1024
webhookTestRequestBodyLimit = 64 * 1024
completedQueueRetentionDays = 7
deadLetterQueueRetentionDays = 30
)
// NotificationManager defines the interface for notification management operations.
type NotificationManager interface {
GetEmailConfig() notifications.EmailConfig
SetEmailConfig(notifications.EmailConfig)
GetAppriseConfig() notifications.AppriseConfig
SetAppriseConfig(notifications.AppriseConfig)
GetWebhooks() []notifications.WebhookConfig
ValidateWebhookURL(string) error
AddWebhook(notifications.WebhookConfig)
UpdateWebhook(string, notifications.WebhookConfig) error
DeleteWebhook(string) error
SendTestWebhook(notifications.WebhookConfig) error
SendTestNotificationWithConfig(string, *notifications.EmailConfig, *notifications.TestNodeInfo) error
SendTestAppriseWithConfig(notifications.AppriseConfig) error
SendTestNotification(string) error
GetWebhookHistory() []notifications.WebhookDelivery
TestEnhancedWebhook(notifications.EnhancedWebhookConfig) (int, string, error)
GetQueueStats() (map[string]int, error)
GetTelemetryStats(time.Time) (notifications.TelemetryStats, error)
GetDeliveryLog(time.Time, int) ([]notifications.DeliveryLogEntry, error)
IsEnabled() bool
}
// NotificationConfigPersistence defines the interface for saving notification configuration.
type NotificationConfigPersistence interface {
SaveEmailConfig(notifications.EmailConfig) error
SaveAppriseConfig(notifications.AppriseConfig) error
SaveWebhooks([]notifications.WebhookConfig) error
IsEncryptionEnabled() bool
}
// NotificationMonitor defines the interface for monitoring operations used by notification handlers.
type NotificationMonitor interface {
GetNotificationManager() NotificationManager
GetConfigPersistence() NotificationConfigPersistence
}
// NotificationHandlers handles notification-related HTTP endpoints
type NotificationHandlers struct {
stateMu sync.RWMutex
configMu sync.Mutex
mtMonitor *monitoring.MultiTenantMonitor
defaultMonitor NotificationMonitor
readState unifiedresources.ReadState
}
// NewNotificationHandlers creates new notification handlers
func NewNotificationHandlers(mtm *monitoring.MultiTenantMonitor, monitor NotificationMonitor) *NotificationHandlers {
// If mtm is provided, try to populate defaultMonitor from "default" org if not provided.
if monitor == nil && mtm != nil {
if m, err := mtm.GetMonitor("default"); err == nil {
monitor = NewNotificationMonitorWrapper(m)
}
}
return &NotificationHandlers{
mtMonitor: mtm,
defaultMonitor: monitor,
}
}
// SetMonitor updates the monitor reference for notification handlers.
func (h *NotificationHandlers) SetMonitor(m NotificationMonitor) {
h.stateMu.Lock()
defer h.stateMu.Unlock()
h.defaultMonitor = m
}
// SetMultiTenantMonitor updates the multi-tenant monitor reference
func (h *NotificationHandlers) SetMultiTenantMonitor(mtm *monitoring.MultiTenantMonitor) {
var defaultMonitor NotificationMonitor
if mtm != nil {
if m, err := mtm.GetMonitor("default"); err == nil {
defaultMonitor = NewNotificationMonitorWrapper(m)
}
}
h.stateMu.Lock()
defer h.stateMu.Unlock()
h.mtMonitor = mtm
if defaultMonitor != nil {
h.defaultMonitor = defaultMonitor
}
}
// SetReadState updates the ReadState reference for notification handlers.
func (h *NotificationHandlers) SetReadState(rs unifiedresources.ReadState) {
h.stateMu.Lock()
defer h.stateMu.Unlock()
h.readState = rs
}
func (h *NotificationHandlers) getMonitor(ctx context.Context) NotificationMonitor {
h.stateMu.RLock()
mtMonitor := h.mtMonitor
defaultMonitor := h.defaultMonitor
h.stateMu.RUnlock()
orgID := apicontext.OrgID(ctx)
if mtMonitor != nil {
if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
return NewNotificationMonitorWrapper(m)
}
}
return defaultMonitor
}
// MonitorForContext resolves the tenant-scoped notification runtime used by
// handlers and extensions.
func (h *NotificationHandlers) MonitorForContext(ctx context.Context) NotificationMonitor {
return h.getMonitor(ctx)
}
// getReadState returns the tenant-scoped ReadState for the request context.
// It resolves the correct monitor for the org and returns its ReadState.
// Falls back to the handler-level readState only for default/empty org.
// Non-default orgs fail closed (return nil) if tenant ReadState is unavailable.
func (h *NotificationHandlers) getReadState(ctx context.Context) unifiedresources.ReadState {
h.stateMu.RLock()
mtMonitor := h.mtMonitor
fallback := h.readState
h.stateMu.RUnlock()
orgID := apicontext.OrgID(ctx)
if mtMonitor != nil {
if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
if rs := m.GetUnifiedReadState(); rs != nil {
return rs
}
}
// Fail closed for non-default orgs: never leak default-org metadata.
if orgID != "" && orgID != "default" {
return nil
}
}
return fallback
}
// GetEmailConfig returns the current email configuration
func (h *NotificationHandlers) GetEmailConfig(w http.ResponseWriter, r *http.Request) {
config := h.getMonitor(r.Context()).GetNotificationManager().GetEmailConfig()
// For security, don't return the password
config.Password = ""
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(config)
}
// UpdateEmailConfig updates the email configuration
func (h *NotificationHandlers) UpdateEmailConfig(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Limit request body to 32KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 32*1024)
// Read raw body for debugging
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// NEVER log the body as it contains passwords
log.Info().
Msg("Received email config update")
// Parse strict subset to check for presence of fields
var presenceCheck struct {
RateLimit *int `json:"rateLimit"`
TagFilter *[]string `json:"tagFilter"`
TagMode *string `json:"tagFilterMode"`
MinimumSeverity *string `json:"minimumSeverity"`
}
if err := json.Unmarshal(body, &presenceCheck); err != nil {
// Non-fatal, just means we can't do presence check
}
var config notifications.EmailConfig
if err := json.Unmarshal(body, &config); err != nil {
log.Error().Err(err).Msg("Failed to parse email config") // Don't log body with passwords
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
monitor := h.getMonitor(r.Context())
manager := monitor.GetNotificationManager()
existingConfig := manager.GetEmailConfig()
// If password is empty, preserve the existing password
if config.Password == "" {
config.Password = existingConfig.Password
}
// If rateLimit was NOT provided (nil in presence check), preserve existing
if presenceCheck.RateLimit == nil {
config.RateLimit = existingConfig.RateLimit
}
if presenceCheck.TagFilter == nil {
config.TagFilter = existingConfig.TagFilter
}
if presenceCheck.TagMode == nil {
config.TagMode = existingConfig.TagMode
}
if presenceCheck.MinimumSeverity == nil {
config.MinimumSeverity = existingConfig.MinimumSeverity
}
log.Info().
Bool("enabled", config.Enabled).
Str("smtp", config.SMTPHost).
Str("from", config.From).
Int("toCount", len(config.To)).
Bool("hasPassword", config.Password != "").
Int("rateLimit", config.RateLimit).
Msg("Parsed email config")
// Durable state is the publication boundary. Applying the live config first
// can cancel queued deliveries or change routing even when the update cannot
// survive a restart.
if err := monitor.GetConfigPersistence().SaveEmailConfig(config); err != nil {
log.Error().Err(err).Msg("Failed to save email configuration")
http.Error(w, "failed to save email configuration", http.StatusInternalServerError)
return
}
manager.SetEmailConfig(config)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]string{"status": "success"})
}
// appriseConfigResponse is the API projection of an AppriseConfig. The stored
// API key never leaves the server; callers only learn whether one is saved,
// matching how the email handler blanks the SMTP password.
type appriseConfigResponse struct {
notifications.AppriseConfig
HasAPIKey bool `json:"hasApiKey"`
}
func redactAppriseConfig(config notifications.AppriseConfig) appriseConfigResponse {
response := appriseConfigResponse{AppriseConfig: config, HasAPIKey: config.APIKey != ""}
response.APIKey = ""
return response
}
// GetAppriseConfig returns the current Apprise configuration.
func (h *NotificationHandlers) GetAppriseConfig(w http.ResponseWriter, r *http.Request) {
config := h.getMonitor(r.Context()).GetNotificationManager().GetAppriseConfig()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(redactAppriseConfig(config)); err != nil {
log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
}
}
// UpdateAppriseConfig updates the Apprise configuration.
func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Limit request body to 64KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var config notifications.AppriseConfig
if err := json.Unmarshal(body, &config); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var presenceCheck struct {
MinimumSeverity *string `json:"minimumSeverity"`
}
_ = json.Unmarshal(body, &presenceCheck)
monitor := h.getMonitor(r.Context())
manager := monitor.GetNotificationManager()
existingConfig := manager.GetAppriseConfig()
// An empty API key means "keep the saved key": responses never include the
// stored value, so the settings form cannot round-trip it.
if config.APIKey == "" {
config.APIKey = existingConfig.APIKey
}
if presenceCheck.MinimumSeverity == nil {
config.MinimumSeverity = existingConfig.MinimumSeverity
}
diagnosticConfig := notifications.NormalizeAppriseConfig(config)
log.Info().
Bool("enabled", diagnosticConfig.Enabled).
Str("mode", string(diagnosticConfig.Mode)).
Int("targetCount", len(diagnosticConfig.Targets)).
Bool("hasServerURL", diagnosticConfig.ServerURL != "").
Bool("hasConfigKey", diagnosticConfig.ConfigKey != "").
Bool("hasApiKey", diagnosticConfig.APIKey != "").
Bool("skipTlsVerify", diagnosticConfig.SkipTLSVerify).
Int("timeoutSeconds", diagnosticConfig.TimeoutSeconds).
Msg("Parsed Apprise configuration update")
if err := monitor.GetConfigPersistence().SaveAppriseConfig(config); err != nil {
log.Error().Err(err).Msg("Failed to save Apprise configuration")
http.Error(w, "failed to save Apprise configuration", http.StatusInternalServerError)
return
}
manager.SetAppriseConfig(config)
normalized := manager.GetAppriseConfig()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(redactAppriseConfig(normalized)); err != nil {
log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
}
}
// GetWebhooks returns all webhook configurations with secrets masked
func (h *NotificationHandlers) GetWebhooks(w http.ResponseWriter, r *http.Request) {
webhooks := h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks()
maskedWebhooks := make([]map[string]interface{}, len(webhooks))
for i, webhook := range webhooks {
maskedWebhooks[i] = maskedWebhookResponse(webhook)
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(maskedWebhooks)
}
// maskedWebhookSecret stands in for every stored header, custom field and
// signing secret value the webhook API returns. A request that sends it back
// for a saved webhook means "keep the saved value"; see
// restoreMaskedWebhookSecrets.
const maskedWebhookSecret = "***REDACTED***"
// maskedWebhookResponse is the single response shape for webhook list, create
// and update: header and custom field values and the signing secret are
// masked, so no response returns a stored secret.
func maskedWebhookResponse(webhook notifications.WebhookConfig) map[string]interface{} {
whMap := map[string]interface{}{
"id": webhook.ID,
"name": webhook.Name,
"url": webhook.URL,
"method": webhook.Method,
"enabled": webhook.Enabled,
"service": webhook.Service,
}
// Mask headers - only show keys, not values
if len(webhook.Headers) > 0 {
maskedHeaders := make(map[string]string)
for key := range webhook.Headers {
maskedHeaders[key] = maskedWebhookSecret
}
whMap["headers"] = maskedHeaders
}
// Mask custom fields - only show keys, not values
if len(webhook.CustomFields) > 0 {
maskedFields := make(map[string]string)
for key := range webhook.CustomFields {
maskedFields[key] = maskedWebhookSecret
}
whMap["customFields"] = maskedFields
}
// Include template if present
if webhook.Template != "" {
whMap["template"] = webhook.Template
}
// Include the configured mention so the UI can render it after reload (#1118)
if webhook.Mention != "" {
whMap["mention"] = webhook.Mention
}
if len(webhook.TagFilter) > 0 {
whMap["tagFilter"] = append([]string(nil), webhook.TagFilter...)
whMap["tagFilterMode"] = webhook.TagMode
}
whMap["minimumSeverity"] = webhook.MinimumSeverity
// Signal that a signing secret is configured without revealing it
if webhook.SigningSecret != "" {
whMap["signingSecret"] = maskedWebhookSecret
}
return whMap
}
// restoreMaskedWebhookSecrets replaces each masked value in an incoming
// webhook with the saved value of the same header, custom field or signing
// secret. Restoring per key keeps any value the user typed in the same edit;
// a masked key with no saved value is dropped, so the mask is never stored or
// sent as a real value.
func restoreMaskedWebhookSecrets(incoming, saved notifications.WebhookConfig) notifications.WebhookConfig {
incoming.Headers = restoreMaskedWebhookValues(incoming.Headers, saved.Headers)
incoming.CustomFields = restoreMaskedWebhookValues(incoming.CustomFields, saved.CustomFields)
if incoming.SigningSecret == maskedWebhookSecret {
incoming.SigningSecret = saved.SigningSecret
}
return incoming
}
func restoreMaskedWebhookValues(incoming, saved map[string]string) map[string]string {
if len(incoming) == 0 {
return incoming
}
restored := make(map[string]string, len(incoming))
for key, value := range incoming {
if value != maskedWebhookSecret {
restored[key] = value
continue
}
if savedValue, ok := saved[key]; ok {
restored[key] = savedValue
}
}
return restored
}
// stripMaskedWebhookSecrets drops masked values from a webhook that has no
// saved counterpart, such as a create or an unsaved form test.
func stripMaskedWebhookSecrets(webhook notifications.WebhookConfig) notifications.WebhookConfig {
return restoreMaskedWebhookSecrets(webhook, notifications.WebhookConfig{})
}
// CreateWebhook creates a new webhook
func (h *NotificationHandlers) CreateWebhook(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Limit request body to 64KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
// Read the raw body to preserve all fields
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var webhook notifications.WebhookConfig
if err := json.Unmarshal(bodyBytes, &webhook); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
webhook = notifications.NormalizeWebhookConfig(stripMaskedWebhookSecrets(webhook))
monitor := h.getMonitor(r.Context())
manager := monitor.GetNotificationManager()
// Validate webhook URL
if err := manager.ValidateWebhookURL(webhook.URL); err != nil {
http.Error(w, fmt.Sprintf("Invalid webhook URL: %v", err), http.StatusBadRequest)
return
}
// Generate ID if not provided
if webhook.ID == "" {
webhook.ID = utils.GenerateID("webhook")
}
// Persist the complete candidate inventory before exposing the destination
// to live alert delivery.
webhooks := append(manager.GetWebhooks(), webhook)
if err := monitor.GetConfigPersistence().SaveWebhooks(webhooks); err != nil {
log.Error().Err(err).Msg("Failed to save webhooks")
http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError)
return
}
manager.AddWebhook(webhook)
if err := utils.WriteJSONResponse(w, maskedWebhookResponse(webhook)); err != nil {
log.Error().Err(err).Msg("Failed to write webhook creation response")
}
}
// UpdateWebhook updates an existing webhook
func (h *NotificationHandlers) UpdateWebhook(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Extract webhook ID from URL path
// Path is like /api/notifications/webhooks/{id} after routing
path := strings.TrimPrefix(r.URL.Path, "/api/notifications/webhooks/")
webhookID := path
if webhookID == "" {
http.Error(w, "Invalid URL - missing webhook ID", http.StatusBadRequest)
return
}
// Limit request body to 64KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
// Read the raw body to preserve all fields
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var webhook notifications.WebhookConfig
if err := json.Unmarshal(bodyBytes, &webhook); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var routingPresence struct {
TagFilter *[]string `json:"tagFilter"`
TagMode *string `json:"tagFilterMode"`
MinimumSeverity *string `json:"minimumSeverity"`
}
_ = json.Unmarshal(bodyBytes, &routingPresence)
monitor := h.getMonitor(r.Context())
manager := monitor.GetNotificationManager()
// The edit form sends back the masked values GetWebhooks returned for
// anything the user did not retype; those keep their saved values.
existingWebhooks := manager.GetWebhooks()
found := false
for _, existing := range existingWebhooks {
if existing.ID == webhookID {
found = true
if routingPresence.TagFilter == nil {
webhook.TagFilter = existing.TagFilter
}
if routingPresence.TagMode == nil {
webhook.TagMode = existing.TagMode
}
if routingPresence.MinimumSeverity == nil {
webhook.MinimumSeverity = existing.MinimumSeverity
}
webhook = restoreMaskedWebhookSecrets(webhook, existing)
break
}
}
if !found {
http.Error(w, fmt.Sprintf("webhook not found: %s", webhookID), http.StatusNotFound)
return
}
webhook = notifications.NormalizeWebhookConfig(webhook)
// Validate webhook URL
if err := manager.ValidateWebhookURL(webhook.URL); err != nil {
http.Error(w, fmt.Sprintf("Invalid webhook URL: %v", err), http.StatusBadRequest)
return
}
webhook.ID = webhookID
candidateWebhooks := make([]notifications.WebhookConfig, len(existingWebhooks))
copy(candidateWebhooks, existingWebhooks)
for i := range candidateWebhooks {
if candidateWebhooks[i].ID == webhookID {
candidateWebhooks[i] = webhook
break
}
}
if err := monitor.GetConfigPersistence().SaveWebhooks(candidateWebhooks); err != nil {
log.Error().Err(err).Msg("Failed to save webhooks")
http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError)
return
}
if err := manager.UpdateWebhook(webhookID, webhook); err != nil {
// The handler serializes destination writes, so this indicates an
// out-of-band mutation. Restore the prior durable inventory rather than
// leave restart-time state ahead of the live manager.
if rollbackErr := monitor.GetConfigPersistence().SaveWebhooks(existingWebhooks); rollbackErr != nil {
log.Error().Err(rollbackErr).Msg("Failed to roll back webhook configuration")
}
http.Error(w, "failed to publish webhook configuration", http.StatusInternalServerError)
return
}
if err := utils.WriteJSONResponse(w, maskedWebhookResponse(webhook)); err != nil {
log.Error().Err(err).Str("webhookID", webhookID).Msg("Failed to write webhook update response")
}
}
// DeleteWebhook deletes a webhook
func (h *NotificationHandlers) DeleteWebhook(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Extract webhook ID from URL path
// Path is like /api/notifications/webhooks/{id} after routing
path := strings.TrimPrefix(r.URL.Path, "/api/notifications/webhooks/")
webhookID := path
if webhookID == "" {
http.Error(w, "Invalid URL - missing webhook ID", http.StatusBadRequest)
return
}
monitor := h.getMonitor(r.Context())
manager := monitor.GetNotificationManager()
existingWebhooks := manager.GetWebhooks()
candidateWebhooks := make([]notifications.WebhookConfig, 0, len(existingWebhooks))
found := false
for _, webhook := range existingWebhooks {
if webhook.ID == webhookID {
found = true
continue
}
candidateWebhooks = append(candidateWebhooks, webhook)
}
if !found {
http.Error(w, fmt.Sprintf("webhook not found: %s", webhookID), http.StatusNotFound)
return
}
if err := monitor.GetConfigPersistence().SaveWebhooks(candidateWebhooks); err != nil {
log.Error().Err(err).Msg("Failed to save webhooks")
http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError)
return
}
if err := manager.DeleteWebhook(webhookID); err != nil {
if rollbackErr := monitor.GetConfigPersistence().SaveWebhooks(existingWebhooks); rollbackErr != nil {
log.Error().Err(rollbackErr).Msg("Failed to roll back webhook configuration")
}
http.Error(w, "failed to publish webhook configuration", http.StatusInternalServerError)
return
}
if err := utils.WriteJSONResponse(w, map[string]string{"status": "success"}); err != nil {
log.Error().Err(err).Str("webhookID", webhookID).Msg("Failed to write webhook deletion response")
}
}
// classifyNotificationError maps raw Go errors to user-friendly summaries.
// Returns a user-facing message and the original error string as detail.
func classifyNotificationError(err error) (summary, detail string) {
raw := err.Error()
detail = raw
lower := strings.ToLower(raw)
switch {
case strings.Contains(lower, "connection refused"):
summary = "Could not connect to the server — check host, port, and firewall settings"
case strings.Contains(lower, "no such host"):
summary = "Server hostname not found — check the server address"
case strings.Contains(lower, "i/o timeout") || strings.Contains(lower, "deadline exceeded"):
summary = "Connection timed out — the server may be unreachable or the port may be blocked"
case strings.Contains(lower, "x509:") || strings.Contains(lower, "certificate"):
summary = "TLS certificate error — check certificate settings or try enabling 'Skip TLS Verify'"
case strings.Contains(lower, "535") || strings.Contains(lower, "authentication"):
summary = "Authentication failed — check username and password"
case strings.Contains(lower, "executable file not found"):
summary = "Required program not found — ensure it is installed on the server"
case strings.Contains(lower, "permission denied"):
summary = "Permission denied — the server process lacks access to the required resource"
case strings.Contains(lower, "eof"):
summary = "The server closed the connection unexpectedly — check if TLS/StartTLS settings are correct"
default:
summary = raw
detail = ""
}
return summary, detail
}
// writeTestNotificationError writes a JSON error response for test notification failures.
func writeTestNotificationError(w http.ResponseWriter, err error, statusCode int) {
summary, detail := classifyNotificationError(err)
resp := map[string]string{"error": summary}
if detail != "" {
resp["detail"] = detail
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(statusCode)
json.NewEncoder(w).Encode(resp)
}
// TestNotification sends a test notification
func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, notificationTestRequestBodyLimit)
// Read body for debugging
body, err := io.ReadAll(r.Body)
if err != nil {
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
http.Error(w, "Request body too large", http.StatusRequestEntityTooLarge)
return
}
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// NEVER log the body as it contains passwords
log.Info().
Msg("Test notification request received")
var req struct {
Method string `json:"method"` // "email", "webhook", or "apprise"
Type string `json:"type"` // Alternative field name used by frontend
Config json.RawMessage `json:"config,omitempty"` // Optional config for testing (email or apprise)
WebhookID string `json:"webhookId,omitempty"` // For webhook testing
}
if err := json.Unmarshal(body, &req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Support both "method" and "type" field names
if req.Method == "" && req.Type != "" {
req.Method = req.Type
}
// Get actual node info from ReadState (unified resources registry)
var nodeInfo *notifications.TestNodeInfo
if rs := h.getReadState(r.Context()); rs != nil {
if nodes := rs.Nodes(); len(nodes) > 0 {
nodeInfo = &notifications.TestNodeInfo{
NodeName: nodes[0].Name(),
InstanceURL: nodes[0].Instance(),
}
}
}
// Handle webhook testing
if req.Method == "webhook" && req.WebhookID != "" {
log.Info().
Str("webhookId", req.WebhookID).
Msg("Testing specific webhook")
// Get the webhook by ID and test it
webhooks := h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks()
var foundWebhook *notifications.WebhookConfig
for _, wh := range webhooks {
if wh.ID == req.WebhookID {
foundWebhook = &wh
break
}
}
if foundWebhook == nil {
http.Error(w, "Webhook not found", http.StatusNotFound)
return
}
// Send test webhook
if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestWebhook(*foundWebhook); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
} else if req.Method == "email" && len(req.Config) > 0 {
var emailConfig notifications.EmailConfig
if err := json.Unmarshal(req.Config, &emailConfig); err != nil {
http.Error(w, fmt.Sprintf("Invalid email config: %v", err), http.StatusBadRequest)
return
}
// If password is empty, use the saved password
if emailConfig.Password == "" {
savedConfig := h.getMonitor(r.Context()).GetNotificationManager().GetEmailConfig()
emailConfig.Password = savedConfig.Password
}
log.Info().
Bool("enabled", emailConfig.Enabled).
Str("smtp", emailConfig.SMTPHost).
Str("from", emailConfig.From).
Int("toCount", len(emailConfig.To)).
Strs("to", emailConfig.To).
Bool("hasPassword", emailConfig.Password != "").
Msg("Testing email with provided config")
if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestNotificationWithConfig(req.Method, &emailConfig, nodeInfo); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
} else if req.Method == "apprise" && len(req.Config) > 0 {
var appriseConfig notifications.AppriseConfig
if err := json.Unmarshal(req.Config, &appriseConfig); err != nil {
http.Error(w, fmt.Sprintf("Invalid Apprise config: %v", err), http.StatusBadRequest)
return
}
// If the API key is empty, use the saved key
if appriseConfig.APIKey == "" {
savedConfig := h.getMonitor(r.Context()).GetNotificationManager().GetAppriseConfig()
appriseConfig.APIKey = savedConfig.APIKey
}
if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestAppriseWithConfig(appriseConfig); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
} else {
// Use saved config
if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestNotification(req.Method); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
}
// A test send bypasses the activation gate real alerts honor, so a bare
// success here is exactly how installs end up believing delivery works
// while every real alert is suppressed. Say so in the result itself.
response := map[string]interface{}{
"status": "success",
"message": "Test notification sent",
}
if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() {
response["deliveryPaused"] = true
response["message"] = "Test notification sent, but alert delivery is paused: real alerts are not being sent"
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
// GetWebhookTemplates returns available webhook templates
func (h *NotificationHandlers) GetWebhookTemplates(w http.ResponseWriter, r *http.Request) {
templates := notifications.GetWebhookTemplates()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(templates)
}
// GetWebhookHistory returns recent webhook delivery history with URLs redacted
func (h *NotificationHandlers) GetWebhookHistory(w http.ResponseWriter, r *http.Request) {
history := h.getMonitor(r.Context()).GetNotificationManager().GetWebhookHistory()
// Redact secrets from URLs in history
for i := range history {
history[i].WebhookURL = redactSecretsFromURL(history[i].WebhookURL)
// Note: ResponseBody is not stored in WebhookDelivery struct
// Error messages are already limited in length
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(history)
}
// redactSecretsFromURL is retained as the API package boundary for delivery
// history while the canonical redaction policy lives with webhook execution.
func redactSecretsFromURL(urlStr string) string {
return notifications.RedactWebhookURLSecrets(urlStr)
}
// GetEmailProviders returns available email providers
func (h *NotificationHandlers) GetEmailProviders(w http.ResponseWriter, r *http.Request) {
providers := notifications.GetEmailProviders()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(providers)
}
// TestWebhook tests a webhook configuration
func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Request) {
// First try to decode as basic webhook config
var basicWebhook notifications.WebhookConfig
r.Body = http.MaxBytesReader(w, r.Body, webhookTestRequestBodyLimit)
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
http.Error(w, "Request body too large", http.StatusRequestEntityTooLarge)
return
}
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := json.Unmarshal(bodyBytes, &basicWebhook); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Testing an edit of a saved webhook sends the form, whose untouched
// secrets are the masked values from GetWebhooks. Test with the saved
// values, as the saved webhook would send; this route requires the same
// settings:write scope as UpdateWebhook, which keeps them the same way.
saved := notifications.WebhookConfig{}
if basicWebhook.ID != "" {
for _, candidate := range h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks() {
if candidate.ID == basicWebhook.ID {
saved = candidate
break
}
}
}
basicWebhook = notifications.NormalizeWebhookConfig(restoreMaskedWebhookSecrets(basicWebhook, saved))
// Try to extract service from body if present
var serviceCheck struct {
Service string `json:"service"`
}
if err := json.Unmarshal(bodyBytes, &serviceCheck); err == nil && serviceCheck.Service != "" {
basicWebhook.Service = serviceCheck.Service
}
webhook := notifications.BuildEnhancedWebhookTestConfig(basicWebhook, serviceCheck.Service)
log.Info().
Str("service", webhook.Service).
Str("url", notifications.RedactWebhookURLSecrets(webhook.URL)).
Str("name", webhook.Name).
Msg("Testing webhook")
// Test the webhook
status, response, err := h.getMonitor(r.Context()).GetNotificationManager().TestEnhancedWebhook(webhook)
result := map[string]interface{}{
"status": status,
"response": response,
}
if err != nil {
summary, detail := classifyNotificationError(err)
result["error"] = summary
if detail != "" {
result["detail"] = detail
}
w.WriteHeader(http.StatusBadRequest)
} else if status < 200 || status >= 300 {
// HTTP error from webhook endpoint
result["error"] = fmt.Sprintf("Webhook returned HTTP %d: %s", status, response)
result["success"] = false
w.WriteHeader(http.StatusBadRequest)
} else {
result["success"] = true
// Same honesty as TestNotification: a webhook test bypasses the
// activation gate, so its success must not imply live alerts flow.
if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() {
result["deliveryPaused"] = true
}
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(result)
}
// GetNotificationHealth returns health status of notification system
func (h *NotificationHandlers) GetNotificationHealth(w http.ResponseWriter, r *http.Request) {
monitor := h.getMonitor(r.Context())
manager := monitor.GetNotificationManager()
failureClasses := notifications.NotificationFailureClassCounts{}.AsMap()
failureClassesAvailable := false
if deliveryStats, statsErr := manager.GetTelemetryStats(
time.Now().Add(-completedQueueRetentionDays * 24 * time.Hour),
); statsErr != nil {
log.Warn().Err(statsErr).Msg("Failed to get notification failure classes for health check")
} else {
failureClasses = deliveryStats.FailureClasses.AsMap()
failureClassesAvailable = true
}
queueStats := make(map[string]interface{})
stats, err := manager.GetQueueStats()
if err != nil {
log.Warn().Err(err).Msg("Failed to get queue stats for health check")
queueStats = map[string]interface{}{
"healthy": false,
"status": "unavailable",
"attention_required": 0,
"reason_codes": []string{"queue_stats_unavailable"},
"completed_retention_days": completedQueueRetentionDays,
"dead_letter_retention_days": deadLetterQueueRetentionDays,
"counts_are_retention_bounded": true,
"retry_attempts_affect_health": false,
"terminal_failures_affect_health": true,
"failure_classes_7d": failureClasses,
"failure_classes_available": failureClassesAvailable,
"failure_class_window_days": completedQueueRetentionDays,
}
} else {
healthy, attentionRequired, reasonCodes := classifyNotificationQueueHealth(stats)
status := "healthy"
if !healthy {
status = "degraded"
}
queueStats = map[string]interface{}{
"pending": stats["pending"],
"sending": stats["sending"],
"sent": stats["sent"],
"failed": stats["failed"],
"dlq": stats["dlq"],
"healthy": healthy,
"status": status,
"attention_required": attentionRequired,
"reason_codes": reasonCodes,
"completed_retention_days": completedQueueRetentionDays,
"dead_letter_retention_days": deadLetterQueueRetentionDays,
"counts_are_retention_bounded": true,
"retry_attempts_affect_health": false,
"terminal_failures_affect_health": true,
"failure_classes_7d": failureClasses,
"failure_classes_available": failureClassesAvailable,
"failure_class_window_days": completedQueueRetentionDays,
}
}
// Get config status
emailCfg := manager.GetEmailConfig()
webhooks := manager.GetWebhooks()
health := map[string]interface{}{
"queue": queueStats,
"email": map[string]interface{}{
"enabled": emailCfg.Enabled,
"configured": emailCfg.SMTPHost != "",
},
"webhooks": map[string]interface{}{
"total": len(webhooks),
"enabled": countEnabledWebhooks(webhooks),
},
"encryption": map[string]interface{}{
"enabled": monitor.GetConfigPersistence().IsEncryptionEnabled(),
},
"overall_healthy": queueStats["healthy"] == true,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(health)
}
// GetDeliveryLog returns recent recorded delivery attempts, newest first.
// This is the per-attempt evidence behind the aggregate health verdict: what
// fired, which destination it went to, and what happened. Entries share the
// queue's retention windows, so the payload names the window rather than
// presenting itself as lifetime history.
func (h *NotificationHandlers) GetDeliveryLog(w http.ResponseWriter, r *http.Request) {
manager := h.getMonitor(r.Context()).GetNotificationManager()
limit := 0
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
parsed, err := strconv.Atoi(raw)
if err != nil || parsed <= 0 {
http.Error(w, "limit must be a positive integer", http.StatusBadRequest)
return
}
limit = parsed
}
entries, err := manager.GetDeliveryLog(
time.Now().Add(-deadLetterQueueRetentionDays*24*time.Hour), limit,
)
if err != nil {
log.Warn().Err(err).Msg("Failed to read notification delivery log")
http.Error(w, "Delivery log unavailable", http.StatusServiceUnavailable)
return
}
// Webhook failure text can embed the destination URL, and destination URLs
// can embed credentials.
for i := range entries {
if entries[i].ErrorMessage != "" {
entries[i].ErrorMessage = notifications.RedactWebhookDiagnosticSecrets(entries[i].ErrorMessage)
}
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"entries": entries,
"window_days": deadLetterQueueRetentionDays,
"completed_retention_days": completedQueueRetentionDays,
"dead_letter_retention_days": deadLetterQueueRetentionDays,
"entries_are_retention_bounded": true,
})
}
// classifyNotificationQueueHealth delegates to the canonical rule in
// internal/notifications so this endpoint and the monitoring loop that raises
// the notification-delivery system alert cannot drift apart.
func classifyNotificationQueueHealth(stats map[string]int) (bool, int, []string) {
health := notifications.ClassifyQueueHealth(stats)
return health.Healthy, health.AttentionRequired, health.ReasonCodes
}
func countEnabledWebhooks(webhooks []notifications.WebhookConfig) int {
count := 0
for _, wh := range webhooks {
if wh.Enabled {
count++
}
}
return count
}
// HandleNotifications routes notification requests to appropriate handlers
func (h *NotificationHandlers) HandleNotifications(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, "/api/notifications")
requireAnyScope := func(required string, scopes ...string) bool {
record := internalauth.GetAPIToken(r.Context())
if record == nil {
return true
}
for _, scope := range scopes {
if scope != "" && record.HasScope(scope) {
return true
}
}
apihttp.RespondMissingScope(w, required)
return false
}
switch {
case path == "/email" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetEmailConfig(w, r)
case path == "/email" && r.Method == http.MethodPut:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.UpdateEmailConfig(w, r)
case path == "/apprise" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetAppriseConfig(w, r)
case path == "/apprise" && r.Method == http.MethodPut:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.UpdateAppriseConfig(w, r)
case path == "/webhooks" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetWebhooks(w, r)
case path == "/webhooks" && r.Method == http.MethodPost:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.CreateWebhook(w, r)
case path == "/webhooks/test" && r.Method == http.MethodPost:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.TestWebhook(w, r)
case strings.HasPrefix(path, "/webhooks/") && r.Method == http.MethodPut:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.UpdateWebhook(w, r)
case strings.HasPrefix(path, "/webhooks/") && r.Method == http.MethodDelete:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.DeleteWebhook(w, r)
case path == "/webhook-templates" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetWebhookTemplates(w, r)
case path == "/webhook-history" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetWebhookHistory(w, r)
case path == "/email-providers" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetEmailProviders(w, r)
case path == "/test" && r.Method == http.MethodPost:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.TestNotification(w, r)
case path == "/health" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetNotificationHealth(w, r)
case path == "/delivery-log" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetDeliveryLog(w, r)
default:
http.Error(w, "Not found", http.StatusNotFound)
}
}