Files
pulse/internal/api/alerting/notifications.go
T
rcourtman 94f96eb043 Stop tenant monitor lookups falling back to the default org
The agent, alert, notification and system settings handlers resolved
the request organization's monitor through the tenant manager and, when
that lookup failed for any reason, quietly used the default
organization's monitor instead. A request for another organization
could then list, change or remove the default organization's agents,
alert configuration, webhooks and email settings.

The router's tenant guard already answers 503 tenant_unavailable when a
non-default organization's monitor cannot be resolved, so normal
requests never reached that fallback. The handlers still used it when
a lookup failed after the guard had passed (an organization being
deleted mid-request) and for callers outside the guarded chain. These
routes are open to organization owners, so the handlers have to fail
closed on their own.

Each getMonitor now falls back to the primary monitor only for the
default organization. Any other organization whose monitor cannot be
resolved gets nil, and the HTTP handlers resolve the monitor once
through requireMonitor, which answers the same 503 tenant_unavailable
as the router guard. Default-organization and single-tenant behaviour is
unchanged.

The configuration handlers' getContextState already refused non-default
organizations and is covered by an existing test.

Contract-Neutral: Handlers now return the router tenant guard's existing 503 tenant_unavailable response instead of the default org's data; no new API surface.
2026-10-17 10:14:21 +01:00

1334 lines
45 KiB
Go

package alerting
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext"
"github.com/rcourtman/pulse-go-rewrite/internal/api/apihttp"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
internalauth "github.com/rcourtman/pulse-go-rewrite/pkg/auth"
"github.com/rs/zerolog/log"
)
const (
notificationTestRequestBodyLimit = 64 * 1024
webhookTestRequestBodyLimit = 64 * 1024
completedQueueRetentionDays = 7
deadLetterQueueRetentionDays = 30
)
// NotificationManager defines the interface for notification management operations.
type NotificationManager interface {
GetEmailConfig() notifications.EmailConfig
SetEmailConfig(notifications.EmailConfig)
GetAppriseConfig() notifications.AppriseConfig
SetAppriseConfig(notifications.AppriseConfig)
GetWebhooks() []notifications.WebhookConfig
ValidateWebhookURL(string) error
AddWebhook(notifications.WebhookConfig)
UpdateWebhook(string, notifications.WebhookConfig) error
DeleteWebhook(string) error
SendTestWebhook(notifications.WebhookConfig) error
SendTestNotificationWithConfig(string, *notifications.EmailConfig, *notifications.TestNodeInfo) error
SendTestAppriseWithConfig(notifications.AppriseConfig) error
SendTestNotification(string) error
GetWebhookHistory() []notifications.WebhookDelivery
TestEnhancedWebhook(notifications.EnhancedWebhookConfig) (int, string, error)
GetQueueStats() (map[string]int, error)
GetTelemetryStats(time.Time) (notifications.TelemetryStats, error)
GetDeliveryLog(time.Time, int) ([]notifications.DeliveryLogEntry, error)
IsEnabled() bool
}
// NotificationConfigPersistence defines the interface for saving notification configuration.
type NotificationConfigPersistence interface {
SaveEmailConfig(notifications.EmailConfig) error
SaveAppriseConfig(notifications.AppriseConfig) error
SaveWebhooks([]notifications.WebhookConfig) error
IsEncryptionEnabled() bool
}
// NotificationMonitor defines the interface for monitoring operations used by notification handlers.
type NotificationMonitor interface {
GetNotificationManager() NotificationManager
GetConfigPersistence() NotificationConfigPersistence
}
// NotificationHandlers handles notification-related HTTP endpoints
type NotificationHandlers struct {
stateMu sync.RWMutex
configMu sync.Mutex
mtMonitor *monitoring.MultiTenantMonitor
defaultMonitor NotificationMonitor
readState unifiedresources.ReadState
}
// NewNotificationHandlers creates new notification handlers
func NewNotificationHandlers(mtm *monitoring.MultiTenantMonitor, monitor NotificationMonitor) *NotificationHandlers {
// If mtm is provided, try to populate defaultMonitor from "default" org if not provided.
if monitor == nil && mtm != nil {
if m, err := mtm.GetMonitor("default"); err == nil {
monitor = NewNotificationMonitorWrapper(m)
}
}
return &NotificationHandlers{
mtMonitor: mtm,
defaultMonitor: monitor,
}
}
// SetMonitor updates the monitor reference for notification handlers.
func (h *NotificationHandlers) SetMonitor(m NotificationMonitor) {
h.stateMu.Lock()
defer h.stateMu.Unlock()
h.defaultMonitor = m
}
// SetMultiTenantMonitor updates the multi-tenant monitor reference
func (h *NotificationHandlers) SetMultiTenantMonitor(mtm *monitoring.MultiTenantMonitor) {
var defaultMonitor NotificationMonitor
if mtm != nil {
if m, err := mtm.GetMonitor("default"); err == nil {
defaultMonitor = NewNotificationMonitorWrapper(m)
}
}
h.stateMu.Lock()
defer h.stateMu.Unlock()
h.mtMonitor = mtm
if defaultMonitor != nil {
h.defaultMonitor = defaultMonitor
}
}
// SetReadState updates the ReadState reference for notification handlers.
func (h *NotificationHandlers) SetReadState(rs unifiedresources.ReadState) {
h.stateMu.Lock()
defer h.stateMu.Unlock()
h.readState = rs
}
func (h *NotificationHandlers) getMonitor(ctx context.Context) NotificationMonitor {
h.stateMu.RLock()
mtMonitor := h.mtMonitor
defaultMonitor := h.defaultMonitor
h.stateMu.RUnlock()
orgID := strings.TrimSpace(apicontext.OrgID(ctx))
if orgID == "" || orgID == "default" {
if mtMonitor != nil {
if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
return NewNotificationMonitorWrapper(m)
}
}
return defaultMonitor
}
// Never fall back to the default organization's runtime for another
// organization: its requests would read or change the wrong tenant.
if mtMonitor == nil {
log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for notification request")
return nil
}
m, err := mtMonitor.GetMonitor(orgID)
if err != nil || m == nil {
log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for notification request")
return nil
}
return NewNotificationMonitorWrapper(m)
}
// requireMonitor resolves the request organization's monitor, writing a 503
// and returning nil when it is unavailable.
func (h *NotificationHandlers) requireMonitor(w http.ResponseWriter, r *http.Request) NotificationMonitor {
monitor := h.getMonitor(r.Context())
if monitor == nil {
http.Error(w, "Tenant monitor is not available", http.StatusServiceUnavailable)
return nil
}
return monitor
}
// MonitorForContext resolves the tenant-scoped notification runtime used by
// handlers and extensions.
func (h *NotificationHandlers) MonitorForContext(ctx context.Context) NotificationMonitor {
return h.getMonitor(ctx)
}
// getReadState returns the tenant-scoped ReadState for the request context.
// It resolves the correct monitor for the org and returns its ReadState.
// Falls back to the handler-level readState only for default/empty org.
// Non-default orgs fail closed (return nil) if tenant ReadState is unavailable.
func (h *NotificationHandlers) getReadState(ctx context.Context) unifiedresources.ReadState {
h.stateMu.RLock()
mtMonitor := h.mtMonitor
fallback := h.readState
h.stateMu.RUnlock()
orgID := apicontext.OrgID(ctx)
if mtMonitor != nil {
if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
if rs := m.GetUnifiedReadState(); rs != nil {
return rs
}
}
// Fail closed for non-default orgs: never leak default-org metadata.
if orgID != "" && orgID != "default" {
return nil
}
}
return fallback
}
// GetEmailConfig returns the current email configuration
func (h *NotificationHandlers) GetEmailConfig(w http.ResponseWriter, r *http.Request) {
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
config := monitor.GetNotificationManager().GetEmailConfig()
// For security, don't return the password
config.Password = ""
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(config)
}
// UpdateEmailConfig updates the email configuration
func (h *NotificationHandlers) UpdateEmailConfig(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Limit request body to 32KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 32*1024)
// Read raw body for debugging
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// NEVER log the body as it contains passwords
log.Info().
Msg("Received email config update")
// Parse strict subset to check for presence of fields
var presenceCheck struct {
RateLimit *int `json:"rateLimit"`
TagFilter *[]string `json:"tagFilter"`
TagMode *string `json:"tagFilterMode"`
MinimumSeverity *string `json:"minimumSeverity"`
}
if err := json.Unmarshal(body, &presenceCheck); err != nil {
// Non-fatal, just means we can't do presence check
}
var config notifications.EmailConfig
if err := json.Unmarshal(body, &config); err != nil {
log.Error().Err(err).Msg("Failed to parse email config") // Don't log body with passwords
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
manager := monitor.GetNotificationManager()
existingConfig := manager.GetEmailConfig()
// If password is empty, preserve the existing password
if config.Password == "" {
config.Password = existingConfig.Password
}
// If rateLimit was NOT provided (nil in presence check), preserve existing
if presenceCheck.RateLimit == nil {
config.RateLimit = existingConfig.RateLimit
}
if presenceCheck.TagFilter == nil {
config.TagFilter = existingConfig.TagFilter
}
if presenceCheck.TagMode == nil {
config.TagMode = existingConfig.TagMode
}
if presenceCheck.MinimumSeverity == nil {
config.MinimumSeverity = existingConfig.MinimumSeverity
}
log.Info().
Bool("enabled", config.Enabled).
Str("smtp", config.SMTPHost).
Str("from", config.From).
Int("toCount", len(config.To)).
Bool("hasPassword", config.Password != "").
Int("rateLimit", config.RateLimit).
Msg("Parsed email config")
// Durable state is the publication boundary. Applying the live config first
// can cancel queued deliveries or change routing even when the update cannot
// survive a restart.
if err := monitor.GetConfigPersistence().SaveEmailConfig(config); err != nil {
log.Error().Err(err).Msg("Failed to save email configuration")
http.Error(w, "failed to save email configuration", http.StatusInternalServerError)
return
}
manager.SetEmailConfig(config)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]string{"status": "success"})
}
// appriseConfigResponse is the API projection of an AppriseConfig. The stored
// API key never leaves the server; callers only learn whether one is saved,
// matching how the email handler blanks the SMTP password.
type appriseConfigResponse struct {
notifications.AppriseConfig
HasAPIKey bool `json:"hasApiKey"`
}
func redactAppriseConfig(config notifications.AppriseConfig) appriseConfigResponse {
response := appriseConfigResponse{AppriseConfig: config, HasAPIKey: config.APIKey != ""}
response.APIKey = ""
return response
}
// GetAppriseConfig returns the current Apprise configuration.
func (h *NotificationHandlers) GetAppriseConfig(w http.ResponseWriter, r *http.Request) {
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
config := monitor.GetNotificationManager().GetAppriseConfig()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(redactAppriseConfig(config)); err != nil {
log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
}
}
// UpdateAppriseConfig updates the Apprise configuration.
func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Limit request body to 64KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var config notifications.AppriseConfig
if err := json.Unmarshal(body, &config); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var presenceCheck struct {
MinimumSeverity *string `json:"minimumSeverity"`
}
_ = json.Unmarshal(body, &presenceCheck)
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
manager := monitor.GetNotificationManager()
existingConfig := manager.GetAppriseConfig()
// An empty API key means "keep the saved key": responses never include the
// stored value, so the settings form cannot round-trip it.
if config.APIKey == "" {
config.APIKey = existingConfig.APIKey
}
if presenceCheck.MinimumSeverity == nil {
config.MinimumSeverity = existingConfig.MinimumSeverity
}
diagnosticConfig := notifications.NormalizeAppriseConfig(config)
log.Info().
Bool("enabled", diagnosticConfig.Enabled).
Str("mode", string(diagnosticConfig.Mode)).
Int("targetCount", len(diagnosticConfig.Targets)).
Bool("hasServerURL", diagnosticConfig.ServerURL != "").
Bool("hasConfigKey", diagnosticConfig.ConfigKey != "").
Bool("hasApiKey", diagnosticConfig.APIKey != "").
Bool("skipTlsVerify", diagnosticConfig.SkipTLSVerify).
Int("timeoutSeconds", diagnosticConfig.TimeoutSeconds).
Msg("Parsed Apprise configuration update")
if err := monitor.GetConfigPersistence().SaveAppriseConfig(config); err != nil {
log.Error().Err(err).Msg("Failed to save Apprise configuration")
http.Error(w, "failed to save Apprise configuration", http.StatusInternalServerError)
return
}
manager.SetAppriseConfig(config)
normalized := manager.GetAppriseConfig()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(redactAppriseConfig(normalized)); err != nil {
log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
}
}
// GetWebhooks returns all webhook configurations with secrets masked
func (h *NotificationHandlers) GetWebhooks(w http.ResponseWriter, r *http.Request) {
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
webhooks := monitor.GetNotificationManager().GetWebhooks()
maskedWebhooks := make([]map[string]interface{}, len(webhooks))
for i, webhook := range webhooks {
maskedWebhooks[i] = maskedWebhookResponse(webhook)
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(maskedWebhooks)
}
// maskedWebhookSecret stands in for every stored header, custom field and
// signing secret value the webhook API returns. A request that sends it back
// for a saved webhook means "keep the saved value"; see
// restoreMaskedWebhookSecrets.
const maskedWebhookSecret = "***REDACTED***"
// maskedWebhookResponse is the single response shape for webhook list, create
// and update: header and custom field values and the signing secret are
// masked, so no response returns a stored secret.
func maskedWebhookResponse(webhook notifications.WebhookConfig) map[string]interface{} {
whMap := map[string]interface{}{
"id": webhook.ID,
"name": webhook.Name,
"url": webhook.URL,
"method": webhook.Method,
"enabled": webhook.Enabled,
"service": webhook.Service,
}
// Mask headers - only show keys, not values
if len(webhook.Headers) > 0 {
maskedHeaders := make(map[string]string)
for key := range webhook.Headers {
maskedHeaders[key] = maskedWebhookSecret
}
whMap["headers"] = maskedHeaders
}
// Mask custom fields - only show keys, not values
if len(webhook.CustomFields) > 0 {
maskedFields := make(map[string]string)
for key := range webhook.CustomFields {
maskedFields[key] = maskedWebhookSecret
}
whMap["customFields"] = maskedFields
}
// Include template if present
if webhook.Template != "" {
whMap["template"] = webhook.Template
}
// Include the configured mention so the UI can render it after reload (#1118)
if webhook.Mention != "" {
whMap["mention"] = webhook.Mention
}
if len(webhook.TagFilter) > 0 {
whMap["tagFilter"] = append([]string(nil), webhook.TagFilter...)
whMap["tagFilterMode"] = webhook.TagMode
}
whMap["minimumSeverity"] = webhook.MinimumSeverity
// Signal that a signing secret is configured without revealing it
if webhook.SigningSecret != "" {
whMap["signingSecret"] = maskedWebhookSecret
}
return whMap
}
// restoreMaskedWebhookSecrets is the shared Create/Update/Test policy. A mask
// means keep this saved value, not discard the field or store the placeholder.
// Reject missing/ambiguous saved identities before persistence or any test send.
func restoreMaskedWebhookSecrets(incoming, saved notifications.WebhookConfig) (notifications.WebhookConfig, error) {
var err error
incoming.Headers, err = restoreStrictMaskedWebhookValues(incoming.Headers, saved.Headers, true)
if err != nil {
return notifications.WebhookConfig{}, err
}
incoming.CustomFields, err = restoreStrictMaskedWebhookValues(incoming.CustomFields, saved.CustomFields, false)
if err != nil {
return notifications.WebhookConfig{}, err
}
if incoming.SigningSecret == maskedWebhookSecret {
if saved.SigningSecret == "" || saved.SigningSecret == maskedWebhookSecret {
return notifications.WebhookConfig{}, fmt.Errorf("no saved value for masked signing secret")
}
incoming.SigningSecret = saved.SigningSecret
}
return notifications.NormalizeWebhookConfig(incoming), nil
}
// CreateWebhook creates a new webhook
func (h *NotificationHandlers) CreateWebhook(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Limit request body to 64KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
// Read the raw body to preserve all fields
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var webhook notifications.WebhookConfig
if err := json.Unmarshal(bodyBytes, &webhook); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
webhook, err = restoreMaskedWebhookSecrets(webhook, notifications.WebhookConfig{})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
manager := monitor.GetNotificationManager()
// Validate webhook URL
if err := manager.ValidateWebhookURL(webhook.URL); err != nil {
http.Error(w, fmt.Sprintf("Invalid webhook URL: %v", err), http.StatusBadRequest)
return
}
// Generate ID if not provided
if webhook.ID == "" {
webhook.ID = utils.GenerateID("webhook")
}
// Persist the complete candidate inventory before exposing the destination
// to live alert delivery.
webhooks := append(manager.GetWebhooks(), webhook)
if err := monitor.GetConfigPersistence().SaveWebhooks(webhooks); err != nil {
log.Error().Err(err).Msg("Failed to save webhooks")
http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError)
return
}
manager.AddWebhook(webhook)
if err := utils.WriteJSONResponse(w, maskedWebhookResponse(webhook)); err != nil {
log.Error().Err(err).Msg("Failed to write webhook creation response")
}
}
// restoreStrictMaskedWebhookValues resolves only submitted placeholders. The editor
// sends a replacement map: explicit edits and omitted keys must not be undone
// just because another value is still masked. Header names are case-insensitive;
// custom-field names are not. Never store a placeholder without a saved value.
func restoreStrictMaskedWebhookValues(incoming, existing map[string]string, headers bool) (map[string]string, error) {
if incoming == nil {
return nil, nil
}
// Build the full HTTP identity index, even if an exact spelling exists:
// a legacy case-varied alias must not override or conceal a conflicting value.
savedHeaders := make(map[string]string)
ambiguousHeaders := make(map[string]bool)
if headers {
for key, value := range existing {
identity := strings.ToLower(key)
if previous, found := savedHeaders[identity]; found && previous != value {
ambiguousHeaders[identity] = true
}
savedHeaders[identity] = value
}
}
resolved := make(map[string]string, len(incoming))
incomingHeaders := make(map[string]string)
for key, value := range incoming {
identity := strings.ToLower(key)
if value == maskedWebhookSecret {
stored, found := existing[key]
if headers {
if ambiguousHeaders[identity] {
return nil, fmt.Errorf("ambiguous saved value for masked header %q", key)
}
stored, found = savedHeaders[identity]
}
if !found || stored == maskedWebhookSecret {
return nil, fmt.Errorf("no saved value for masked field %q", key)
}
value = stored
}
if headers {
if previous, found := incomingHeaders[identity]; found && previous != value {
return nil, fmt.Errorf("ambiguous submitted values for header %q", key)
}
incomingHeaders[identity] = value
}
resolved[key] = value
}
return resolved, nil
}
// UpdateWebhook updates an existing webhook.
func (h *NotificationHandlers) UpdateWebhook(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Extract webhook ID from URL path
// Path is like /api/notifications/webhooks/{id} after routing
path := strings.TrimPrefix(r.URL.Path, "/api/notifications/webhooks/")
webhookID := path
if webhookID == "" {
http.Error(w, "Invalid URL - missing webhook ID", http.StatusBadRequest)
return
}
// Limit request body to 64KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
// Read the raw body to preserve all fields
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var webhook notifications.WebhookConfig
if err := json.Unmarshal(bodyBytes, &webhook); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var routingPresence struct {
TagFilter *[]string `json:"tagFilter"`
TagMode *string `json:"tagFilterMode"`
MinimumSeverity *string `json:"minimumSeverity"`
}
_ = json.Unmarshal(bodyBytes, &routingPresence)
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
manager := monitor.GetNotificationManager()
// Restore each masked value returned by GetWebhooks, not its entire map.
existingWebhooks := manager.GetWebhooks()
found := false
for _, existing := range existingWebhooks {
if existing.ID == webhookID {
found = true
if routingPresence.TagFilter == nil {
webhook.TagFilter = existing.TagFilter
}
if routingPresence.TagMode == nil {
webhook.TagMode = existing.TagMode
}
if routingPresence.MinimumSeverity == nil {
webhook.MinimumSeverity = existing.MinimumSeverity
}
webhook, err = restoreMaskedWebhookSecrets(webhook, existing)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
break
}
}
if !found {
http.Error(w, fmt.Sprintf("webhook not found: %s", webhookID), http.StatusNotFound)
return
}
// Validate webhook URL
if err := manager.ValidateWebhookURL(webhook.URL); err != nil {
http.Error(w, fmt.Sprintf("Invalid webhook URL: %v", err), http.StatusBadRequest)
return
}
webhook.ID = webhookID
candidateWebhooks := make([]notifications.WebhookConfig, len(existingWebhooks))
copy(candidateWebhooks, existingWebhooks)
for i := range candidateWebhooks {
if candidateWebhooks[i].ID == webhookID {
candidateWebhooks[i] = webhook
break
}
}
if err := monitor.GetConfigPersistence().SaveWebhooks(candidateWebhooks); err != nil {
log.Error().Err(err).Msg("Failed to save webhooks")
http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError)
return
}
if err := manager.UpdateWebhook(webhookID, webhook); err != nil {
// The handler serializes destination writes, so this indicates an
// out-of-band mutation. Restore the prior durable inventory rather than
// leave restart-time state ahead of the live manager.
if rollbackErr := monitor.GetConfigPersistence().SaveWebhooks(existingWebhooks); rollbackErr != nil {
log.Error().Err(rollbackErr).Msg("Failed to roll back webhook configuration")
}
http.Error(w, "failed to publish webhook configuration", http.StatusInternalServerError)
return
}
if err := utils.WriteJSONResponse(w, maskedWebhookResponse(webhook)); err != nil {
log.Error().Err(err).Str("webhookID", webhookID).Msg("Failed to write webhook update response")
}
}
// DeleteWebhook deletes a webhook
func (h *NotificationHandlers) DeleteWebhook(w http.ResponseWriter, r *http.Request) {
h.configMu.Lock()
defer h.configMu.Unlock()
// Extract webhook ID from URL path
// Path is like /api/notifications/webhooks/{id} after routing
path := strings.TrimPrefix(r.URL.Path, "/api/notifications/webhooks/")
webhookID := path
if webhookID == "" {
http.Error(w, "Invalid URL - missing webhook ID", http.StatusBadRequest)
return
}
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
manager := monitor.GetNotificationManager()
existingWebhooks := manager.GetWebhooks()
candidateWebhooks := make([]notifications.WebhookConfig, 0, len(existingWebhooks))
found := false
for _, webhook := range existingWebhooks {
if webhook.ID == webhookID {
found = true
continue
}
candidateWebhooks = append(candidateWebhooks, webhook)
}
if !found {
http.Error(w, fmt.Sprintf("webhook not found: %s", webhookID), http.StatusNotFound)
return
}
if err := monitor.GetConfigPersistence().SaveWebhooks(candidateWebhooks); err != nil {
log.Error().Err(err).Msg("Failed to save webhooks")
http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError)
return
}
if err := manager.DeleteWebhook(webhookID); err != nil {
if rollbackErr := monitor.GetConfigPersistence().SaveWebhooks(existingWebhooks); rollbackErr != nil {
log.Error().Err(rollbackErr).Msg("Failed to roll back webhook configuration")
}
http.Error(w, "failed to publish webhook configuration", http.StatusInternalServerError)
return
}
if err := utils.WriteJSONResponse(w, map[string]string{"status": "success"}); err != nil {
log.Error().Err(err).Str("webhookID", webhookID).Msg("Failed to write webhook deletion response")
}
}
// classifyNotificationError maps raw Go errors to user-friendly summaries.
// Returns a user-facing message and the original error string as detail.
func classifyNotificationError(err error) (summary, detail string) {
raw := err.Error()
detail = raw
lower := strings.ToLower(raw)
switch {
case strings.Contains(lower, "connection refused"):
summary = "Could not connect to the server — check host, port, and firewall settings"
case strings.Contains(lower, "no such host"):
summary = "Server hostname not found — check the server address"
case strings.Contains(lower, "i/o timeout") || strings.Contains(lower, "deadline exceeded"):
summary = "Connection timed out — the server may be unreachable or the port may be blocked"
case strings.Contains(lower, "x509:") || strings.Contains(lower, "certificate"):
summary = "TLS certificate error — check certificate settings or try enabling 'Skip TLS Verify'"
case strings.Contains(lower, "535") || strings.Contains(lower, "authentication"):
summary = "Authentication failed — check username and password"
case strings.Contains(lower, "executable file not found"):
summary = "Required program not found — ensure it is installed on the server"
case strings.Contains(lower, "permission denied"):
summary = "Permission denied — the server process lacks access to the required resource"
case strings.Contains(lower, "eof"):
summary = "The server closed the connection unexpectedly — check if TLS/StartTLS settings are correct"
default:
summary = raw
detail = ""
}
return summary, detail
}
// writeTestNotificationError writes a JSON error response for test notification failures.
func writeTestNotificationError(w http.ResponseWriter, err error, statusCode int) {
summary, detail := classifyNotificationError(err)
resp := map[string]string{"error": summary}
if detail != "" {
resp["detail"] = detail
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(statusCode)
json.NewEncoder(w).Encode(resp)
}
// TestNotification sends a test notification
func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, notificationTestRequestBodyLimit)
// Read body for debugging
body, err := io.ReadAll(r.Body)
if err != nil {
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
http.Error(w, "Request body too large", http.StatusRequestEntityTooLarge)
return
}
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// NEVER log the body as it contains passwords
log.Info().
Msg("Test notification request received")
var req struct {
Method string `json:"method"` // "email", "webhook", or "apprise"
Type string `json:"type"` // Alternative field name used by frontend
Config json.RawMessage `json:"config,omitempty"` // Optional config for testing (email or apprise)
WebhookID string `json:"webhookId,omitempty"` // For webhook testing
}
if err := json.Unmarshal(body, &req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Support both "method" and "type" field names
if req.Method == "" && req.Type != "" {
req.Method = req.Type
}
// Get actual node info from ReadState (unified resources registry)
var nodeInfo *notifications.TestNodeInfo
if rs := h.getReadState(r.Context()); rs != nil {
if nodes := rs.Nodes(); len(nodes) > 0 {
nodeInfo = &notifications.TestNodeInfo{
NodeName: nodes[0].Name(),
InstanceURL: nodes[0].Instance(),
}
}
}
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
// Handle webhook testing
if req.Method == "webhook" && req.WebhookID != "" {
log.Info().
Str("webhookId", req.WebhookID).
Msg("Testing specific webhook")
// Get the webhook by ID and test it
webhooks := monitor.GetNotificationManager().GetWebhooks()
var foundWebhook *notifications.WebhookConfig
for _, wh := range webhooks {
if wh.ID == req.WebhookID {
foundWebhook = &wh
break
}
}
if foundWebhook == nil {
http.Error(w, "Webhook not found", http.StatusNotFound)
return
}
// Send test webhook
if err := monitor.GetNotificationManager().SendTestWebhook(*foundWebhook); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
} else if req.Method == "email" && len(req.Config) > 0 {
var emailConfig notifications.EmailConfig
if err := json.Unmarshal(req.Config, &emailConfig); err != nil {
http.Error(w, fmt.Sprintf("Invalid email config: %v", err), http.StatusBadRequest)
return
}
// If password is empty, use the saved password
if emailConfig.Password == "" {
savedConfig := monitor.GetNotificationManager().GetEmailConfig()
emailConfig.Password = savedConfig.Password
}
log.Info().
Bool("enabled", emailConfig.Enabled).
Str("smtp", emailConfig.SMTPHost).
Str("from", emailConfig.From).
Int("toCount", len(emailConfig.To)).
Strs("to", emailConfig.To).
Bool("hasPassword", emailConfig.Password != "").
Msg("Testing email with provided config")
if err := monitor.GetNotificationManager().SendTestNotificationWithConfig(req.Method, &emailConfig, nodeInfo); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
} else if req.Method == "apprise" && len(req.Config) > 0 {
var appriseConfig notifications.AppriseConfig
if err := json.Unmarshal(req.Config, &appriseConfig); err != nil {
http.Error(w, fmt.Sprintf("Invalid Apprise config: %v", err), http.StatusBadRequest)
return
}
// If the API key is empty, use the saved key
if appriseConfig.APIKey == "" {
savedConfig := monitor.GetNotificationManager().GetAppriseConfig()
appriseConfig.APIKey = savedConfig.APIKey
}
if err := monitor.GetNotificationManager().SendTestAppriseWithConfig(appriseConfig); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
} else {
// Use saved config
if err := monitor.GetNotificationManager().SendTestNotification(req.Method); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
}
// A test send bypasses the activation gate real alerts honor, so a bare
// success here is exactly how installs end up believing delivery works
// while every real alert is suppressed. Say so in the result itself.
response := map[string]interface{}{
"status": "success",
"message": "Test notification sent",
}
if !monitor.GetNotificationManager().IsEnabled() {
response["deliveryPaused"] = true
response["message"] = "Test notification sent, but alert delivery is paused: real alerts are not being sent"
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(response)
}
// GetWebhookTemplates returns available webhook templates
func (h *NotificationHandlers) GetWebhookTemplates(w http.ResponseWriter, r *http.Request) {
templates := notifications.GetWebhookTemplates()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(templates)
}
// GetWebhookHistory returns recent webhook delivery history with URLs redacted
func (h *NotificationHandlers) GetWebhookHistory(w http.ResponseWriter, r *http.Request) {
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
history := monitor.GetNotificationManager().GetWebhookHistory()
// Redact secrets from URLs in history
for i := range history {
history[i].WebhookURL = redactSecretsFromURL(history[i].WebhookURL)
// Note: ResponseBody is not stored in WebhookDelivery struct
// Error messages are already limited in length
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(history)
}
// redactSecretsFromURL is retained as the API package boundary for delivery
// history while the canonical redaction policy lives with webhook execution.
func redactSecretsFromURL(urlStr string) string {
return notifications.RedactWebhookURLSecrets(urlStr)
}
// GetEmailProviders returns available email providers
func (h *NotificationHandlers) GetEmailProviders(w http.ResponseWriter, r *http.Request) {
providers := notifications.GetEmailProviders()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(providers)
}
// TestWebhook tests a webhook configuration
func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Request) {
// First try to decode as basic webhook config
var basicWebhook notifications.WebhookConfig
r.Body = http.MaxBytesReader(w, r.Body, webhookTestRequestBodyLimit)
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
http.Error(w, "Request body too large", http.StatusRequestEntityTooLarge)
return
}
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := json.Unmarshal(bodyBytes, &basicWebhook); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Testing an edit of a saved webhook sends the form, whose untouched
// secrets are the masked values from GetWebhooks. Test with the saved
// values, as the saved webhook would send; this route requires the same
// settings:write scope as UpdateWebhook, which keeps them the same way.
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
saved := notifications.WebhookConfig{}
if basicWebhook.ID != "" {
for _, candidate := range monitor.GetNotificationManager().GetWebhooks() {
if candidate.ID == basicWebhook.ID {
saved = candidate
break
}
}
}
basicWebhook, err = restoreMaskedWebhookSecrets(basicWebhook, saved)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Try to extract service from body if present
var serviceCheck struct {
Service string `json:"service"`
}
if err := json.Unmarshal(bodyBytes, &serviceCheck); err == nil && serviceCheck.Service != "" {
basicWebhook.Service = serviceCheck.Service
}
webhook := notifications.BuildEnhancedWebhookTestConfig(basicWebhook, serviceCheck.Service)
log.Info().
Str("service", webhook.Service).
Str("url", notifications.RedactWebhookURLSecrets(webhook.URL)).
Str("name", webhook.Name).
Msg("Testing webhook")
// Test the webhook
status, response, err := monitor.GetNotificationManager().TestEnhancedWebhook(webhook)
result := map[string]interface{}{
"status": status,
"response": response,
}
if err != nil {
summary, detail := classifyNotificationError(err)
result["error"] = summary
if detail != "" {
result["detail"] = detail
}
w.WriteHeader(http.StatusBadRequest)
} else if status < 200 || status >= 300 {
// HTTP error from webhook endpoint
result["error"] = fmt.Sprintf("Webhook returned HTTP %d: %s", status, response)
result["success"] = false
w.WriteHeader(http.StatusBadRequest)
} else {
result["success"] = true
// Same honesty as TestNotification: a webhook test bypasses the
// activation gate, so its success must not imply live alerts flow.
if !monitor.GetNotificationManager().IsEnabled() {
result["deliveryPaused"] = true
}
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(result)
}
// GetNotificationHealth returns health status of notification system
func (h *NotificationHandlers) GetNotificationHealth(w http.ResponseWriter, r *http.Request) {
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
manager := monitor.GetNotificationManager()
failureClasses := notifications.NotificationFailureClassCounts{}.AsMap()
failureClassesAvailable := false
if deliveryStats, statsErr := manager.GetTelemetryStats(
time.Now().Add(-completedQueueRetentionDays * 24 * time.Hour),
); statsErr != nil {
log.Warn().Err(statsErr).Msg("Failed to get notification failure classes for health check")
} else {
failureClasses = deliveryStats.FailureClasses.AsMap()
failureClassesAvailable = true
}
queueStats := make(map[string]interface{})
stats, err := manager.GetQueueStats()
if err != nil {
log.Warn().Err(err).Msg("Failed to get queue stats for health check")
queueStats = map[string]interface{}{
"healthy": false,
"status": "unavailable",
"attention_required": 0,
"reason_codes": []string{"queue_stats_unavailable"},
"completed_retention_days": completedQueueRetentionDays,
"dead_letter_retention_days": deadLetterQueueRetentionDays,
"counts_are_retention_bounded": true,
"retry_attempts_affect_health": false,
"terminal_failures_affect_health": true,
"failure_classes_7d": failureClasses,
"failure_classes_available": failureClassesAvailable,
"failure_class_window_days": completedQueueRetentionDays,
}
} else {
healthy, attentionRequired, reasonCodes := classifyNotificationQueueHealth(stats)
status := "healthy"
if !healthy {
status = "degraded"
}
queueStats = map[string]interface{}{
"pending": stats["pending"],
"sending": stats["sending"],
"sent": stats["sent"],
"failed": stats["failed"],
"dlq": stats["dlq"],
"healthy": healthy,
"status": status,
"attention_required": attentionRequired,
"reason_codes": reasonCodes,
"completed_retention_days": completedQueueRetentionDays,
"dead_letter_retention_days": deadLetterQueueRetentionDays,
"counts_are_retention_bounded": true,
"retry_attempts_affect_health": false,
"terminal_failures_affect_health": true,
"failure_classes_7d": failureClasses,
"failure_classes_available": failureClassesAvailable,
"failure_class_window_days": completedQueueRetentionDays,
}
}
// Get config status
emailCfg := manager.GetEmailConfig()
webhooks := manager.GetWebhooks()
health := map[string]interface{}{
"queue": queueStats,
"email": map[string]interface{}{
"enabled": emailCfg.Enabled,
"configured": emailCfg.SMTPHost != "",
},
"webhooks": map[string]interface{}{
"total": len(webhooks),
"enabled": countEnabledWebhooks(webhooks),
},
"encryption": map[string]interface{}{
"enabled": monitor.GetConfigPersistence().IsEncryptionEnabled(),
},
"overall_healthy": queueStats["healthy"] == true,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(health)
}
// GetDeliveryLog returns recent recorded delivery attempts, newest first.
// This is the per-attempt evidence behind the aggregate health verdict: what
// fired, which destination it went to, and what happened. Entries share the
// queue's retention windows, so the payload names the window rather than
// presenting itself as lifetime history.
func (h *NotificationHandlers) GetDeliveryLog(w http.ResponseWriter, r *http.Request) {
monitor := h.requireMonitor(w, r)
if monitor == nil {
return
}
manager := monitor.GetNotificationManager()
limit := 0
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
parsed, err := strconv.Atoi(raw)
if err != nil || parsed <= 0 {
http.Error(w, "limit must be a positive integer", http.StatusBadRequest)
return
}
limit = parsed
}
entries, err := manager.GetDeliveryLog(
time.Now().Add(-deadLetterQueueRetentionDays*24*time.Hour), limit,
)
if err != nil {
log.Warn().Err(err).Msg("Failed to read notification delivery log")
http.Error(w, "Delivery log unavailable", http.StatusServiceUnavailable)
return
}
// Webhook failure text can embed the destination URL, and destination URLs
// can embed credentials.
for i := range entries {
if entries[i].ErrorMessage != "" {
entries[i].ErrorMessage = notifications.RedactWebhookDiagnosticSecrets(entries[i].ErrorMessage)
}
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"entries": entries,
"window_days": deadLetterQueueRetentionDays,
"completed_retention_days": completedQueueRetentionDays,
"dead_letter_retention_days": deadLetterQueueRetentionDays,
"entries_are_retention_bounded": true,
})
}
// classifyNotificationQueueHealth delegates to the canonical rule in
// internal/notifications so this endpoint and the monitoring loop that raises
// the notification-delivery system alert cannot drift apart.
func classifyNotificationQueueHealth(stats map[string]int) (bool, int, []string) {
health := notifications.ClassifyQueueHealth(stats)
return health.Healthy, health.AttentionRequired, health.ReasonCodes
}
func countEnabledWebhooks(webhooks []notifications.WebhookConfig) int {
count := 0
for _, wh := range webhooks {
if wh.Enabled {
count++
}
}
return count
}
// HandleNotifications routes notification requests to appropriate handlers
func (h *NotificationHandlers) HandleNotifications(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, "/api/notifications")
requireAnyScope := func(required string, scopes ...string) bool {
record := internalauth.GetAPIToken(r.Context())
if record == nil {
return true
}
for _, scope := range scopes {
if scope != "" && record.HasScope(scope) {
return true
}
}
apihttp.RespondMissingScope(w, required)
return false
}
switch {
case path == "/email" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetEmailConfig(w, r)
case path == "/email" && r.Method == http.MethodPut:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.UpdateEmailConfig(w, r)
case path == "/apprise" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetAppriseConfig(w, r)
case path == "/apprise" && r.Method == http.MethodPut:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.UpdateAppriseConfig(w, r)
case path == "/webhooks" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetWebhooks(w, r)
case path == "/webhooks" && r.Method == http.MethodPost:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.CreateWebhook(w, r)
case path == "/webhooks/test" && r.Method == http.MethodPost:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.TestWebhook(w, r)
case strings.HasPrefix(path, "/webhooks/") && r.Method == http.MethodPut:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.UpdateWebhook(w, r)
case strings.HasPrefix(path, "/webhooks/") && r.Method == http.MethodDelete:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.DeleteWebhook(w, r)
case path == "/webhook-templates" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetWebhookTemplates(w, r)
case path == "/webhook-history" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetWebhookHistory(w, r)
case path == "/email-providers" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetEmailProviders(w, r)
case path == "/test" && r.Method == http.MethodPost:
if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) {
return
}
h.TestNotification(w, r)
case path == "/health" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetNotificationHealth(w, r)
case path == "/delivery-log" && r.Method == http.MethodGet:
if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) {
return
}
h.GetDeliveryLog(w, r)
default:
http.Error(w, "Not found", http.StatusNotFound)
}
}