package alerting import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "strconv" "strings" "sync" "time" "github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext" "github.com/rcourtman/pulse-go-rewrite/internal/api/apihttp" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" "github.com/rcourtman/pulse-go-rewrite/internal/notifications" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/internal/utils" internalauth "github.com/rcourtman/pulse-go-rewrite/pkg/auth" "github.com/rs/zerolog/log" ) const ( notificationTestRequestBodyLimit = 64 * 1024 webhookTestRequestBodyLimit = 64 * 1024 completedQueueRetentionDays = 7 deadLetterQueueRetentionDays = 30 ) // NotificationManager defines the interface for notification management operations. type NotificationManager interface { GetEmailConfig() notifications.EmailConfig SetEmailConfig(notifications.EmailConfig) GetAppriseConfig() notifications.AppriseConfig SetAppriseConfig(notifications.AppriseConfig) GetWebhooks() []notifications.WebhookConfig ValidateWebhookURL(string) error AddWebhook(notifications.WebhookConfig) UpdateWebhook(string, notifications.WebhookConfig) error DeleteWebhook(string) error SendTestWebhook(notifications.WebhookConfig) error SendTestNotificationWithConfig(string, *notifications.EmailConfig, *notifications.TestNodeInfo) error SendTestAppriseWithConfig(notifications.AppriseConfig) error SendTestNotification(string) error GetWebhookHistory() []notifications.WebhookDelivery TestEnhancedWebhook(notifications.EnhancedWebhookConfig) (int, string, error) GetQueueStats() (map[string]int, error) GetTelemetryStats(time.Time) (notifications.TelemetryStats, error) GetDeliveryLog(time.Time, int) ([]notifications.DeliveryLogEntry, error) IsEnabled() bool } // NotificationConfigPersistence defines the interface for saving notification configuration. type NotificationConfigPersistence interface { SaveEmailConfig(notifications.EmailConfig) error SaveAppriseConfig(notifications.AppriseConfig) error SaveWebhooks([]notifications.WebhookConfig) error IsEncryptionEnabled() bool } // NotificationMonitor defines the interface for monitoring operations used by notification handlers. type NotificationMonitor interface { GetNotificationManager() NotificationManager GetConfigPersistence() NotificationConfigPersistence } // NotificationHandlers handles notification-related HTTP endpoints type NotificationHandlers struct { stateMu sync.RWMutex configMu sync.Mutex mtMonitor *monitoring.MultiTenantMonitor defaultMonitor NotificationMonitor readState unifiedresources.ReadState } // NewNotificationHandlers creates new notification handlers func NewNotificationHandlers(mtm *monitoring.MultiTenantMonitor, monitor NotificationMonitor) *NotificationHandlers { // If mtm is provided, try to populate defaultMonitor from "default" org if not provided. if monitor == nil && mtm != nil { if m, err := mtm.GetMonitor("default"); err == nil { monitor = NewNotificationMonitorWrapper(m) } } return &NotificationHandlers{ mtMonitor: mtm, defaultMonitor: monitor, } } // SetMonitor updates the monitor reference for notification handlers. func (h *NotificationHandlers) SetMonitor(m NotificationMonitor) { h.stateMu.Lock() defer h.stateMu.Unlock() h.defaultMonitor = m } // SetMultiTenantMonitor updates the multi-tenant monitor reference func (h *NotificationHandlers) SetMultiTenantMonitor(mtm *monitoring.MultiTenantMonitor) { var defaultMonitor NotificationMonitor if mtm != nil { if m, err := mtm.GetMonitor("default"); err == nil { defaultMonitor = NewNotificationMonitorWrapper(m) } } h.stateMu.Lock() defer h.stateMu.Unlock() h.mtMonitor = mtm if defaultMonitor != nil { h.defaultMonitor = defaultMonitor } } // SetReadState updates the ReadState reference for notification handlers. func (h *NotificationHandlers) SetReadState(rs unifiedresources.ReadState) { h.stateMu.Lock() defer h.stateMu.Unlock() h.readState = rs } func (h *NotificationHandlers) getMonitor(ctx context.Context) NotificationMonitor { h.stateMu.RLock() mtMonitor := h.mtMonitor defaultMonitor := h.defaultMonitor h.stateMu.RUnlock() orgID := apicontext.OrgID(ctx) if mtMonitor != nil { if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil { return NewNotificationMonitorWrapper(m) } } return defaultMonitor } // MonitorForContext resolves the tenant-scoped notification runtime used by // handlers and extensions. func (h *NotificationHandlers) MonitorForContext(ctx context.Context) NotificationMonitor { return h.getMonitor(ctx) } // getReadState returns the tenant-scoped ReadState for the request context. // It resolves the correct monitor for the org and returns its ReadState. // Falls back to the handler-level readState only for default/empty org. // Non-default orgs fail closed (return nil) if tenant ReadState is unavailable. func (h *NotificationHandlers) getReadState(ctx context.Context) unifiedresources.ReadState { h.stateMu.RLock() mtMonitor := h.mtMonitor fallback := h.readState h.stateMu.RUnlock() orgID := apicontext.OrgID(ctx) if mtMonitor != nil { if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil { if rs := m.GetUnifiedReadState(); rs != nil { return rs } } // Fail closed for non-default orgs: never leak default-org metadata. if orgID != "" && orgID != "default" { return nil } } return fallback } // GetEmailConfig returns the current email configuration func (h *NotificationHandlers) GetEmailConfig(w http.ResponseWriter, r *http.Request) { config := h.getMonitor(r.Context()).GetNotificationManager().GetEmailConfig() // For security, don't return the password config.Password = "" w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(config) } // UpdateEmailConfig updates the email configuration func (h *NotificationHandlers) UpdateEmailConfig(w http.ResponseWriter, r *http.Request) { h.configMu.Lock() defer h.configMu.Unlock() // Limit request body to 32KB to prevent memory exhaustion r.Body = http.MaxBytesReader(w, r.Body, 32*1024) // Read raw body for debugging body, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } // NEVER log the body as it contains passwords log.Info(). Msg("Received email config update") // Parse strict subset to check for presence of fields var presenceCheck struct { RateLimit *int `json:"rateLimit"` TagFilter *[]string `json:"tagFilter"` TagMode *string `json:"tagFilterMode"` MinimumSeverity *string `json:"minimumSeverity"` } if err := json.Unmarshal(body, &presenceCheck); err != nil { // Non-fatal, just means we can't do presence check } var config notifications.EmailConfig if err := json.Unmarshal(body, &config); err != nil { log.Error().Err(err).Msg("Failed to parse email config") // Don't log body with passwords http.Error(w, err.Error(), http.StatusBadRequest) return } monitor := h.getMonitor(r.Context()) manager := monitor.GetNotificationManager() existingConfig := manager.GetEmailConfig() // If password is empty, preserve the existing password if config.Password == "" { config.Password = existingConfig.Password } // If rateLimit was NOT provided (nil in presence check), preserve existing if presenceCheck.RateLimit == nil { config.RateLimit = existingConfig.RateLimit } if presenceCheck.TagFilter == nil { config.TagFilter = existingConfig.TagFilter } if presenceCheck.TagMode == nil { config.TagMode = existingConfig.TagMode } if presenceCheck.MinimumSeverity == nil { config.MinimumSeverity = existingConfig.MinimumSeverity } log.Info(). Bool("enabled", config.Enabled). Str("smtp", config.SMTPHost). Str("from", config.From). Int("toCount", len(config.To)). Bool("hasPassword", config.Password != ""). Int("rateLimit", config.RateLimit). Msg("Parsed email config") // Durable state is the publication boundary. Applying the live config first // can cancel queued deliveries or change routing even when the update cannot // survive a restart. if err := monitor.GetConfigPersistence().SaveEmailConfig(config); err != nil { log.Error().Err(err).Msg("Failed to save email configuration") http.Error(w, "failed to save email configuration", http.StatusInternalServerError) return } manager.SetEmailConfig(config) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]string{"status": "success"}) } // appriseConfigResponse is the API projection of an AppriseConfig. The stored // API key never leaves the server; callers only learn whether one is saved, // matching how the email handler blanks the SMTP password. type appriseConfigResponse struct { notifications.AppriseConfig HasAPIKey bool `json:"hasApiKey"` } func redactAppriseConfig(config notifications.AppriseConfig) appriseConfigResponse { response := appriseConfigResponse{AppriseConfig: config, HasAPIKey: config.APIKey != ""} response.APIKey = "" return response } // GetAppriseConfig returns the current Apprise configuration. func (h *NotificationHandlers) GetAppriseConfig(w http.ResponseWriter, r *http.Request) { config := h.getMonitor(r.Context()).GetNotificationManager().GetAppriseConfig() w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(redactAppriseConfig(config)); err != nil { log.Error().Err(err).Msg("Failed to encode Apprise configuration response") } } // UpdateAppriseConfig updates the Apprise configuration. func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *http.Request) { h.configMu.Lock() defer h.configMu.Unlock() // Limit request body to 64KB to prevent memory exhaustion r.Body = http.MaxBytesReader(w, r.Body, 64*1024) body, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } var config notifications.AppriseConfig if err := json.Unmarshal(body, &config); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } var presenceCheck struct { MinimumSeverity *string `json:"minimumSeverity"` } _ = json.Unmarshal(body, &presenceCheck) monitor := h.getMonitor(r.Context()) manager := monitor.GetNotificationManager() existingConfig := manager.GetAppriseConfig() // An empty API key means "keep the saved key": responses never include the // stored value, so the settings form cannot round-trip it. if config.APIKey == "" { config.APIKey = existingConfig.APIKey } if presenceCheck.MinimumSeverity == nil { config.MinimumSeverity = existingConfig.MinimumSeverity } log.Info(). Bool("enabled", config.Enabled). Str("mode", string(config.Mode)). Int("targetCount", len(config.Targets)). Str("cliPath", config.CLIPath). Str("serverUrl", config.ServerURL). Str("configKey", config.ConfigKey). Bool("hasApiKey", config.APIKey != ""). Str("apiKeyHeader", config.APIKeyHeader). Bool("skipTlsVerify", config.SkipTLSVerify). Int("timeoutSeconds", config.TimeoutSeconds). Msg("Parsed Apprise configuration update") if err := monitor.GetConfigPersistence().SaveAppriseConfig(config); err != nil { log.Error().Err(err).Msg("Failed to save Apprise configuration") http.Error(w, "failed to save Apprise configuration", http.StatusInternalServerError) return } manager.SetAppriseConfig(config) normalized := manager.GetAppriseConfig() w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(redactAppriseConfig(normalized)); err != nil { log.Error().Err(err).Msg("Failed to encode Apprise configuration response") } } // GetWebhooks returns all webhook configurations with secrets masked func (h *NotificationHandlers) GetWebhooks(w http.ResponseWriter, r *http.Request) { webhooks := h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks() // Mask sensitive fields in headers and customFields maskedWebhooks := make([]map[string]interface{}, len(webhooks)) for i, webhook := range webhooks { whMap := map[string]interface{}{ "id": webhook.ID, "name": webhook.Name, "url": webhook.URL, "method": webhook.Method, "enabled": webhook.Enabled, "service": webhook.Service, } // Mask headers - only show keys, not values if len(webhook.Headers) > 0 { maskedHeaders := make(map[string]string) for key := range webhook.Headers { maskedHeaders[key] = "***REDACTED***" } whMap["headers"] = maskedHeaders } // Mask custom fields - only show keys, not values if len(webhook.CustomFields) > 0 { maskedFields := make(map[string]string) for key := range webhook.CustomFields { maskedFields[key] = "***REDACTED***" } whMap["customFields"] = maskedFields } // Include template if present if webhook.Template != "" { whMap["template"] = webhook.Template } // Include the configured mention so the UI can render it after reload (#1118) if webhook.Mention != "" { whMap["mention"] = webhook.Mention } if len(webhook.TagFilter) > 0 { whMap["tagFilter"] = append([]string(nil), webhook.TagFilter...) whMap["tagFilterMode"] = webhook.TagMode } whMap["minimumSeverity"] = webhook.MinimumSeverity // Signal that a signing secret is configured without revealing it if webhook.SigningSecret != "" { whMap["signingSecret"] = "***REDACTED***" } maskedWebhooks[i] = whMap } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(maskedWebhooks) } // CreateWebhook creates a new webhook func (h *NotificationHandlers) CreateWebhook(w http.ResponseWriter, r *http.Request) { h.configMu.Lock() defer h.configMu.Unlock() // Limit request body to 64KB to prevent memory exhaustion r.Body = http.MaxBytesReader(w, r.Body, 64*1024) // Read the raw body to preserve all fields bodyBytes, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } var webhook notifications.WebhookConfig if err := json.Unmarshal(bodyBytes, &webhook); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } webhook = notifications.NormalizeWebhookConfig(webhook) monitor := h.getMonitor(r.Context()) manager := monitor.GetNotificationManager() // Validate webhook URL if err := manager.ValidateWebhookURL(webhook.URL); err != nil { http.Error(w, fmt.Sprintf("Invalid webhook URL: %v", err), http.StatusBadRequest) return } // Generate ID if not provided if webhook.ID == "" { webhook.ID = utils.GenerateID("webhook") } // Persist the complete candidate inventory before exposing the destination // to live alert delivery. webhooks := append(manager.GetWebhooks(), webhook) if err := monitor.GetConfigPersistence().SaveWebhooks(webhooks); err != nil { log.Error().Err(err).Msg("Failed to save webhooks") http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError) return } manager.AddWebhook(webhook) // Return the full webhook data including any extra fields like 'service' var responseData map[string]interface{} if err := json.Unmarshal(bodyBytes, &responseData); err != nil { log.Warn().Err(err).Msg("Failed to unmarshal webhook response data") responseData = make(map[string]interface{}) } responseData["id"] = webhook.ID responseData["customFields"] = webhook.CustomFields responseData["tagFilter"] = webhook.TagFilter responseData["tagFilterMode"] = webhook.TagMode responseData["minimumSeverity"] = webhook.MinimumSeverity if err := utils.WriteJSONResponse(w, responseData); err != nil { log.Error().Err(err).Msg("Failed to write webhook creation response") } } // UpdateWebhook updates an existing webhook func (h *NotificationHandlers) UpdateWebhook(w http.ResponseWriter, r *http.Request) { h.configMu.Lock() defer h.configMu.Unlock() // Extract webhook ID from URL path // Path is like /api/notifications/webhooks/{id} after routing path := strings.TrimPrefix(r.URL.Path, "/api/notifications/webhooks/") webhookID := path if webhookID == "" { http.Error(w, "Invalid URL - missing webhook ID", http.StatusBadRequest) return } // Limit request body to 64KB to prevent memory exhaustion r.Body = http.MaxBytesReader(w, r.Body, 64*1024) // Read the raw body to preserve all fields bodyBytes, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } var webhook notifications.WebhookConfig if err := json.Unmarshal(bodyBytes, &webhook); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } var routingPresence struct { TagFilter *[]string `json:"tagFilter"` TagMode *string `json:"tagFilterMode"` MinimumSeverity *string `json:"minimumSeverity"` } _ = json.Unmarshal(bodyBytes, &routingPresence) webhook = notifications.NormalizeWebhookConfig(webhook) monitor := h.getMonitor(r.Context()) manager := monitor.GetNotificationManager() // Preserve original headers/customFields if the incoming values are redacted // This happens when the frontend sends back masked values from GetWebhooks existingWebhooks := manager.GetWebhooks() found := false for _, existing := range existingWebhooks { if existing.ID == webhookID { found = true if routingPresence.TagFilter == nil { webhook.TagFilter = existing.TagFilter } if routingPresence.TagMode == nil { webhook.TagMode = existing.TagMode } if routingPresence.MinimumSeverity == nil { webhook.MinimumSeverity = existing.MinimumSeverity } // Preserve headers if incoming contains redacted values if len(webhook.Headers) > 0 && len(existing.Headers) > 0 { hasRedacted := false for _, v := range webhook.Headers { if v == "***REDACTED***" { hasRedacted = true break } } if hasRedacted { webhook.Headers = existing.Headers } } // Preserve customFields if incoming contains redacted values if len(webhook.CustomFields) > 0 && len(existing.CustomFields) > 0 { hasRedacted := false for _, v := range webhook.CustomFields { if v == "***REDACTED***" { hasRedacted = true break } } if hasRedacted { webhook.CustomFields = existing.CustomFields } } // Preserve the signing secret if the incoming value is redacted if webhook.SigningSecret == "***REDACTED***" { webhook.SigningSecret = existing.SigningSecret } break } } if !found { http.Error(w, fmt.Sprintf("webhook not found: %s", webhookID), http.StatusNotFound) return } // Validate webhook URL if err := manager.ValidateWebhookURL(webhook.URL); err != nil { http.Error(w, fmt.Sprintf("Invalid webhook URL: %v", err), http.StatusBadRequest) return } webhook.ID = webhookID candidateWebhooks := make([]notifications.WebhookConfig, len(existingWebhooks)) copy(candidateWebhooks, existingWebhooks) for i := range candidateWebhooks { if candidateWebhooks[i].ID == webhookID { candidateWebhooks[i] = webhook break } } if err := monitor.GetConfigPersistence().SaveWebhooks(candidateWebhooks); err != nil { log.Error().Err(err).Msg("Failed to save webhooks") http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError) return } if err := manager.UpdateWebhook(webhookID, webhook); err != nil { // The handler serializes destination writes, so this indicates an // out-of-band mutation. Restore the prior durable inventory rather than // leave restart-time state ahead of the live manager. if rollbackErr := monitor.GetConfigPersistence().SaveWebhooks(existingWebhooks); rollbackErr != nil { log.Error().Err(rollbackErr).Msg("Failed to roll back webhook configuration") } http.Error(w, "failed to publish webhook configuration", http.StatusInternalServerError) return } // Return the full webhook data including any extra fields like 'service' var responseData map[string]interface{} if err := json.Unmarshal(bodyBytes, &responseData); err != nil { log.Warn().Err(err).Msg("Failed to unmarshal webhook response data") responseData = make(map[string]interface{}) } responseData["id"] = webhookID responseData["customFields"] = webhook.CustomFields responseData["tagFilter"] = webhook.TagFilter responseData["tagFilterMode"] = webhook.TagMode responseData["minimumSeverity"] = webhook.MinimumSeverity if err := utils.WriteJSONResponse(w, responseData); err != nil { log.Error().Err(err).Str("webhookID", webhookID).Msg("Failed to write webhook update response") } } // DeleteWebhook deletes a webhook func (h *NotificationHandlers) DeleteWebhook(w http.ResponseWriter, r *http.Request) { h.configMu.Lock() defer h.configMu.Unlock() // Extract webhook ID from URL path // Path is like /api/notifications/webhooks/{id} after routing path := strings.TrimPrefix(r.URL.Path, "/api/notifications/webhooks/") webhookID := path if webhookID == "" { http.Error(w, "Invalid URL - missing webhook ID", http.StatusBadRequest) return } monitor := h.getMonitor(r.Context()) manager := monitor.GetNotificationManager() existingWebhooks := manager.GetWebhooks() candidateWebhooks := make([]notifications.WebhookConfig, 0, len(existingWebhooks)) found := false for _, webhook := range existingWebhooks { if webhook.ID == webhookID { found = true continue } candidateWebhooks = append(candidateWebhooks, webhook) } if !found { http.Error(w, fmt.Sprintf("webhook not found: %s", webhookID), http.StatusNotFound) return } if err := monitor.GetConfigPersistence().SaveWebhooks(candidateWebhooks); err != nil { log.Error().Err(err).Msg("Failed to save webhooks") http.Error(w, "failed to save webhook configuration", http.StatusInternalServerError) return } if err := manager.DeleteWebhook(webhookID); err != nil { if rollbackErr := monitor.GetConfigPersistence().SaveWebhooks(existingWebhooks); rollbackErr != nil { log.Error().Err(rollbackErr).Msg("Failed to roll back webhook configuration") } http.Error(w, "failed to publish webhook configuration", http.StatusInternalServerError) return } if err := utils.WriteJSONResponse(w, map[string]string{"status": "success"}); err != nil { log.Error().Err(err).Str("webhookID", webhookID).Msg("Failed to write webhook deletion response") } } // classifyNotificationError maps raw Go errors to user-friendly summaries. // Returns a user-facing message and the original error string as detail. func classifyNotificationError(err error) (summary, detail string) { raw := err.Error() detail = raw lower := strings.ToLower(raw) switch { case strings.Contains(lower, "connection refused"): summary = "Could not connect to the server — check host, port, and firewall settings" case strings.Contains(lower, "no such host"): summary = "Server hostname not found — check the server address" case strings.Contains(lower, "i/o timeout") || strings.Contains(lower, "deadline exceeded"): summary = "Connection timed out — the server may be unreachable or the port may be blocked" case strings.Contains(lower, "x509:") || strings.Contains(lower, "certificate"): summary = "TLS certificate error — check certificate settings or try enabling 'Skip TLS Verify'" case strings.Contains(lower, "535") || strings.Contains(lower, "authentication"): summary = "Authentication failed — check username and password" case strings.Contains(lower, "executable file not found"): summary = "Required program not found — ensure it is installed on the server" case strings.Contains(lower, "permission denied"): summary = "Permission denied — the server process lacks access to the required resource" case strings.Contains(lower, "eof"): summary = "The server closed the connection unexpectedly — check if TLS/StartTLS settings are correct" default: summary = raw detail = "" } return summary, detail } // writeTestNotificationError writes a JSON error response for test notification failures. func writeTestNotificationError(w http.ResponseWriter, err error, statusCode int) { summary, detail := classifyNotificationError(err) resp := map[string]string{"error": summary} if detail != "" { resp["detail"] = detail } w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) json.NewEncoder(w).Encode(resp) } // TestNotification sends a test notification func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, notificationTestRequestBodyLimit) // Read body for debugging body, err := io.ReadAll(r.Body) if err != nil { var maxBytesErr *http.MaxBytesError if errors.As(err, &maxBytesErr) { http.Error(w, "Request body too large", http.StatusRequestEntityTooLarge) return } http.Error(w, err.Error(), http.StatusBadRequest) return } // NEVER log the body as it contains passwords log.Info(). Msg("Test notification request received") var req struct { Method string `json:"method"` // "email", "webhook", or "apprise" Type string `json:"type"` // Alternative field name used by frontend Config json.RawMessage `json:"config,omitempty"` // Optional config for testing (email or apprise) WebhookID string `json:"webhookId,omitempty"` // For webhook testing } if err := json.Unmarshal(body, &req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } // Support both "method" and "type" field names if req.Method == "" && req.Type != "" { req.Method = req.Type } // Get actual node info from ReadState (unified resources registry) var nodeInfo *notifications.TestNodeInfo if rs := h.getReadState(r.Context()); rs != nil { if nodes := rs.Nodes(); len(nodes) > 0 { nodeInfo = ¬ifications.TestNodeInfo{ NodeName: nodes[0].Name(), InstanceURL: nodes[0].Instance(), } } } // Handle webhook testing if req.Method == "webhook" && req.WebhookID != "" { log.Info(). Str("webhookId", req.WebhookID). Msg("Testing specific webhook") // Get the webhook by ID and test it webhooks := h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks() var foundWebhook *notifications.WebhookConfig for _, wh := range webhooks { if wh.ID == req.WebhookID { foundWebhook = &wh break } } if foundWebhook == nil { http.Error(w, "Webhook not found", http.StatusNotFound) return } // Send test webhook if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestWebhook(*foundWebhook); err != nil { writeTestNotificationError(w, err, http.StatusBadRequest) return } } else if req.Method == "email" && len(req.Config) > 0 { var emailConfig notifications.EmailConfig if err := json.Unmarshal(req.Config, &emailConfig); err != nil { http.Error(w, fmt.Sprintf("Invalid email config: %v", err), http.StatusBadRequest) return } // If password is empty, use the saved password if emailConfig.Password == "" { savedConfig := h.getMonitor(r.Context()).GetNotificationManager().GetEmailConfig() emailConfig.Password = savedConfig.Password } log.Info(). Bool("enabled", emailConfig.Enabled). Str("smtp", emailConfig.SMTPHost). Str("from", emailConfig.From). Int("toCount", len(emailConfig.To)). Strs("to", emailConfig.To). Bool("hasPassword", emailConfig.Password != ""). Msg("Testing email with provided config") if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestNotificationWithConfig(req.Method, &emailConfig, nodeInfo); err != nil { writeTestNotificationError(w, err, http.StatusBadRequest) return } } else if req.Method == "apprise" && len(req.Config) > 0 { var appriseConfig notifications.AppriseConfig if err := json.Unmarshal(req.Config, &appriseConfig); err != nil { http.Error(w, fmt.Sprintf("Invalid Apprise config: %v", err), http.StatusBadRequest) return } // If the API key is empty, use the saved key if appriseConfig.APIKey == "" { savedConfig := h.getMonitor(r.Context()).GetNotificationManager().GetAppriseConfig() appriseConfig.APIKey = savedConfig.APIKey } if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestAppriseWithConfig(appriseConfig); err != nil { writeTestNotificationError(w, err, http.StatusBadRequest) return } } else { // Use saved config if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestNotification(req.Method); err != nil { writeTestNotificationError(w, err, http.StatusBadRequest) return } } // A test send bypasses the activation gate real alerts honor, so a bare // success here is exactly how installs end up believing delivery works // while every real alert is suppressed. Say so in the result itself. response := map[string]interface{}{ "status": "success", "message": "Test notification sent", } if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() { response["deliveryPaused"] = true response["message"] = "Test notification sent, but alert delivery is paused: real alerts are not being sent" } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(response) } // GetWebhookTemplates returns available webhook templates func (h *NotificationHandlers) GetWebhookTemplates(w http.ResponseWriter, r *http.Request) { templates := notifications.GetWebhookTemplates() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(templates) } // GetWebhookHistory returns recent webhook delivery history with URLs redacted func (h *NotificationHandlers) GetWebhookHistory(w http.ResponseWriter, r *http.Request) { history := h.getMonitor(r.Context()).GetNotificationManager().GetWebhookHistory() // Redact secrets from URLs in history for i := range history { history[i].WebhookURL = redactSecretsFromURL(history[i].WebhookURL) // Note: ResponseBody is not stored in WebhookDelivery struct // Error messages are already limited in length } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(history) } // redactSecretsFromURL is retained as the API package boundary for delivery // history while the canonical redaction policy lives with webhook execution. func redactSecretsFromURL(urlStr string) string { return notifications.RedactWebhookURLSecrets(urlStr) } // GetEmailProviders returns available email providers func (h *NotificationHandlers) GetEmailProviders(w http.ResponseWriter, r *http.Request) { providers := notifications.GetEmailProviders() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(providers) } // TestWebhook tests a webhook configuration func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Request) { // First try to decode as basic webhook config var basicWebhook notifications.WebhookConfig r.Body = http.MaxBytesReader(w, r.Body, webhookTestRequestBodyLimit) bodyBytes, err := io.ReadAll(r.Body) if err != nil { var maxBytesErr *http.MaxBytesError if errors.As(err, &maxBytesErr) { http.Error(w, "Request body too large", http.StatusRequestEntityTooLarge) return } http.Error(w, err.Error(), http.StatusBadRequest) return } if err := json.Unmarshal(bodyBytes, &basicWebhook); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } basicWebhook = notifications.NormalizeWebhookConfig(basicWebhook) // Try to extract service from body if present var serviceCheck struct { Service string `json:"service"` } if err := json.Unmarshal(bodyBytes, &serviceCheck); err == nil && serviceCheck.Service != "" { basicWebhook.Service = serviceCheck.Service } webhook := notifications.BuildEnhancedWebhookTestConfig(basicWebhook, serviceCheck.Service) log.Info(). Str("service", webhook.Service). Str("url", notifications.RedactWebhookURLSecrets(webhook.URL)). Str("name", webhook.Name). Msg("Testing webhook") // Test the webhook status, response, err := h.getMonitor(r.Context()).GetNotificationManager().TestEnhancedWebhook(webhook) result := map[string]interface{}{ "status": status, "response": response, } if err != nil { summary, detail := classifyNotificationError(err) result["error"] = summary if detail != "" { result["detail"] = detail } w.WriteHeader(http.StatusBadRequest) } else if status < 200 || status >= 300 { // HTTP error from webhook endpoint result["error"] = fmt.Sprintf("Webhook returned HTTP %d: %s", status, response) result["success"] = false w.WriteHeader(http.StatusBadRequest) } else { result["success"] = true // Same honesty as TestNotification: a webhook test bypasses the // activation gate, so its success must not imply live alerts flow. if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() { result["deliveryPaused"] = true } } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(result) } // GetNotificationHealth returns health status of notification system func (h *NotificationHandlers) GetNotificationHealth(w http.ResponseWriter, r *http.Request) { monitor := h.getMonitor(r.Context()) manager := monitor.GetNotificationManager() failureClasses := notifications.NotificationFailureClassCounts{}.AsMap() failureClassesAvailable := false if deliveryStats, statsErr := manager.GetTelemetryStats( time.Now().Add(-completedQueueRetentionDays * 24 * time.Hour), ); statsErr != nil { log.Warn().Err(statsErr).Msg("Failed to get notification failure classes for health check") } else { failureClasses = deliveryStats.FailureClasses.AsMap() failureClassesAvailable = true } queueStats := make(map[string]interface{}) stats, err := manager.GetQueueStats() if err != nil { log.Warn().Err(err).Msg("Failed to get queue stats for health check") queueStats = map[string]interface{}{ "healthy": false, "status": "unavailable", "attention_required": 0, "reason_codes": []string{"queue_stats_unavailable"}, "completed_retention_days": completedQueueRetentionDays, "dead_letter_retention_days": deadLetterQueueRetentionDays, "counts_are_retention_bounded": true, "retry_attempts_affect_health": false, "terminal_failures_affect_health": true, "failure_classes_7d": failureClasses, "failure_classes_available": failureClassesAvailable, "failure_class_window_days": completedQueueRetentionDays, } } else { healthy, attentionRequired, reasonCodes := classifyNotificationQueueHealth(stats) status := "healthy" if !healthy { status = "degraded" } queueStats = map[string]interface{}{ "pending": stats["pending"], "sending": stats["sending"], "sent": stats["sent"], "failed": stats["failed"], "dlq": stats["dlq"], "healthy": healthy, "status": status, "attention_required": attentionRequired, "reason_codes": reasonCodes, "completed_retention_days": completedQueueRetentionDays, "dead_letter_retention_days": deadLetterQueueRetentionDays, "counts_are_retention_bounded": true, "retry_attempts_affect_health": false, "terminal_failures_affect_health": true, "failure_classes_7d": failureClasses, "failure_classes_available": failureClassesAvailable, "failure_class_window_days": completedQueueRetentionDays, } } // Get config status emailCfg := manager.GetEmailConfig() webhooks := manager.GetWebhooks() health := map[string]interface{}{ "queue": queueStats, "email": map[string]interface{}{ "enabled": emailCfg.Enabled, "configured": emailCfg.SMTPHost != "", }, "webhooks": map[string]interface{}{ "total": len(webhooks), "enabled": countEnabledWebhooks(webhooks), }, "encryption": map[string]interface{}{ "enabled": monitor.GetConfigPersistence().IsEncryptionEnabled(), }, "overall_healthy": queueStats["healthy"] == true, } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(health) } // GetDeliveryLog returns recent recorded delivery attempts, newest first. // This is the per-attempt evidence behind the aggregate health verdict: what // fired, which destination it went to, and what happened. Entries share the // queue's retention windows, so the payload names the window rather than // presenting itself as lifetime history. func (h *NotificationHandlers) GetDeliveryLog(w http.ResponseWriter, r *http.Request) { manager := h.getMonitor(r.Context()).GetNotificationManager() limit := 0 if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" { parsed, err := strconv.Atoi(raw) if err != nil || parsed <= 0 { http.Error(w, "limit must be a positive integer", http.StatusBadRequest) return } limit = parsed } entries, err := manager.GetDeliveryLog( time.Now().Add(-deadLetterQueueRetentionDays*24*time.Hour), limit, ) if err != nil { log.Warn().Err(err).Msg("Failed to read notification delivery log") http.Error(w, "Delivery log unavailable", http.StatusServiceUnavailable) return } // Webhook failure text can embed the destination URL, and destination URLs // can embed credentials. for i := range entries { if entries[i].ErrorMessage != "" { entries[i].ErrorMessage = notifications.RedactWebhookURLSecrets(entries[i].ErrorMessage) } } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]interface{}{ "entries": entries, "window_days": deadLetterQueueRetentionDays, "completed_retention_days": completedQueueRetentionDays, "dead_letter_retention_days": deadLetterQueueRetentionDays, "entries_are_retention_bounded": true, }) } // classifyNotificationQueueHealth delegates to the canonical rule in // internal/notifications so this endpoint and the monitoring loop that raises // the notification-delivery system alert cannot drift apart. func classifyNotificationQueueHealth(stats map[string]int) (bool, int, []string) { health := notifications.ClassifyQueueHealth(stats) return health.Healthy, health.AttentionRequired, health.ReasonCodes } func countEnabledWebhooks(webhooks []notifications.WebhookConfig) int { count := 0 for _, wh := range webhooks { if wh.Enabled { count++ } } return count } // HandleNotifications routes notification requests to appropriate handlers func (h *NotificationHandlers) HandleNotifications(w http.ResponseWriter, r *http.Request) { path := strings.TrimPrefix(r.URL.Path, "/api/notifications") requireAnyScope := func(required string, scopes ...string) bool { record := internalauth.GetAPIToken(r.Context()) if record == nil { return true } for _, scope := range scopes { if scope != "" && record.HasScope(scope) { return true } } apihttp.RespondMissingScope(w, required) return false } switch { case path == "/email" && r.Method == http.MethodGet: if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) { return } h.GetEmailConfig(w, r) case path == "/email" && r.Method == http.MethodPut: if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) { return } h.UpdateEmailConfig(w, r) case path == "/apprise" && r.Method == http.MethodGet: if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) { return } h.GetAppriseConfig(w, r) case path == "/apprise" && r.Method == http.MethodPut: if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) { return } h.UpdateAppriseConfig(w, r) case path == "/webhooks" && r.Method == http.MethodGet: if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) { return } h.GetWebhooks(w, r) case path == "/webhooks" && r.Method == http.MethodPost: if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) { return } h.CreateWebhook(w, r) case path == "/webhooks/test" && r.Method == http.MethodPost: if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) { return } h.TestWebhook(w, r) case strings.HasPrefix(path, "/webhooks/") && r.Method == http.MethodPut: if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) { return } h.UpdateWebhook(w, r) case strings.HasPrefix(path, "/webhooks/") && r.Method == http.MethodDelete: if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) { return } h.DeleteWebhook(w, r) case path == "/webhook-templates" && r.Method == http.MethodGet: if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) { return } h.GetWebhookTemplates(w, r) case path == "/webhook-history" && r.Method == http.MethodGet: if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) { return } h.GetWebhookHistory(w, r) case path == "/email-providers" && r.Method == http.MethodGet: if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) { return } h.GetEmailProviders(w, r) case path == "/test" && r.Method == http.MethodPost: if !requireAnyScope(config.ScopeSettingsWrite, config.ScopeSettingsWrite) { return } h.TestNotification(w, r) case path == "/health" && r.Method == http.MethodGet: if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) { return } h.GetNotificationHealth(w, r) case path == "/delivery-log" && r.Method == http.MethodGet: if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) { return } h.GetDeliveryLog(w, r) default: http.Error(w, "Not found", http.StatusNotFound) } }