mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 05:57:27 +00:00
f7ac26b573
The Thresholds page builds its PUT /api/alerts/config body from the settings it edits. It has no control for flapping detection, alert TTL cleanup or custom rules, so it never sends flappingEnabled, maxAlertAgeDays, maxAcknowledgedAgeDays, autoAcknowledgeAfterHours or customRules. The handler decoded the body into a zero AlertConfig, so every thresholds save turned flapping detection and TTL cleanup off, reset the flapping tuning, dropped custom rules, and persisted that. The page's flapping, aggregation and ioNormalization objects are not AlertConfig fields and never enabled anything. The handler now hands the body to Manager.ApplyConfigUpdate, which reads the stored config, merges and applies it under the manager lock, so two concurrent partial saves cannot revert each other. The body is decoded on its own, so the keys it carries mean exactly what encoding/json makes of them; every top-level field it left out is copied from a JSON round trip of the stored config, sharing no maps with it. ApplyConfigUpdate returns a cloned snapshot, which the handler persists, because SaveAlertConfig normalizes the config it is handed in place and used to write into the live config's maps. Saves run one at a time so alerts.json ends with the last applied config. A body that is not a JSON object, null included, is a 400. Stored configs are not rewritten, so a value an earlier save zeroed stays as it is. On a private stack, a node CPU change saved from the Thresholds page zeroed all four fields in memory and in alerts.json on main; with this change the same save kept them and stored the new threshold.
1626 lines
53 KiB
Go
1626 lines
53 KiB
Go
package alerting
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/api/apihttp"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/utils"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// AlertManager defines the interface for alert management operations.
|
|
type AlertManager interface {
|
|
GetConfig() alerts.AlertConfig
|
|
UpdateConfig(alerts.AlertConfig)
|
|
ApplyConfigUpdate(update []byte) (alerts.AlertConfig, error)
|
|
GetActiveAlerts() []alerts.Alert
|
|
DiagnoseAlertDelivery(alertIdentifier string) (alerts.AlertDeliveryDiagnosis, bool)
|
|
DiagnoseActiveAlertDeliveries() []alerts.AlertDeliveryDiagnosis
|
|
AlertEvents(filter eventlog.Filter) ([]eventlog.Event, error)
|
|
NotifyExistingAlert(id string)
|
|
ClearAlertHistory() error
|
|
UnacknowledgeAlert(id string) error
|
|
AcknowledgeAlert(id, user string) error
|
|
SnoozeAlert(id, user string, until time.Time) error
|
|
UnsnoozeAlert(id, user string) error
|
|
ClearAlert(id string) bool
|
|
GetAlertHistory(limit int) []alerts.Alert
|
|
GetAlertHistorySince(since time.Time, limit int) []alerts.Alert
|
|
}
|
|
|
|
// ConfigPersistence defines the interface for saving configuration.
|
|
type ConfigPersistence interface {
|
|
SaveAlertConfig(alerts.AlertConfig) error
|
|
}
|
|
|
|
type alertIntentPolicyManager interface {
|
|
GetIntentPolicies() alerts.AlertIntentPolicyDocument
|
|
LoadIntentPolicies(alerts.AlertIntentPolicyDocument) error
|
|
UpdateIntentPolicies(alerts.AlertIntentPolicyDocument) (alerts.AlertIntentPolicyDocument, error)
|
|
PreviewIntentPolicy(alerts.AlertIntentPolicyPreviewRequest) (alerts.AlertIntentPolicyPreview, error)
|
|
}
|
|
|
|
type alertIntentPolicyPersistence interface {
|
|
SaveAlertIntentPolicies(alerts.AlertIntentPolicyDocument) error
|
|
}
|
|
|
|
// AlertMonitor defines the interface for monitoring operations used by alert handlers.
|
|
type AlertMonitor interface {
|
|
GetAlertManager() AlertManager
|
|
GetConfigPersistence() ConfigPersistence
|
|
GetIncidentStore() *memory.IncidentStore
|
|
GetNotificationManager() *notifications.NotificationManager
|
|
DeadManConfig() notifications.DeadManConfig
|
|
UpdateDeadManConfig(notifications.DeadManConfig) error
|
|
DeadManStatus() monitoring.DeadManStatus
|
|
SyncAlertState()
|
|
BuildFrontendState() models.StateFrontend
|
|
}
|
|
|
|
// AlertHandlers handles alert-related HTTP endpoints
|
|
type AlertHandlers struct {
|
|
stateMu sync.RWMutex
|
|
intentPolicyMu sync.Mutex
|
|
configSaveMu sync.Mutex
|
|
mtMonitor *monitoring.MultiTenantMonitor
|
|
defaultMonitor AlertMonitor
|
|
wsHub *websocket.Hub
|
|
}
|
|
|
|
// NewAlertHandlers creates new alert handlers
|
|
func NewAlertHandlers(mtm *monitoring.MultiTenantMonitor, monitor AlertMonitor, wsHub *websocket.Hub) *AlertHandlers {
|
|
// If mtm is provided, try to populate defaultMonitor from "default" org if not provided.
|
|
if monitor == nil && mtm != nil {
|
|
if m, err := mtm.GetMonitor("default"); err == nil {
|
|
monitor = NewAlertMonitorWrapper(m)
|
|
}
|
|
}
|
|
return &AlertHandlers{
|
|
mtMonitor: mtm,
|
|
defaultMonitor: monitor,
|
|
wsHub: wsHub,
|
|
}
|
|
}
|
|
|
|
// SetMultiTenantMonitor updates the multi-tenant monitor reference
|
|
func (h *AlertHandlers) SetMultiTenantMonitor(mtm *monitoring.MultiTenantMonitor) {
|
|
var defaultMonitor AlertMonitor
|
|
if mtm != nil {
|
|
if m, err := mtm.GetMonitor("default"); err == nil {
|
|
defaultMonitor = NewAlertMonitorWrapper(m)
|
|
}
|
|
}
|
|
|
|
h.stateMu.Lock()
|
|
defer h.stateMu.Unlock()
|
|
h.mtMonitor = mtm
|
|
if defaultMonitor != nil {
|
|
h.defaultMonitor = defaultMonitor
|
|
}
|
|
}
|
|
|
|
// SetMonitor updates the monitor reference for alert handlers.
|
|
func (h *AlertHandlers) SetMonitor(m AlertMonitor) {
|
|
h.stateMu.Lock()
|
|
defer h.stateMu.Unlock()
|
|
h.defaultMonitor = m
|
|
}
|
|
|
|
func (h *AlertHandlers) getMonitor(ctx context.Context) AlertMonitor {
|
|
h.stateMu.RLock()
|
|
mtMonitor := h.mtMonitor
|
|
defaultMonitor := h.defaultMonitor
|
|
h.stateMu.RUnlock()
|
|
|
|
orgID := apicontext.OrgID(ctx)
|
|
if mtMonitor != nil {
|
|
if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
|
|
return NewAlertMonitorWrapper(m)
|
|
}
|
|
}
|
|
return defaultMonitor
|
|
}
|
|
|
|
// MonitorForContext resolves the tenant-scoped alert runtime used by handlers.
|
|
// It is the supported extension boundary for integrations that need the same
|
|
// tenant selection semantics as HTTP alert routes.
|
|
func (h *AlertHandlers) MonitorForContext(ctx context.Context) AlertMonitor {
|
|
return h.getMonitor(ctx)
|
|
}
|
|
|
|
func (h *AlertHandlers) broadcastStateForContext(ctx context.Context) {
|
|
if h.wsHub == nil {
|
|
return
|
|
}
|
|
|
|
orgID := apicontext.OrgID(ctx)
|
|
if orgID != "" {
|
|
if h.wsHub.GetTenantClientCount(orgID) == 0 {
|
|
return
|
|
}
|
|
} else if h.wsHub.GetClientCount() == 0 {
|
|
return
|
|
}
|
|
|
|
monitor := h.getMonitor(ctx)
|
|
if monitor == nil {
|
|
return
|
|
}
|
|
if orgID != "" {
|
|
h.wsHub.BroadcastCurrentStateToTenant(orgID)
|
|
return
|
|
}
|
|
h.wsHub.BroadcastCurrentState()
|
|
}
|
|
|
|
// validateAlertIdentifier validates an alert identifier for security.
|
|
// Alert identifiers may contain user-supplied data (e.g., Docker hostnames), so we allow
|
|
// printable ASCII characters while blocking control characters and path traversal.
|
|
func validateAlertIdentifier(alertIdentifier string) bool {
|
|
// Guard against empty strings or extremely large payloads that could impact memory usage.
|
|
if len(alertIdentifier) == 0 || len(alertIdentifier) > 500 {
|
|
return false
|
|
}
|
|
|
|
// Reject attempts to traverse directories via crafted path segments.
|
|
if strings.Contains(alertIdentifier, "../") || strings.Contains(alertIdentifier, "/..") {
|
|
return false
|
|
}
|
|
|
|
// Allow printable ASCII characters (32-126) to support user-supplied identifiers
|
|
// like Docker hostnames that may contain parentheses, brackets, etc.
|
|
// Reject control characters (0-31) and DEL (127) which could cause logging issues.
|
|
for _, r := range alertIdentifier {
|
|
if r < 32 || r > 126 {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// Reject IDs that start or end with spaces (likely malformed)
|
|
if alertIdentifier[0] == ' ' || alertIdentifier[len(alertIdentifier)-1] == ' ' {
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// GetAlertConfig returns the current alert configuration
|
|
func (h *AlertHandlers) GetAlertConfig(w http.ResponseWriter, r *http.Request) {
|
|
config := h.getMonitor(r.Context()).GetAlertManager().GetConfig()
|
|
|
|
if err := utils.WriteJSONResponse(w, config); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write alert config response")
|
|
}
|
|
}
|
|
|
|
// GetDeadManStatus returns the live external-watchdog read model without
|
|
// echoing the secret-bearing ping URL.
|
|
func (h *AlertHandlers) GetDeadManStatus(w http.ResponseWriter, r *http.Request) {
|
|
monitor := h.getMonitor(r.Context())
|
|
if monitor == nil {
|
|
http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if err := utils.WriteJSONResponse(w, monitor.DeadManStatus()); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write dead-man status response")
|
|
}
|
|
}
|
|
|
|
const deadManRedactedPingURL = "***REDACTED***"
|
|
|
|
type deadManConfigView struct {
|
|
PingURL string `json:"pingUrl"`
|
|
Configured bool `json:"configured"`
|
|
}
|
|
|
|
// GetDeadManConfig returns only a presence marker for the credential-bearing
|
|
// ping URL. The sentinel supports an edit round-trip without disclosing it.
|
|
func (h *AlertHandlers) GetDeadManConfig(w http.ResponseWriter, r *http.Request) {
|
|
monitor := h.getMonitor(r.Context())
|
|
if monitor == nil {
|
|
http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
configured := strings.TrimSpace(monitor.DeadManConfig().PingURL) != ""
|
|
view := deadManConfigView{Configured: configured}
|
|
if configured {
|
|
view.PingURL = deadManRedactedPingURL
|
|
}
|
|
if err := utils.WriteJSONResponse(w, view); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write dead-man config response")
|
|
}
|
|
}
|
|
|
|
// UpdateDeadManConfig validates and durably encrypts the watchdog destination
|
|
// before its live worker is reconfigured.
|
|
func (h *AlertHandlers) UpdateDeadManConfig(w http.ResponseWriter, r *http.Request) {
|
|
monitor := h.getMonitor(r.Context())
|
|
if monitor == nil {
|
|
http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
r.Body = http.MaxBytesReader(w, r.Body, notifications.MaxDeadManPingURLLength+1024)
|
|
var request deadManConfigView
|
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
pingURL := strings.TrimSpace(request.PingURL)
|
|
if pingURL == deadManRedactedPingURL {
|
|
pingURL = monitor.DeadManConfig().PingURL
|
|
}
|
|
config := notifications.DeadManConfig{PingURL: pingURL}
|
|
if err := notifications.ValidateDeadManPingURL(config.PingURL); err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if err := monitor.UpdateDeadManConfig(config); err != nil {
|
|
log.Error().Err(err).Msg("Failed to update dead-man configuration")
|
|
http.Error(w, "Failed to save external watchdog configuration", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
|
"success": true,
|
|
"configured": config.PingURL != "",
|
|
}); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write dead-man config update response")
|
|
}
|
|
}
|
|
|
|
// UpdateAlertConfig updates the alert configuration
|
|
func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request) {
|
|
// Config size scales with the fleet: every per-resource toggle adds an
|
|
// Overrides entry keyed by a long resource ID, so 64KB rejected saves
|
|
// from instances with a few hundred disabled containers (#1601).
|
|
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
|
|
|
body, err := io.ReadAll(r.Body)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
// One save at a time, so a slower save cannot write an older config to
|
|
// disk after a newer one was applied.
|
|
h.configSaveMu.Lock()
|
|
defer h.configSaveMu.Unlock()
|
|
|
|
// Keys the client left out keep their stored values, so a thresholds
|
|
// save no longer turns off flapping detection or alert TTL cleanup.
|
|
updatedConfig, err := h.getMonitor(r.Context()).GetAlertManager().ApplyConfigUpdate(body)
|
|
if errors.Is(err, alerts.ErrConfigSnapshot) {
|
|
log.Error().Err(err).Msg("Failed to snapshot applied alert configuration")
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Update notification manager with schedule settings
|
|
notificationMgr := h.getMonitor(r.Context()).GetNotificationManager()
|
|
notificationMgr.SetEnabled(updatedConfig.Enabled && updatedConfig.ActivationState == alerts.ActivationActive)
|
|
notificationMgr.SetCooldown(updatedConfig.Schedule.Cooldown)
|
|
notificationMgr.SetGroupingConfig(
|
|
updatedConfig.Schedule.Grouping.Enabled,
|
|
updatedConfig.Schedule.Grouping.Window,
|
|
updatedConfig.Schedule.Grouping.ByNode,
|
|
updatedConfig.Schedule.Grouping.ByGuest,
|
|
)
|
|
notificationMgr.SetInitialNotifyTarget(updatedConfig.Schedule.InitialNotify)
|
|
notificationMgr.SetNotifyOnResolve(updatedConfig.Schedule.NotifyOnResolve)
|
|
|
|
// Save to persistent storage. Failure here used to be swallowed (logged
|
|
// and reported "success"), which let the in-memory state diverge from
|
|
// what was actually persisted. On the next restart or config reload,
|
|
// the override silently vanished. Surface the error so the client can
|
|
// show a real save-failed signal.
|
|
if err := h.getMonitor(r.Context()).GetConfigPersistence().SaveAlertConfig(updatedConfig); err != nil {
|
|
log.Error().Err(err).Msg("Failed to save alert configuration")
|
|
http.Error(w, fmt.Sprintf("Failed to save alert configuration: %v", err), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
|
"success": true,
|
|
"message": "Alert configuration updated successfully",
|
|
}); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write alert config update response")
|
|
}
|
|
}
|
|
|
|
// ActivateAlerts activates alert notifications
|
|
func (h *AlertHandlers) ActivateAlerts(w http.ResponseWriter, r *http.Request) {
|
|
// Get current config
|
|
config := h.getMonitor(r.Context()).GetAlertManager().GetConfig()
|
|
|
|
// Check if already active
|
|
if config.ActivationState == alerts.ActivationActive {
|
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
|
"success": true,
|
|
"message": "Alerts already activated",
|
|
"state": string(config.ActivationState),
|
|
"activationTime": config.ActivationTime,
|
|
}); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write activate response")
|
|
}
|
|
return
|
|
}
|
|
|
|
// Activate notifications
|
|
now := time.Now()
|
|
config.ActivationState = alerts.ActivationActive
|
|
config.ActivationTime = &now
|
|
|
|
// Update config
|
|
h.getMonitor(r.Context()).GetAlertManager().UpdateConfig(config)
|
|
h.getMonitor(r.Context()).GetNotificationManager().SetEnabled(
|
|
config.Enabled && config.ActivationState == alerts.ActivationActive,
|
|
)
|
|
|
|
// Save to persistent storage
|
|
if err := h.getMonitor(r.Context()).GetConfigPersistence().SaveAlertConfig(config); err != nil {
|
|
log.Error().Err(err).Msg("Failed to save alert configuration after activation")
|
|
http.Error(w, "Failed to save configuration", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
// Notify about existing critical alerts after activation
|
|
activeAlerts := h.getMonitor(r.Context()).GetAlertManager().GetActiveAlerts()
|
|
criticalCount := 0
|
|
for _, alert := range activeAlerts {
|
|
if alert.Level == alerts.AlertLevelCritical && !alert.Acknowledged {
|
|
// Re-dispatch critical alerts to trigger notifications
|
|
h.getMonitor(r.Context()).GetAlertManager().NotifyExistingAlert(alert.ID)
|
|
criticalCount++
|
|
}
|
|
}
|
|
|
|
if criticalCount > 0 {
|
|
log.Info().
|
|
Int("criticalAlerts", criticalCount).
|
|
Msg("Sent notifications for existing critical alerts after activation")
|
|
}
|
|
|
|
log.Info().Msg("Alert notifications activated")
|
|
|
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
|
"success": true,
|
|
"message": "Alert notifications activated",
|
|
"state": string(config.ActivationState),
|
|
"activationTime": config.ActivationTime,
|
|
}); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write activate response")
|
|
}
|
|
}
|
|
|
|
// GetActiveAlerts returns all active alerts
|
|
func (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request) {
|
|
alerts := h.getMonitor(r.Context()).GetAlertManager().GetActiveAlerts()
|
|
|
|
if err := utils.WriteJSONResponse(w, alerts); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write active alerts response")
|
|
}
|
|
}
|
|
|
|
func (h *AlertHandlers) GetAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
|
|
h.intentPolicyMu.Lock()
|
|
defer h.intentPolicyMu.Unlock()
|
|
manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)
|
|
if !ok {
|
|
http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if err := utils.WriteJSONResponse(w, manager.GetIntentPolicies()); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write alert intent policies response")
|
|
}
|
|
}
|
|
|
|
func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
|
|
// Per-resource policy rules scale with the fleet the same way alert
|
|
// config overrides do, so this shares their 1MB bound.
|
|
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
|
var document alerts.AlertIntentPolicyDocument
|
|
decoder := json.NewDecoder(r.Body)
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(&document); err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
|
http.Error(w, "request body must contain one JSON object", http.StatusBadRequest)
|
|
return
|
|
}
|
|
// Treat revision check, in-memory install, and durable save as one API
|
|
// transaction. This also prevents a failed earlier writer from rolling
|
|
// back a later successful update.
|
|
h.intentPolicyMu.Lock()
|
|
defer h.intentPolicyMu.Unlock()
|
|
monitor := h.getMonitor(r.Context())
|
|
manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)
|
|
if !ok {
|
|
http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
persistence, ok := monitor.GetConfigPersistence().(alertIntentPolicyPersistence)
|
|
if !ok {
|
|
http.Error(w, "alert intent policy persistence is unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
previous := manager.GetIntentPolicies()
|
|
updated, err := manager.UpdateIntentPolicies(document)
|
|
if err != nil {
|
|
if errors.Is(err, alerts.ErrAlertIntentPolicyRevisionConflict) {
|
|
http.Error(w, err.Error(), http.StatusConflict)
|
|
return
|
|
}
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if err := persistence.SaveAlertIntentPolicies(updated); err != nil {
|
|
if rollbackErr := manager.LoadIntentPolicies(previous); rollbackErr != nil {
|
|
log.Error().Err(rollbackErr).Msg("Failed to roll back in-memory alert intent policies after persistence failure")
|
|
}
|
|
http.Error(w, fmt.Sprintf("Failed to save alert intent policies: %v", err), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
if err := utils.WriteJSONResponse(w, updated); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write updated alert intent policies response")
|
|
}
|
|
}
|
|
|
|
func (h *AlertHandlers) PreviewAlertIntentPolicy(w http.ResponseWriter, r *http.Request) {
|
|
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
|
|
var request alerts.AlertIntentPolicyPreviewRequest
|
|
decoder := json.NewDecoder(r.Body)
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(&request); err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
|
http.Error(w, "request body must contain one JSON object", http.StatusBadRequest)
|
|
return
|
|
}
|
|
h.intentPolicyMu.Lock()
|
|
defer h.intentPolicyMu.Unlock()
|
|
manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)
|
|
if !ok {
|
|
http.Error(w, "alert intent policy preview is unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
preview, err := manager.PreviewIntentPolicy(request)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if err := utils.WriteJSONResponse(w, preview); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write alert intent policy preview response")
|
|
}
|
|
}
|
|
|
|
// GetAlertDeliveryDiagnosis explains current notification delivery policy
|
|
// without sending a notification or mutating delivery state. With an
|
|
// alertIdentifier query parameter it returns one diagnosis object; without
|
|
// one it returns the diagnosis array for every active alert in a single
|
|
// pass so list surfaces do not need a request per alert.
|
|
func (h *AlertHandlers) GetAlertDeliveryDiagnosis(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
alertIdentifier := strings.TrimSpace(r.URL.Query().Get("alertIdentifier"))
|
|
if alertIdentifier == "" {
|
|
alertIdentifier = strings.TrimSpace(r.URL.Query().Get("id"))
|
|
}
|
|
if alertIdentifier != "" && !validateAlertIdentifier(alertIdentifier) {
|
|
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
monitor := h.getMonitor(r.Context())
|
|
if monitor == nil {
|
|
http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
manager := monitor.GetAlertManager()
|
|
if manager == nil {
|
|
http.Error(w, "alert manager is not initialized", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
if alertIdentifier == "" {
|
|
diagnoses := manager.DiagnoseActiveAlertDeliveries()
|
|
if diagnoses == nil {
|
|
diagnoses = []alerts.AlertDeliveryDiagnosis{}
|
|
}
|
|
if err := utils.WriteJSONResponse(w, diagnoses); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write alert delivery diagnoses response")
|
|
}
|
|
return
|
|
}
|
|
|
|
diagnosis, exists := manager.DiagnoseAlertDelivery(alertIdentifier)
|
|
if !exists {
|
|
http.Error(w, "alert not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
if err := utils.WriteJSONResponse(w, diagnosis); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write alert delivery diagnosis response")
|
|
}
|
|
}
|
|
|
|
// GetAlertEvents returns entries from the append-only alert event log:
|
|
// lifecycle transitions and notification decisions, including suppressions
|
|
// with reasons. Newest first.
|
|
func (h *AlertHandlers) GetAlertEvents(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
query := r.URL.Query()
|
|
filter := eventlog.Filter{
|
|
AlertID: strings.TrimSpace(query.Get("alertIdentifier")),
|
|
}
|
|
if filter.AlertID != "" && !validateAlertIdentifier(filter.AlertID) {
|
|
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
|
|
return
|
|
}
|
|
for _, raw := range query["type"] {
|
|
for _, part := range strings.Split(raw, ",") {
|
|
if trimmed := strings.TrimSpace(part); trimmed != "" {
|
|
filter.Types = append(filter.Types, trimmed)
|
|
}
|
|
}
|
|
}
|
|
if since := strings.TrimSpace(query.Get("since")); since != "" {
|
|
parsed, err := time.Parse(time.RFC3339, since)
|
|
if err != nil {
|
|
http.Error(w, "since must be RFC3339", http.StatusBadRequest)
|
|
return
|
|
}
|
|
filter.Since = parsed
|
|
}
|
|
if limitRaw := strings.TrimSpace(query.Get("limit")); limitRaw != "" {
|
|
limit, err := strconv.Atoi(limitRaw)
|
|
if err != nil || limit <= 0 {
|
|
http.Error(w, "limit must be a positive integer", http.StatusBadRequest)
|
|
return
|
|
}
|
|
filter.Limit = limit
|
|
}
|
|
|
|
monitor := h.getMonitor(r.Context())
|
|
if monitor == nil {
|
|
http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
manager := monitor.GetAlertManager()
|
|
if manager == nil {
|
|
http.Error(w, "alert manager is not initialized", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
events, err := manager.AlertEvents(filter)
|
|
if err != nil {
|
|
log.Error().Err(err).Msg("Failed to query alert event log")
|
|
http.Error(w, "failed to query alert events", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
if events == nil {
|
|
events = []eventlog.Event{}
|
|
}
|
|
if err := utils.WriteJSONResponse(w, events); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write alert events response")
|
|
}
|
|
}
|
|
|
|
// GetAlertHistory returns alert history
|
|
func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request) {
|
|
query := r.URL.Query()
|
|
|
|
limit := 100
|
|
if limitStr := query.Get("limit"); limitStr != "" {
|
|
l, err := strconv.Atoi(limitStr)
|
|
switch {
|
|
case err != nil:
|
|
log.Warn().Str("limit", limitStr).Msg("Invalid limit parameter, using default")
|
|
case l < 0:
|
|
http.Error(w, "limit must be non-negative", http.StatusBadRequest)
|
|
return
|
|
case l == 0:
|
|
limit = 0
|
|
case l > 10000:
|
|
log.Warn().Int("limit", l).Msg("Limit exceeds maximum, capping at 10000")
|
|
limit = 10000
|
|
default:
|
|
limit = l
|
|
}
|
|
}
|
|
|
|
offset := 0
|
|
if offsetStr := query.Get("offset"); offsetStr != "" {
|
|
if o, err := strconv.Atoi(offsetStr); err == nil {
|
|
if o < 0 {
|
|
http.Error(w, "offset must be non-negative", http.StatusBadRequest)
|
|
return
|
|
}
|
|
offset = o
|
|
} else {
|
|
log.Warn().Str("offset", offsetStr).Msg("Invalid offset parameter, ignoring")
|
|
}
|
|
}
|
|
|
|
var startTime *time.Time
|
|
if startStr := query.Get("startTime"); startStr != "" {
|
|
parsed, err := time.Parse(time.RFC3339, startStr)
|
|
if err != nil {
|
|
http.Error(w, "invalid startTime parameter", http.StatusBadRequest)
|
|
return
|
|
}
|
|
startTime = &parsed
|
|
}
|
|
|
|
var endTime *time.Time
|
|
if endStr := query.Get("endTime"); endStr != "" {
|
|
parsed, err := time.Parse(time.RFC3339, endStr)
|
|
if err != nil {
|
|
http.Error(w, "invalid endTime parameter", http.StatusBadRequest)
|
|
return
|
|
}
|
|
endTime = &parsed
|
|
}
|
|
|
|
if startTime != nil && endTime != nil && endTime.Before(*startTime) {
|
|
http.Error(w, "endTime must be after startTime", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
severity := strings.ToLower(strings.TrimSpace(query.Get("severity")))
|
|
switch severity {
|
|
case "", "all":
|
|
severity = ""
|
|
case "info", "warning", "critical":
|
|
default:
|
|
log.Warn().Str("severity", severity).Msg("Invalid severity filter, ignoring")
|
|
severity = ""
|
|
}
|
|
|
|
resourceID := strings.TrimSpace(query.Get("resourceId"))
|
|
|
|
matchesFilters := func(alertTime time.Time, alertLevel string, alertResourceID string) bool {
|
|
if startTime != nil && alertTime.Before(*startTime) {
|
|
return false
|
|
}
|
|
if endTime != nil && alertTime.After(*endTime) {
|
|
return false
|
|
}
|
|
if severity != "" && !strings.EqualFold(alertLevel, severity) {
|
|
return false
|
|
}
|
|
if resourceID != "" && alertResourceID != resourceID {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
trimAlerts := func(alerts []alerts.Alert) []alerts.Alert {
|
|
if offset > 0 {
|
|
if offset >= len(alerts) {
|
|
return alerts[:0]
|
|
}
|
|
alerts = alerts[offset:]
|
|
}
|
|
if limit > 0 && len(alerts) > limit {
|
|
alerts = alerts[:limit]
|
|
}
|
|
return alerts
|
|
}
|
|
|
|
trimMockAlerts := func(alerts []models.Alert) []models.Alert {
|
|
if offset > 0 {
|
|
if offset >= len(alerts) {
|
|
return alerts[:0]
|
|
}
|
|
alerts = alerts[offset:]
|
|
}
|
|
if limit > 0 && len(alerts) > limit {
|
|
alerts = alerts[:limit]
|
|
}
|
|
return alerts
|
|
}
|
|
|
|
// Check if mock mode is enabled
|
|
mockEnabled := mock.IsMockEnabled()
|
|
log.Debug().Bool("mockEnabled", mockEnabled).Msg("GetAlertHistory: mock mode status")
|
|
|
|
fetchLimit := limit
|
|
if fetchLimit > 0 && offset > 0 {
|
|
fetchLimit += offset
|
|
}
|
|
|
|
if mockEnabled {
|
|
mockHistory := mock.GetMockAlertHistory(fetchLimit)
|
|
filtered := make([]models.Alert, 0, len(mockHistory))
|
|
for _, alert := range mockHistory {
|
|
if matchesFilters(alert.StartTime, alert.Level, alert.ResourceID) {
|
|
filtered = append(filtered, alert)
|
|
}
|
|
}
|
|
filtered = trimMockAlerts(filtered)
|
|
if err := utils.WriteJSONResponse(w, filtered); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write mock alert history response")
|
|
}
|
|
return
|
|
}
|
|
|
|
if h.getMonitor(r.Context()) == nil {
|
|
http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
manager := h.getMonitor(r.Context()).GetAlertManager()
|
|
var history []alerts.Alert
|
|
if startTime != nil {
|
|
history = manager.GetAlertHistorySince(*startTime, fetchLimit)
|
|
} else {
|
|
history = manager.GetAlertHistory(fetchLimit)
|
|
}
|
|
|
|
filtered := make([]alerts.Alert, 0, len(history))
|
|
for _, alert := range history {
|
|
if matchesFilters(alert.StartTime, string(alert.Level), alert.ResourceID) {
|
|
filtered = append(filtered, alert)
|
|
}
|
|
}
|
|
filtered = trimAlerts(filtered)
|
|
|
|
if err := utils.WriteJSONResponse(w, filtered); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write alert history response")
|
|
}
|
|
}
|
|
|
|
// GetAlertIncidentTimeline returns the incident timeline for an alert or resource.
|
|
func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
query := r.URL.Query()
|
|
alertID := strings.TrimSpace(query.Get("alertIdentifier"))
|
|
resourceID := strings.TrimSpace(query.Get("resource_id"))
|
|
startedAtRaw := strings.TrimSpace(query.Get("started_at"))
|
|
if startedAtRaw == "" {
|
|
startedAtRaw = strings.TrimSpace(query.Get("start_time"))
|
|
}
|
|
limit := 0
|
|
if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" {
|
|
if parsed, err := strconv.Atoi(rawLimit); err == nil && parsed > 0 {
|
|
limit = parsed
|
|
}
|
|
}
|
|
|
|
if alertID != "" {
|
|
if !validateAlertIdentifier(alertID) {
|
|
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
|
|
return
|
|
}
|
|
var startedAt time.Time
|
|
if startedAtRaw != "" {
|
|
parsed, err := time.Parse(time.RFC3339, startedAtRaw)
|
|
if err != nil {
|
|
http.Error(w, "Invalid started_at time", http.StatusBadRequest)
|
|
return
|
|
}
|
|
startedAt = parsed
|
|
}
|
|
if mockIncident := mock.GetMockAlertIncidentTimeline(alertID, startedAt); mockIncident != nil {
|
|
if err := utils.WriteJSONResponse(w, exportIncident(mockIncident)); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write mock incident timeline response")
|
|
}
|
|
return
|
|
}
|
|
|
|
store := h.getMonitor(r.Context()).GetIncidentStore()
|
|
if store == nil {
|
|
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
page, err := store.QueryIncidents(memory.IncidentQuery{AlertIdentifier: alertID, StartedAt: startedAt, Limit: 1})
|
|
if err != nil {
|
|
http.Error(w, "Incident history unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
var incident *memory.Incident
|
|
if len(page.Incidents) > 0 {
|
|
incident = page.Incidents[0]
|
|
}
|
|
|
|
if !startedAt.IsZero() && (incident == nil || len(incident.Events) == 0) {
|
|
if alert, resolvedAt := findAlertOccurrenceForTimeline(h.getMonitor(r.Context()).GetAlertManager(), alertID, startedAt); alert != nil {
|
|
incident = store.EnsureAlertOccurrence(alert, resolvedAt)
|
|
}
|
|
}
|
|
if err := utils.WriteJSONResponse(w, exportIncident(incident)); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write incident timeline response")
|
|
}
|
|
return
|
|
}
|
|
|
|
if resourceID != "" {
|
|
if len(resourceID) > 500 {
|
|
http.Error(w, "Invalid resource ID", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if mock.IsMockEnabled() {
|
|
incidents := mock.GetMockAlertIncidentsForResource(resourceID, limit)
|
|
if err := utils.WriteJSONResponse(w, exportIncidents(incidents)); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write mock incident list response")
|
|
}
|
|
return
|
|
}
|
|
store := h.getMonitor(r.Context()).GetIncidentStore()
|
|
if store == nil {
|
|
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
page, err := store.QueryIncidents(memory.IncidentQuery{ResourceID: resourceID, Limit: limit})
|
|
if err != nil {
|
|
http.Error(w, "Incident history unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
if err := utils.WriteJSONResponse(w, exportIncidents(page.Incidents)); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write incident list response")
|
|
}
|
|
return
|
|
}
|
|
|
|
http.Error(w, "Missing alertIdentifier or resource_id", http.StatusBadRequest)
|
|
}
|
|
|
|
func findAlertOccurrenceForTimeline(manager AlertManager, alertID string, startedAt time.Time) (*alerts.Alert, *time.Time) {
|
|
if manager == nil || strings.TrimSpace(alertID) == "" {
|
|
return nil, nil
|
|
}
|
|
|
|
for _, active := range manager.GetActiveAlerts() {
|
|
if alertOccurrenceMatches(&active, alertID, startedAt) {
|
|
return active.Clone(), nil
|
|
}
|
|
}
|
|
|
|
for _, historical := range manager.GetAlertHistory(0) {
|
|
if !alertOccurrenceMatches(&historical, alertID, startedAt) {
|
|
continue
|
|
}
|
|
resolvedAt := historical.LastSeen
|
|
if historical.OperationalRecord != nil && historical.OperationalRecord.ResolvedAt != nil && !historical.OperationalRecord.ResolvedAt.IsZero() {
|
|
resolvedAt = *historical.OperationalRecord.ResolvedAt
|
|
}
|
|
if resolvedAt.IsZero() {
|
|
return historical.Clone(), nil
|
|
}
|
|
if !historical.StartTime.IsZero() && resolvedAt.Before(historical.StartTime) {
|
|
resolvedAt = historical.StartTime
|
|
}
|
|
return historical.Clone(), &resolvedAt
|
|
}
|
|
|
|
return nil, nil
|
|
}
|
|
|
|
func alertOccurrenceMatches(alert *alerts.Alert, alertID string, startedAt time.Time) bool {
|
|
if alert == nil || strings.TrimSpace(alert.ID) != strings.TrimSpace(alertID) {
|
|
return false
|
|
}
|
|
if startedAt.IsZero() || alert.StartTime.IsZero() {
|
|
return true
|
|
}
|
|
delta := alert.StartTime.Sub(startedAt)
|
|
if delta < 0 {
|
|
delta = -delta
|
|
}
|
|
return delta <= time.Second
|
|
}
|
|
|
|
// SaveAlertIncidentNote stores a user note in the incident timeline.
|
|
func (h *AlertHandlers) SaveAlertIncidentNote(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
r.Body = http.MaxBytesReader(w, r.Body, 8*1024)
|
|
var req struct {
|
|
AlertIdentifier string `json:"alertIdentifier"`
|
|
IncidentID string `json:"incident_id"`
|
|
Note string `json:"note"`
|
|
User string `json:"user,omitempty"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
req.AlertIdentifier = strings.TrimSpace(req.AlertIdentifier)
|
|
req.IncidentID = strings.TrimSpace(req.IncidentID)
|
|
req.Note = strings.TrimSpace(req.Note)
|
|
req.User = strings.TrimSpace(req.User)
|
|
|
|
alertIdentifier := req.AlertIdentifier
|
|
|
|
if alertIdentifier == "" && req.IncidentID == "" {
|
|
http.Error(w, "alertIdentifier or incident_id is required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if alertIdentifier != "" && !validateAlertIdentifier(alertIdentifier) {
|
|
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if req.Note == "" {
|
|
http.Error(w, "note is required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if mock.AddMockAlertIncidentNote(alertIdentifier, req.IncidentID, req.Note, req.User) {
|
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{"success": true}); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write mock incident note response")
|
|
}
|
|
return
|
|
}
|
|
|
|
store := h.getMonitor(r.Context()).GetIncidentStore()
|
|
if store == nil {
|
|
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
if ok := store.RecordNote(alertIdentifier, req.IncidentID, req.Note, req.User); !ok {
|
|
http.Error(w, "Failed to save note", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
|
"success": true,
|
|
}); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write incident note response")
|
|
}
|
|
}
|
|
|
|
type incidentEventView struct {
|
|
Source string `json:"source,omitempty"`
|
|
Evidence *unifiedresources.ResourceChange `json:"evidence,omitempty"`
|
|
ID string `json:"id"`
|
|
Type memory.IncidentEventType `json:"type"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
Summary string `json:"summary"`
|
|
Details map[string]interface{} `json:"details"`
|
|
}
|
|
|
|
func (v incidentEventView) NormalizeCollections() incidentEventView {
|
|
if v.Details == nil {
|
|
v.Details = map[string]interface{}{}
|
|
}
|
|
return v
|
|
}
|
|
|
|
type incidentView struct {
|
|
History *memory.IncidentHistoryCoverage `json:"history,omitempty"`
|
|
ID string `json:"id"`
|
|
AlertIdentifier string `json:"alertIdentifier"`
|
|
AlertType string `json:"alertType"`
|
|
Level string `json:"level"`
|
|
ResourceID string `json:"resourceId"`
|
|
ResourceName string `json:"resourceName"`
|
|
ResourceType string `json:"resourceType,omitempty"`
|
|
Node string `json:"node,omitempty"`
|
|
Instance string `json:"instance,omitempty"`
|
|
Message string `json:"message,omitempty"`
|
|
Status memory.IncidentStatus `json:"status"`
|
|
OpenedAt time.Time `json:"openedAt"`
|
|
ClosedAt *time.Time `json:"closedAt,omitempty"`
|
|
Acknowledged bool `json:"acknowledged"`
|
|
AckUser string `json:"ackUser,omitempty"`
|
|
AckTime *time.Time `json:"ackTime,omitempty"`
|
|
Events []incidentEventView `json:"events"`
|
|
}
|
|
|
|
func emptyIncidentView() incidentView {
|
|
return incidentView{}.NormalizeCollections()
|
|
}
|
|
|
|
func (v incidentView) NormalizeCollections() incidentView {
|
|
if v.Events == nil {
|
|
v.Events = []incidentEventView{}
|
|
}
|
|
for i := range v.Events {
|
|
v.Events[i] = v.Events[i].NormalizeCollections()
|
|
}
|
|
return v
|
|
}
|
|
|
|
func exportIncident(incident *memory.Incident) *incidentView {
|
|
if incident == nil {
|
|
return nil
|
|
}
|
|
events := make([]incidentEventView, 0, len(incident.Events))
|
|
for _, event := range incident.Events {
|
|
events = append(events, incidentEventView{
|
|
Source: event.Source, Evidence: event.Evidence,
|
|
ID: event.ID,
|
|
Type: event.Type,
|
|
Timestamp: event.Timestamp,
|
|
Summary: event.Summary,
|
|
Details: event.Details,
|
|
}.NormalizeCollections())
|
|
}
|
|
alertIdentifier := strings.TrimSpace(incident.AlertIdentifier)
|
|
view := emptyIncidentView()
|
|
view.ID = incident.ID
|
|
view.History = incident.History
|
|
view.AlertIdentifier = alertIdentifier
|
|
view.AlertType = incident.AlertType
|
|
view.Level = incident.Level
|
|
view.ResourceID = incident.ResourceID
|
|
view.ResourceName = incident.ResourceName
|
|
view.ResourceType = incident.ResourceType
|
|
view.Node = incident.Node
|
|
view.Instance = incident.Instance
|
|
view.Message = incident.Message
|
|
view.Status = incident.Status
|
|
view.OpenedAt = incident.OpenedAt
|
|
view.ClosedAt = incident.ClosedAt
|
|
view.Acknowledged = incident.Acknowledged
|
|
view.AckUser = incident.AckUser
|
|
view.AckTime = incident.AckTime
|
|
view.Events = events
|
|
returnValue := view.NormalizeCollections()
|
|
return &returnValue
|
|
}
|
|
|
|
func exportIncidents(incidents []*memory.Incident) []*incidentView {
|
|
if len(incidents) == 0 {
|
|
return []*incidentView{}
|
|
}
|
|
exported := make([]*incidentView, 0, len(incidents))
|
|
for _, incident := range incidents {
|
|
if view := exportIncident(incident); view != nil {
|
|
exported = append(exported, view)
|
|
}
|
|
}
|
|
return exported
|
|
}
|
|
|
|
// ClearAlertHistory clears all alert history
|
|
func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {
|
|
if err := h.getMonitor(r.Context()).GetAlertManager().ClearAlertHistory(); err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
if err := utils.WriteJSONResponse(w, map[string]string{"status": "success", "message": "Alert history cleared"}); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write alert history clear confirmation")
|
|
}
|
|
}
|
|
|
|
// alertIdentifierRequest is used for endpoints that accept one canonical alert identifier in the request body.
|
|
type alertIdentifierRequest struct {
|
|
AlertIdentifier string `json:"alertIdentifier"`
|
|
}
|
|
|
|
type snoozeAlertRequest struct {
|
|
AlertIdentifier string `json:"alertIdentifier"`
|
|
Until time.Time `json:"until"`
|
|
}
|
|
|
|
func (r alertIdentifierRequest) identifier() string {
|
|
return strings.TrimSpace(r.AlertIdentifier)
|
|
}
|
|
|
|
type alertIdentifiersRequest struct {
|
|
AlertIdentifiers []string `json:"alertIdentifiers"`
|
|
User string `json:"user,omitempty"`
|
|
}
|
|
|
|
func (r alertIdentifiersRequest) identifiers() []string {
|
|
raw := r.AlertIdentifiers
|
|
identifiers := make([]string, 0, len(raw))
|
|
for _, identifier := range raw {
|
|
trimmed := strings.TrimSpace(identifier)
|
|
if trimmed != "" {
|
|
identifiers = append(identifiers, trimmed)
|
|
}
|
|
}
|
|
return identifiers
|
|
}
|
|
|
|
// AcknowledgeAlertByBody acknowledges an alert using a canonical alert identifier from the request body.
|
|
// POST /api/alerts/acknowledge with {"alertIdentifier": "alert-id"}
|
|
func (h *AlertHandlers) AcknowledgeAlertByBody(w http.ResponseWriter, r *http.Request) {
|
|
var req alertIdentifierRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
log.Error().Err(err).Msg("Failed to decode acknowledge request body")
|
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
alertID := req.identifier()
|
|
if alertID == "" {
|
|
http.Error(w, "Alert identifier is required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if !validateAlertIdentifier(alertID) {
|
|
log.Error().Str("alertIdentifier", alertID).Msg("Invalid alert identifier")
|
|
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Get the authenticated user from the auth middleware (set in X-Authenticated-User header)
|
|
user := w.Header().Get("X-Authenticated-User")
|
|
if user == "" {
|
|
user = "unknown" // Fallback if auth header not set (shouldn't happen in normal flow)
|
|
}
|
|
|
|
if err := h.getMonitor(r.Context()).GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {
|
|
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to acknowledge alert")
|
|
http.Error(w, err.Error(), http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
h.getMonitor(r.Context()).SyncAlertState()
|
|
|
|
log.Info().Str("alertID", alertID).Str("user", user).Msg("Alert acknowledged successfully")
|
|
|
|
if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {
|
|
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write acknowledge response")
|
|
}
|
|
|
|
if h.wsHub != nil {
|
|
ctx := r.Context()
|
|
go func() {
|
|
h.broadcastStateForContext(ctx)
|
|
}()
|
|
}
|
|
}
|
|
|
|
// UnacknowledgeAlertByBody unacknowledges an alert using a canonical alert identifier from the request body.
|
|
// POST /api/alerts/unacknowledge with {"alertIdentifier": "alert-id"}
|
|
func (h *AlertHandlers) UnacknowledgeAlertByBody(w http.ResponseWriter, r *http.Request) {
|
|
var req alertIdentifierRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
log.Error().Err(err).Msg("Failed to decode unacknowledge request body")
|
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
alertID := req.identifier()
|
|
if alertID == "" {
|
|
http.Error(w, "Alert identifier is required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if !validateAlertIdentifier(alertID) {
|
|
log.Error().Str("alertIdentifier", alertID).Msg("Invalid alert identifier")
|
|
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if err := h.getMonitor(r.Context()).GetAlertManager().UnacknowledgeAlert(alertID); err != nil {
|
|
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to unacknowledge alert")
|
|
http.Error(w, err.Error(), http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
h.getMonitor(r.Context()).SyncAlertState()
|
|
|
|
log.Info().Str("alertID", alertID).Msg("Alert unacknowledged successfully")
|
|
|
|
if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {
|
|
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write unacknowledge response")
|
|
}
|
|
|
|
if h.wsHub != nil {
|
|
ctx := r.Context()
|
|
go func() {
|
|
h.broadcastStateForContext(ctx)
|
|
}()
|
|
}
|
|
}
|
|
|
|
// SnoozeAlertByBody pauses this occurrence's notifications and escalations.
|
|
// POST /api/alerts/snooze with {"alertIdentifier":"alert-id","until":"RFC3339"}
|
|
func (h *AlertHandlers) SnoozeAlertByBody(w http.ResponseWriter, r *http.Request) {
|
|
r.Body = http.MaxBytesReader(w, r.Body, 16<<10)
|
|
var req snoozeAlertRequest
|
|
decoder := json.NewDecoder(r.Body)
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(&req); err != nil {
|
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
alertID := strings.TrimSpace(req.AlertIdentifier)
|
|
if alertID == "" || !validateAlertIdentifier(alertID) || req.Until.IsZero() {
|
|
http.Error(w, "Valid alert identifier and snooze expiry are required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
user := w.Header().Get("X-Authenticated-User")
|
|
if user == "" {
|
|
user = "unknown"
|
|
}
|
|
if err := h.getMonitor(r.Context()).GetAlertManager().SnoozeAlert(alertID, user, req.Until); err != nil {
|
|
status := http.StatusBadRequest
|
|
if errors.Is(err, alerts.ErrAlertNotFound) {
|
|
status = http.StatusNotFound
|
|
}
|
|
http.Error(w, err.Error(), status)
|
|
return
|
|
}
|
|
h.getMonitor(r.Context()).SyncAlertState()
|
|
if err := utils.WriteJSONResponse(w, map[string]any{"success": true, "snoozedUntil": req.Until.UTC()}); err != nil {
|
|
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write snooze response")
|
|
}
|
|
if h.wsHub != nil {
|
|
ctx := r.Context()
|
|
go h.broadcastStateForContext(ctx)
|
|
}
|
|
}
|
|
|
|
// UnsnoozeAlertByBody resumes normal policy without resolving the alert.
|
|
func (h *AlertHandlers) UnsnoozeAlertByBody(w http.ResponseWriter, r *http.Request) {
|
|
var req alertIdentifierRequest
|
|
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 16<<10)).Decode(&req); err != nil {
|
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
alertID := req.identifier()
|
|
if alertID == "" || !validateAlertIdentifier(alertID) {
|
|
http.Error(w, "Valid alert identifier is required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
user := w.Header().Get("X-Authenticated-User")
|
|
if user == "" {
|
|
user = "unknown"
|
|
}
|
|
if err := h.getMonitor(r.Context()).GetAlertManager().UnsnoozeAlert(alertID, user); err != nil {
|
|
status := http.StatusBadRequest
|
|
if errors.Is(err, alerts.ErrAlertNotFound) {
|
|
status = http.StatusNotFound
|
|
}
|
|
http.Error(w, err.Error(), status)
|
|
return
|
|
}
|
|
h.getMonitor(r.Context()).SyncAlertState()
|
|
if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {
|
|
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write unsnooze response")
|
|
}
|
|
if h.wsHub != nil {
|
|
ctx := r.Context()
|
|
go h.broadcastStateForContext(ctx)
|
|
}
|
|
}
|
|
|
|
// ClearAlertByBody clears an alert using a canonical alert identifier from the request body.
|
|
// POST /api/alerts/clear with {"alertIdentifier": "alert-id"}
|
|
func (h *AlertHandlers) ClearAlertByBody(w http.ResponseWriter, r *http.Request) {
|
|
var req alertIdentifierRequest
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
log.Error().Err(err).Msg("Failed to decode clear request body")
|
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
alertID := req.identifier()
|
|
if alertID == "" {
|
|
http.Error(w, "Alert identifier is required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if !validateAlertIdentifier(alertID) {
|
|
log.Error().Str("alertIdentifier", alertID).Msg("Invalid alert identifier")
|
|
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if !h.getMonitor(r.Context()).GetAlertManager().ClearAlert(alertID) {
|
|
http.Error(w, "Alert not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
h.getMonitor(r.Context()).SyncAlertState()
|
|
|
|
log.Info().Str("alertID", alertID).Msg("Alert cleared successfully")
|
|
|
|
if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {
|
|
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write clear response")
|
|
}
|
|
|
|
if h.wsHub != nil {
|
|
ctx := r.Context()
|
|
go func() {
|
|
h.broadcastStateForContext(ctx)
|
|
}()
|
|
}
|
|
}
|
|
|
|
// BulkAcknowledgeAlerts acknowledges multiple alerts at once
|
|
func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Request) {
|
|
// Identifier lists scale with active alert count, and an alert flood is
|
|
// exactly when ack-all gets used, so 32KB starved the one path meant to
|
|
// recover from it (#1601).
|
|
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
|
|
|
var request alertIdentifiersRequest
|
|
|
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
alertIdentifiers := request.identifiers()
|
|
if len(alertIdentifiers) == 0 {
|
|
http.Error(w, "No alert identifiers provided", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Get the authenticated user from the auth middleware - ignore request.User to prevent spoofing
|
|
user := w.Header().Get("X-Authenticated-User")
|
|
if user == "" {
|
|
user = "unknown" // Fallback if auth header not set (shouldn't happen in normal flow)
|
|
}
|
|
|
|
var (
|
|
results []map[string]interface{}
|
|
anySuccess bool
|
|
)
|
|
for _, alertID := range alertIdentifiers {
|
|
result := map[string]interface{}{
|
|
"alertIdentifier": alertID,
|
|
"success": true,
|
|
}
|
|
if err := h.getMonitor(r.Context()).GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {
|
|
result["success"] = false
|
|
result["error"] = err.Error()
|
|
} else {
|
|
anySuccess = true
|
|
}
|
|
results = append(results, result)
|
|
}
|
|
|
|
if anySuccess {
|
|
h.getMonitor(r.Context()).SyncAlertState()
|
|
}
|
|
|
|
// Send response immediately
|
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
|
"results": results,
|
|
}); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write bulk acknowledge response")
|
|
}
|
|
|
|
// Broadcast updated state to all WebSocket clients if any alerts were acknowledged
|
|
// Do this in a goroutine to avoid blocking the HTTP response
|
|
if h.wsHub != nil && anySuccess {
|
|
ctx := r.Context()
|
|
go func() {
|
|
h.broadcastStateForContext(ctx)
|
|
log.Debug().Msg("Broadcasted state after bulk alert acknowledgment")
|
|
}()
|
|
}
|
|
}
|
|
|
|
// BulkClearAlerts clears multiple alerts at once
|
|
func (h *AlertHandlers) BulkClearAlerts(w http.ResponseWriter, r *http.Request) {
|
|
// Same fleet-scaling bound as BulkAcknowledgeAlerts.
|
|
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
|
|
|
var request alertIdentifiersRequest
|
|
|
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
|
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
alertIdentifiers := request.identifiers()
|
|
if len(alertIdentifiers) == 0 {
|
|
http.Error(w, "No alert identifiers provided", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
var (
|
|
results []map[string]interface{}
|
|
anySuccess bool
|
|
)
|
|
for _, alertID := range alertIdentifiers {
|
|
result := map[string]interface{}{
|
|
"alertIdentifier": alertID,
|
|
"success": true,
|
|
}
|
|
if h.getMonitor(r.Context()).GetAlertManager().ClearAlert(alertID) {
|
|
anySuccess = true
|
|
} else {
|
|
result["success"] = false
|
|
result["error"] = "alert not found"
|
|
}
|
|
results = append(results, result)
|
|
}
|
|
|
|
if anySuccess {
|
|
h.getMonitor(r.Context()).SyncAlertState()
|
|
}
|
|
|
|
// Send response immediately
|
|
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
|
"results": results,
|
|
}); err != nil {
|
|
log.Error().Err(err).Msg("Failed to write bulk clear response")
|
|
}
|
|
|
|
// Broadcast updated state to all WebSocket clients after response
|
|
// Do this in a goroutine to avoid blocking the HTTP response
|
|
if h.wsHub != nil && anySuccess {
|
|
ctx := r.Context()
|
|
go func() {
|
|
h.broadcastStateForContext(ctx)
|
|
log.Debug().Msg("Broadcasted state after bulk alert clear")
|
|
}()
|
|
}
|
|
}
|
|
|
|
// HandleAlerts routes alert requests to appropriate handlers
|
|
func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
|
|
path := strings.TrimPrefix(r.URL.Path, "/api/alerts/")
|
|
|
|
log.Debug().
|
|
Str("originalPath", r.URL.Path).
|
|
Str("trimmedPath", path).
|
|
Str("method", r.Method).
|
|
Msg("HandleAlerts routing request")
|
|
|
|
switch {
|
|
case path == "config" && r.Method == http.MethodGet:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.GetAlertConfig(w, r)
|
|
case path == "config" && r.Method == http.MethodPut:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.UpdateAlertConfig(w, r)
|
|
case path == "deadman/status" && r.Method == http.MethodGet:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.GetDeadManStatus(w, r)
|
|
case path == "deadman/config" && r.Method == http.MethodGet:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.GetDeadManConfig(w, r)
|
|
case path == "deadman/config" && r.Method == http.MethodPut:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.UpdateDeadManConfig(w, r)
|
|
case path == "intent-policies" && r.Method == http.MethodGet:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.GetAlertIntentPolicies(w, r)
|
|
case path == "intent-policies" && r.Method == http.MethodPut:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.UpdateAlertIntentPolicies(w, r)
|
|
case path == "intent-policies/preview" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.PreviewAlertIntentPolicy(w, r)
|
|
case path == "activate" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.ActivateAlerts(w, r)
|
|
case path == "active" && r.Method == http.MethodGet:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.GetActiveAlerts(w, r)
|
|
case path == "delivery-diagnosis" && r.Method == http.MethodGet:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.GetAlertDeliveryDiagnosis(w, r)
|
|
case path == "events" && r.Method == http.MethodGet:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.GetAlertEvents(w, r)
|
|
case path == "history" && r.Method == http.MethodGet:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.GetAlertHistory(w, r)
|
|
case path == "incidents" && r.Method == http.MethodGet:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
|
return
|
|
}
|
|
h.GetAlertIncidentTimeline(w, r)
|
|
case path == "incidents/note" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.SaveAlertIncidentNote(w, r)
|
|
case path == "history" && r.Method == http.MethodDelete:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.ClearAlertHistory(w, r)
|
|
case path == "bulk/acknowledge" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.BulkAcknowledgeAlerts(w, r)
|
|
case path == "bulk/clear" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.BulkClearAlerts(w, r)
|
|
// Body-based endpoints (preferred - avoids URL encoding issues with reverse proxies)
|
|
case path == "acknowledge" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.AcknowledgeAlertByBody(w, r)
|
|
case path == "unacknowledge" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.UnacknowledgeAlertByBody(w, r)
|
|
case path == "snooze" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.SnoozeAlertByBody(w, r)
|
|
case path == "unsnooze" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.UnsnoozeAlertByBody(w, r)
|
|
case path == "clear" && r.Method == http.MethodPost:
|
|
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
|
return
|
|
}
|
|
h.ClearAlertByBody(w, r)
|
|
default:
|
|
http.Error(w, "Not found", http.StatusNotFound)
|
|
}
|
|
}
|