mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 02:55:51 +00:00
fix(alerts): raise fleet-scaling request body caps on config and bulk ack
Alert config grows one Overrides entry per toggled resource, so the 64KB cap on PUT /api/alerts/config rejected saves from instances with a few hundred disabled containers with 'http: request body too large'. The 32KB cap on bulk acknowledge/clear failed ack-all during large alert floods, the exact situation it exists for. Intent policies carry per-resource rules with the same scaling shape. All four now share a 1MB bound, which still caps memory per request but no longer rejects legitimate fleet-sized payloads. Endpoint tests pin a >64KB config save and a >32KB bulk ack at 200. Refs #1601 Contract-Neutral: raise alert config and bulk ack request body caps; no payload shape or field changes
This commit is contained in:
+13
-7
@@ -195,8 +195,10 @@ func (h *AlertHandlers) GetAlertConfig(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// UpdateAlertConfig updates the alert configuration
|
||||
func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request) {
|
||||
// Limit request body to 64KB to prevent memory exhaustion
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
|
||||
// Config size scales with the fleet: every per-resource toggle adds an
|
||||
// Overrides entry keyed by a long resource ID, so 64KB rejected saves
|
||||
// from instances with a few hundred disabled containers (#1601).
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
||||
|
||||
var config alerts.AlertConfig
|
||||
if err := json.NewDecoder(r.Body).Decode(&config); err != nil {
|
||||
@@ -327,7 +329,9 @@ func (h *AlertHandlers) GetAlertIntentPolicies(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 256*1024)
|
||||
// Per-resource policy rules scale with the fleet the same way alert
|
||||
// config overrides do, so this shares their 1MB bound.
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
||||
var document alerts.AlertIntentPolicyDocument
|
||||
decoder := json.NewDecoder(r.Body)
|
||||
decoder.DisallowUnknownFields()
|
||||
@@ -1018,8 +1022,10 @@ func (h *AlertHandlers) ClearAlertByBody(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
// BulkAcknowledgeAlerts acknowledges multiple alerts at once
|
||||
func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Request) {
|
||||
// Limit request body to 32KB to prevent memory exhaustion
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 32*1024)
|
||||
// Identifier lists scale with active alert count, and an alert flood is
|
||||
// exactly when ack-all gets used, so 32KB starved the one path meant to
|
||||
// recover from it (#1601).
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
||||
|
||||
var request alertIdentifiersRequest
|
||||
|
||||
@@ -1082,8 +1088,8 @@ func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Req
|
||||
|
||||
// BulkClearAlerts clears multiple alerts at once
|
||||
func (h *AlertHandlers) BulkClearAlerts(w http.ResponseWriter, r *http.Request) {
|
||||
// Limit request body to 32KB to prevent memory exhaustion
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 32*1024)
|
||||
// Same fleet-scaling bound as BulkAcknowledgeAlerts.
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
|
||||
|
||||
var request alertIdentifiersRequest
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package api_test
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
@@ -79,6 +80,36 @@ func TestAlertsEndpoints(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// A fleet with a few hundred per-resource toggles produces a config well
|
||||
// past the old 64KB body cap, which rejected legitimate saves (#1601).
|
||||
t.Run("UpdateAlertConfigLargeFleet", func(t *testing.T) {
|
||||
overrides := make(map[string]alerts.ThresholdConfig, 5000)
|
||||
for i := 0; i < 5000; i++ {
|
||||
key := fmt.Sprintf("docker:integration-host-%04d/container:web-frontend-replica-%04d", i, i)
|
||||
overrides[key] = alerts.ThresholdConfig{}
|
||||
}
|
||||
body, _ := json.Marshal(alerts.AlertConfig{Overrides: overrides})
|
||||
if len(body) <= 64*1024 {
|
||||
t.Fatalf("fixture too small to exercise the old 64KB cap: %d bytes", len(body))
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, srv.server.URL+"/api/alerts/config", bytes.NewBuffer(body))
|
||||
if err != nil {
|
||||
t.Fatalf("create request failed: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
t.Errorf("status code = %d, want %d", res.StatusCode, http.StatusOK)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("AlertIntentPolicies", func(t *testing.T) {
|
||||
res, err := http.Get(srv.server.URL + "/api/alerts/intent-policies")
|
||||
if err != nil {
|
||||
@@ -321,6 +352,35 @@ func TestAlertsEndpoints(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// Identifier lists scale with active alert count. The old 32KB cap made
|
||||
// ack-all fail during exactly the floods it exists to recover from (#1601).
|
||||
t.Run("BulkAcknowledgeLargeList", func(t *testing.T) {
|
||||
identifiers := make([]string, 2000)
|
||||
for i := range identifiers {
|
||||
identifiers[i] = fmt.Sprintf("docker:host-%04d/container:app-%04d-cpu", i, i)
|
||||
}
|
||||
jsonBody, _ := json.Marshal(map[string]interface{}{"alertIdentifiers": identifiers})
|
||||
if len(jsonBody) <= 32*1024 {
|
||||
t.Fatalf("fixture too small to exercise the old 32KB cap: %d bytes", len(jsonBody))
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, srv.server.URL+"/api/alerts/bulk/acknowledge", bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
t.Fatalf("create request failed: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("request failed: %v", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
t.Errorf("status code = %d, want %d", res.StatusCode, http.StatusOK)
|
||||
}
|
||||
})
|
||||
|
||||
// 9. Bulk Clear
|
||||
t.Run("BulkClear", func(t *testing.T) {
|
||||
body := map[string]interface{}{
|
||||
|
||||
Reference in New Issue
Block a user