fix(alerts): raise fleet-scaling request body caps on config and bulk ack

Alert config grows one Overrides entry per toggled resource, so the
64KB cap on PUT /api/alerts/config rejected saves from instances with
a few hundred disabled containers with 'http: request body too large'.
The 32KB cap on bulk acknowledge/clear failed ack-all during large
alert floods, the exact situation it exists for. Intent policies carry
per-resource rules with the same scaling shape.

All four now share a 1MB bound, which still caps memory per request
but no longer rejects legitimate fleet-sized payloads. Endpoint tests
pin a >64KB config save and a >32KB bulk ack at 200.

Refs #1601

Contract-Neutral: raise alert config and bulk ack request body caps; no payload shape or field changes
This commit is contained in:
rcourtman
2026-08-06 17:04:46 +01:00
parent d1e44101b6
commit 38434a513a
2 changed files with 73 additions and 7 deletions
+13 -7
View File
@@ -195,8 +195,10 @@ func (h *AlertHandlers) GetAlertConfig(w http.ResponseWriter, r *http.Request) {
// UpdateAlertConfig updates the alert configuration
func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request) {
// Limit request body to 64KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
// Config size scales with the fleet: every per-resource toggle adds an
// Overrides entry keyed by a long resource ID, so 64KB rejected saves
// from instances with a few hundred disabled containers (#1601).
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
var config alerts.AlertConfig
if err := json.NewDecoder(r.Body).Decode(&config); err != nil {
@@ -327,7 +329,9 @@ func (h *AlertHandlers) GetAlertIntentPolicies(w http.ResponseWriter, r *http.Re
}
func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, 256*1024)
// Per-resource policy rules scale with the fleet the same way alert
// config overrides do, so this shares their 1MB bound.
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
var document alerts.AlertIntentPolicyDocument
decoder := json.NewDecoder(r.Body)
decoder.DisallowUnknownFields()
@@ -1018,8 +1022,10 @@ func (h *AlertHandlers) ClearAlertByBody(w http.ResponseWriter, r *http.Request)
// BulkAcknowledgeAlerts acknowledges multiple alerts at once
func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Request) {
// Limit request body to 32KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 32*1024)
// Identifier lists scale with active alert count, and an alert flood is
// exactly when ack-all gets used, so 32KB starved the one path meant to
// recover from it (#1601).
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
var request alertIdentifiersRequest
@@ -1082,8 +1088,8 @@ func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Req
// BulkClearAlerts clears multiple alerts at once
func (h *AlertHandlers) BulkClearAlerts(w http.ResponseWriter, r *http.Request) {
// Limit request body to 32KB to prevent memory exhaustion
r.Body = http.MaxBytesReader(w, r.Body, 32*1024)
// Same fleet-scaling bound as BulkAcknowledgeAlerts.
r.Body = http.MaxBytesReader(w, r.Body, 1<<20)
var request alertIdentifiersRequest
+60
View File
@@ -3,6 +3,7 @@ package api_test
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"testing"
@@ -79,6 +80,36 @@ func TestAlertsEndpoints(t *testing.T) {
}
})
// A fleet with a few hundred per-resource toggles produces a config well
// past the old 64KB body cap, which rejected legitimate saves (#1601).
t.Run("UpdateAlertConfigLargeFleet", func(t *testing.T) {
overrides := make(map[string]alerts.ThresholdConfig, 5000)
for i := 0; i < 5000; i++ {
key := fmt.Sprintf("docker:integration-host-%04d/container:web-frontend-replica-%04d", i, i)
overrides[key] = alerts.ThresholdConfig{}
}
body, _ := json.Marshal(alerts.AlertConfig{Overrides: overrides})
if len(body) <= 64*1024 {
t.Fatalf("fixture too small to exercise the old 64KB cap: %d bytes", len(body))
}
req, err := http.NewRequest(http.MethodPut, srv.server.URL+"/api/alerts/config", bytes.NewBuffer(body))
if err != nil {
t.Fatalf("create request failed: %v", err)
}
req.Header.Set("Content-Type", "application/json")
res, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
t.Errorf("status code = %d, want %d", res.StatusCode, http.StatusOK)
}
})
t.Run("AlertIntentPolicies", func(t *testing.T) {
res, err := http.Get(srv.server.URL + "/api/alerts/intent-policies")
if err != nil {
@@ -321,6 +352,35 @@ func TestAlertsEndpoints(t *testing.T) {
}
})
// Identifier lists scale with active alert count. The old 32KB cap made
// ack-all fail during exactly the floods it exists to recover from (#1601).
t.Run("BulkAcknowledgeLargeList", func(t *testing.T) {
identifiers := make([]string, 2000)
for i := range identifiers {
identifiers[i] = fmt.Sprintf("docker:host-%04d/container:app-%04d-cpu", i, i)
}
jsonBody, _ := json.Marshal(map[string]interface{}{"alertIdentifiers": identifiers})
if len(jsonBody) <= 32*1024 {
t.Fatalf("fixture too small to exercise the old 32KB cap: %d bytes", len(jsonBody))
}
req, err := http.NewRequest(http.MethodPost, srv.server.URL+"/api/alerts/bulk/acknowledge", bytes.NewBuffer(jsonBody))
if err != nil {
t.Fatalf("create request failed: %v", err)
}
req.Header.Set("Content-Type", "application/json")
res, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
t.Errorf("status code = %d, want %d", res.StatusCode, http.StatusOK)
}
})
// 9. Bulk Clear
t.Run("BulkClear", func(t *testing.T) {
body := map[string]interface{}{