mirror of
https://github.com/PerpetualSoftware/pad.git
synced 2026-09-20 17:43:26 +00:00
715ec70e94
NewRateLimiters spawned 9 ipRateLimiter cleanup goroutines per Server,
each in an unbounded `for { time.Sleep(5*time.Minute); ... }` loop with
no exit signal (middleware_ratelimit.go:78-89). Every testServer(t)
call leaked all 9, accumulating across the 210-test internal/server
suite. Under -race the goroutine count + sync overhead pushed the run
past the default 10m timeout, which is why the `Run tests with race
detector` step (gated to main pushes) has been failing on every main
run since the step was added on 2026-04-13.
This is the same flavor as BUG-842 part 2 (request-handler
fire-and-forget goroutines drained via Server.bg WaitGroup). The
rate-limiter case wasn't in BUG-842's scope: those goroutines are
spawned at construction time, not at request time, so they need a
different drain primitive.
Changes:
- ipRateLimiter gains stopCh + stopOnce + stopWg. cleanup() rewrites
its loop as a select over stopCh and a 5-minute ticker, deferring
stopWg.Done(). New Stop() closes stopCh once and waits for the
cleanup goroutine to return.
- RateLimiters gains a Stop() that walks all 9 limiters (nil-safe
via the (*ipRateLimiter).Stop receiver guard).
- Server.Stop() now also calls s.rateLimiters.Stop() after
s.bg.Wait(). Test cleanups already call Server.Stop() (added in
BUG-842), so no test-helper changes needed.
- New TestServer_Stop_DrainsRateLimiterCleanup pins the contract:
construct + Stop N servers, assert runtime.NumGoroutine() returns
to baseline ±3.
- .github/workflows/ci.yml: bump the -race timeout from the default
10m to 20m. The full server suite under -race takes ~13m on a dev
laptop after the leak fix; 20m gives margin without papering over
an actual hang. Both `Run tests with race detector` (SQLite) and
`Run tests with race detector against PostgreSQL` are bumped.
Verified locally: go test -race -timeout=1500s ./internal/server/
finishes ok in 776s (12m57s). Without the leak fix, the same command
times out at 600s (10m) with a goroutine dump showing hundreds of
ipRateLimiter.cleanup frames.
374 lines
12 KiB
Go
374 lines
12 KiB
Go
package server
|
|
|
|
import (
|
|
"log/slog"
|
|
"math"
|
|
"net"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/time/rate"
|
|
)
|
|
|
|
// rateLimitConfig holds the rate and burst for a limiter.
|
|
type rateLimitConfig struct {
|
|
Rate rate.Limit // events per second
|
|
Burst int // max burst
|
|
// Retention is how long an inactive key stays in memory before the
|
|
// background cleanup evicts it. Must be at least as long as the rate
|
|
// window (≈ burst / rate) or premature eviction lets an attacker reset
|
|
// their bucket by waiting — defeating "N per hour" limits that pause
|
|
// naturally between bursts. Zero means "use the default".
|
|
Retention time.Duration
|
|
}
|
|
|
|
// defaultRetention is the minimum retention for a limiter whose config
|
|
// doesn't specify one. Suitable for sub-minute windows like per-IP login
|
|
// limiting; longer windows must set Retention explicitly.
|
|
const defaultRetention = 30 * time.Minute
|
|
|
|
// ipRateLimiter tracks per-key rate limiters with automatic cleanup.
|
|
type ipRateLimiter struct {
|
|
mu sync.Mutex
|
|
limiters map[string]*rateLimiterEntry
|
|
config rateLimitConfig
|
|
retention time.Duration
|
|
|
|
// stopCh / stopOnce / stopWg let Server.Stop() shut the cleanup
|
|
// goroutine down. Without this, every call to NewRateLimiters spawned
|
|
// 9 forever-sleeping goroutines that never exited — under -race the
|
|
// accumulation pushed the test runtime past the 10-minute timeout.
|
|
// See BUG-851.
|
|
stopCh chan struct{}
|
|
stopOnce sync.Once
|
|
stopWg sync.WaitGroup
|
|
}
|
|
|
|
type rateLimiterEntry struct {
|
|
limiter *rate.Limiter
|
|
lastSeen time.Time
|
|
}
|
|
|
|
func newIPRateLimiter(cfg rateLimitConfig) *ipRateLimiter {
|
|
retention := cfg.Retention
|
|
if retention <= 0 {
|
|
retention = defaultRetention
|
|
}
|
|
rl := &ipRateLimiter{
|
|
limiters: make(map[string]*rateLimiterEntry),
|
|
config: cfg,
|
|
retention: retention,
|
|
stopCh: make(chan struct{}),
|
|
}
|
|
// Background cleanup of stale entries every 5 minutes. Tracked via
|
|
// stopWg so Stop() can drain it before the surrounding Server is torn
|
|
// down (BUG-851).
|
|
rl.stopWg.Add(1)
|
|
go rl.cleanup()
|
|
return rl
|
|
}
|
|
|
|
// Stop signals the cleanup goroutine to exit and blocks until it does.
|
|
// Safe to call multiple times — stopOnce guards the channel close.
|
|
func (rl *ipRateLimiter) Stop() {
|
|
if rl == nil {
|
|
return
|
|
}
|
|
rl.stopOnce.Do(func() { close(rl.stopCh) })
|
|
rl.stopWg.Wait()
|
|
}
|
|
|
|
func (rl *ipRateLimiter) getLimiter(key string) *rate.Limiter {
|
|
rl.mu.Lock()
|
|
defer rl.mu.Unlock()
|
|
|
|
entry, exists := rl.limiters[key]
|
|
if !exists {
|
|
limiter := rate.NewLimiter(rl.config.Rate, rl.config.Burst)
|
|
rl.limiters[key] = &rateLimiterEntry{
|
|
limiter: limiter,
|
|
lastSeen: time.Now(),
|
|
}
|
|
return limiter
|
|
}
|
|
entry.lastSeen = time.Now()
|
|
return entry.limiter
|
|
}
|
|
|
|
func (rl *ipRateLimiter) cleanup() {
|
|
defer rl.stopWg.Done()
|
|
ticker := time.NewTicker(5 * time.Minute)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-rl.stopCh:
|
|
return
|
|
case <-ticker.C:
|
|
rl.mu.Lock()
|
|
for key, entry := range rl.limiters {
|
|
if time.Since(entry.lastSeen) > rl.retention {
|
|
delete(rl.limiters, key)
|
|
}
|
|
}
|
|
rl.mu.Unlock()
|
|
}
|
|
}
|
|
}
|
|
|
|
// RateLimiters holds all the rate limiters used by the server.
|
|
type RateLimiters struct {
|
|
// Auth endpoints: strict limits per IP
|
|
Auth *ipRateLimiter
|
|
// Login attempts per email: catches credential-spraying that bypasses
|
|
// the per-IP limit by rotating through a botnet. Consumed inside
|
|
// handleLogin on every login attempt (success or failure) — a
|
|
// legitimate user who only mistypes a couple of times never notices.
|
|
AuthEmail *ipRateLimiter
|
|
// Password reset: per-IP
|
|
PasswordReset *ipRateLimiter
|
|
// Registration: per-IP
|
|
Register *ipRateLimiter
|
|
// OAuth login: per-IP (higher limit since pad-cloud sidecar calls this)
|
|
OAuthLogin *ipRateLimiter
|
|
// Cloud admin: per-IP for sidecar-to-pad admin endpoints (plan, stripe, user lookup)
|
|
CloudAdmin *ipRateLimiter
|
|
// API: per-user (authenticated)
|
|
API *ipRateLimiter
|
|
// Search: per-user or per-IP
|
|
Search *ipRateLimiter
|
|
// RecoveryCode caps how many recovery codes can be tried against a
|
|
// single 2FA challenge token. Without it an attacker who captures a
|
|
// valid challenge_token can grind through the small recovery-code
|
|
// space before the 5-minute challenge expires.
|
|
RecoveryCode *ipRateLimiter
|
|
}
|
|
|
|
// NewRateLimiters creates rate limiters with sensible defaults.
|
|
func NewRateLimiters() *RateLimiters {
|
|
return &RateLimiters{
|
|
// Login: 5 attempts per minute per IP (= 5/60 per second, burst 5)
|
|
Auth: newIPRateLimiter(rateLimitConfig{
|
|
Rate: rate.Limit(5.0 / 60.0),
|
|
Burst: 5,
|
|
}),
|
|
// Per-email: 10 attempts per hour. Low enough to defeat credential
|
|
// spraying from a botnet (which evades the per-IP limit by rotating
|
|
// source addresses), high enough that a forgetful user mistyping
|
|
// their own password never hits it under normal use.
|
|
//
|
|
// Retention must be ≥ the refill window (10 attempts / (10/hour) =
|
|
// 60 min); otherwise the cleanup could evict the bucket between
|
|
// bursts, letting an attacker pace their guesses to avoid the cap.
|
|
// 2 hours gives plenty of margin.
|
|
AuthEmail: newIPRateLimiter(rateLimitConfig{
|
|
Rate: rate.Limit(10.0 / 3600.0),
|
|
Burst: 10,
|
|
Retention: 2 * time.Hour,
|
|
}),
|
|
// Password reset: 3 per hour per IP (= 3/3600 per second, burst 3)
|
|
PasswordReset: newIPRateLimiter(rateLimitConfig{
|
|
Rate: rate.Limit(3.0 / 3600.0),
|
|
Burst: 3,
|
|
}),
|
|
// Registration: 5 per hour per IP (= 5/3600 per second, burst 5)
|
|
Register: newIPRateLimiter(rateLimitConfig{
|
|
Rate: rate.Limit(5.0 / 3600.0),
|
|
Burst: 5,
|
|
}),
|
|
// OAuth login/link: 20 per minute per IP (sidecar calls this — higher than regular auth)
|
|
OAuthLogin: newIPRateLimiter(rateLimitConfig{
|
|
Rate: rate.Limit(20.0 / 60.0),
|
|
Burst: 20,
|
|
}),
|
|
// Cloud admin: 30 per minute per IP for sidecar admin calls (plan changes, Stripe mapping)
|
|
// These are cloud-secret gated but rate-limited for defense in depth.
|
|
CloudAdmin: newIPRateLimiter(rateLimitConfig{
|
|
Rate: rate.Limit(30.0 / 60.0),
|
|
Burst: 10,
|
|
}),
|
|
// API: 600 requests per minute per user/IP (= 10 per second, burst 60)
|
|
// Local-first tool with SSE-driven UI needs headroom for cascading refreshes.
|
|
API: newIPRateLimiter(rateLimitConfig{
|
|
Rate: rate.Limit(600.0 / 60.0),
|
|
Burst: 60,
|
|
}),
|
|
// Search: 30 requests per minute per user/IP (= 30/60 per second, burst 10)
|
|
Search: newIPRateLimiter(rateLimitConfig{
|
|
Rate: rate.Limit(30.0 / 60.0),
|
|
Burst: 10,
|
|
}),
|
|
// RecoveryCode: up to 6 attempts per challenge token before lockout.
|
|
// Challenge tokens live for 5 minutes, so we only need the limiter to
|
|
// remember that long — but retention defaults to 30 minutes so we
|
|
// pick up a couple of wall-clock minutes of slop. Rate is effectively
|
|
// "no refill over the window" since burst = 6 and the limiter won't
|
|
// meaningfully refill in 5 min at 6/hour.
|
|
RecoveryCode: newIPRateLimiter(rateLimitConfig{
|
|
Rate: rate.Limit(6.0 / 3600.0),
|
|
Burst: 6,
|
|
}),
|
|
}
|
|
}
|
|
|
|
// Stop drains the cleanup goroutine of every limiter in the bundle. Called
|
|
// from Server.Stop() so test cleanup (and graceful shutdown) doesn't leak
|
|
// the 9 forever-sleeping goroutines NewRateLimiters spawns. Safe to call
|
|
// on a nil receiver and idempotent per-limiter via stopOnce. See BUG-851.
|
|
func (rls *RateLimiters) Stop() {
|
|
if rls == nil {
|
|
return
|
|
}
|
|
for _, rl := range []*ipRateLimiter{
|
|
rls.Auth,
|
|
rls.AuthEmail,
|
|
rls.PasswordReset,
|
|
rls.Register,
|
|
rls.OAuthLogin,
|
|
rls.CloudAdmin,
|
|
rls.API,
|
|
rls.Search,
|
|
rls.RecoveryCode,
|
|
} {
|
|
rl.Stop() // nil-safe via the receiver guard in (*ipRateLimiter).Stop
|
|
}
|
|
}
|
|
|
|
// RateLimit is the general-purpose rate limiting middleware.
|
|
// It applies different limits based on the endpoint being hit.
|
|
func (s *Server) RateLimit(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if s.rateLimiters == nil {
|
|
next.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
|
|
path := r.URL.Path
|
|
|
|
// Only rate-limit API endpoints
|
|
if !strings.HasPrefix(path, "/api/") {
|
|
next.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
|
|
ip := clientIP(r)
|
|
|
|
// Auth-specific rate limits
|
|
if strings.HasPrefix(path, "/api/v1/auth/") {
|
|
var limiter *ipRateLimiter
|
|
switch {
|
|
case path == "/api/v1/auth/login" || path == "/api/v1/auth/bootstrap" || path == "/api/v1/auth/2fa/login-verify":
|
|
limiter = s.rateLimiters.Auth
|
|
case path == "/api/v1/auth/forgot-password" || path == "/api/v1/auth/reset-password":
|
|
limiter = s.rateLimiters.PasswordReset
|
|
case path == "/api/v1/auth/register":
|
|
limiter = s.rateLimiters.Register
|
|
case path == "/api/v1/auth/oauth-login" || path == "/api/v1/auth/oauth-link":
|
|
limiter = s.rateLimiters.OAuthLogin
|
|
case path == "/api/v1/auth/oauth-unlink":
|
|
limiter = s.rateLimiters.Auth // Same as login — 5/min, user-initiated
|
|
default:
|
|
// Other auth endpoints (session check, logout) — use general API limit
|
|
limiter = s.rateLimiters.API
|
|
}
|
|
|
|
if limiter != nil {
|
|
l := limiter.getLimiter(ip)
|
|
if !l.Allow() {
|
|
slog.Warn("rate limited", "ip", ip, "path", path, "limiter", "auth")
|
|
writeRateLimitResponse(w, limiter.config)
|
|
return
|
|
}
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
|
|
// Cloud admin endpoints (sidecar → pad): plan changes, Stripe mapping, user lookup
|
|
if strings.HasPrefix(path, "/api/v1/admin/") {
|
|
switch path {
|
|
case "/api/v1/admin/plan", "/api/v1/admin/stripe-customer-id", "/api/v1/admin/user-by-customer", "/api/v1/admin/stripe-event-processed", "/api/v1/admin/stripe-event-unmark", "/api/v1/admin/payment-failed":
|
|
l := s.rateLimiters.CloudAdmin.getLimiter(ip)
|
|
if !l.Allow() {
|
|
slog.Warn("rate limited", "ip", ip, "path", path, "limiter", "cloud_admin")
|
|
writeRateLimitResponse(w, s.rateLimiters.CloudAdmin.config)
|
|
return
|
|
}
|
|
}
|
|
// Other admin endpoints fall through to general API limit below
|
|
}
|
|
|
|
// Search endpoint
|
|
if path == "/api/v1/search" {
|
|
key := rateLimitKey(r, ip)
|
|
if !s.rateLimiters.Search.getLimiter(key).Allow() {
|
|
slog.Warn("rate limited", "key", key, "path", path, "limiter", "search")
|
|
writeRateLimitResponse(w, s.rateLimiters.Search.config)
|
|
return
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
|
|
// General API rate limit
|
|
key := rateLimitKey(r, ip)
|
|
if !s.rateLimiters.API.getLimiter(key).Allow() {
|
|
slog.Warn("rate limited", "key", key, "path", path, "limiter", "api")
|
|
writeRateLimitResponse(w, s.rateLimiters.API.config)
|
|
return
|
|
}
|
|
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
// rateLimitKey returns a key for rate limiting: user ID if authenticated, IP otherwise.
|
|
func rateLimitKey(r *http.Request, ip string) string {
|
|
if user := currentUser(r); user != nil {
|
|
return "user:" + user.ID
|
|
}
|
|
return "ip:" + ip
|
|
}
|
|
|
|
// clientIP extracts the client IP from RemoteAddr. This is safe because
|
|
// TrustedProxyRealIP runs earlier in the chain and — when a trusted
|
|
// proxy is configured — overwrites RemoteAddr with the trusted value
|
|
// from X-Real-IP / X-Forwarded-For. We deliberately do NOT read proxy
|
|
// headers here to prevent clients from spoofing their IP to bypass
|
|
// rate limits.
|
|
//
|
|
// Uses net.SplitHostPort so IPv6 addresses are handled correctly.
|
|
// A naive LastIndex(":") strips the final hextet of a bare IPv6 address
|
|
// like "2001:db8::1" — TrustedProxyRealIP writes the X-Forwarded-For
|
|
// value verbatim (no port, no brackets), so a LastIndex-based parse
|
|
// would mangle it. For bare IPs without a port SplitHostPort returns
|
|
// an error and we return the address as-is.
|
|
func clientIP(r *http.Request) string {
|
|
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
|
|
return host
|
|
}
|
|
return r.RemoteAddr
|
|
}
|
|
|
|
// writeRateLimitResponse sends a 429 response with Retry-After and X-RateLimit-* headers.
|
|
func writeRateLimitResponse(w http.ResponseWriter, cfg rateLimitConfig) {
|
|
// Calculate retry-after from the rate (seconds until one token is available)
|
|
retryAfter := int(math.Ceil(1.0 / float64(cfg.Rate)))
|
|
if retryAfter < 1 {
|
|
retryAfter = 1
|
|
}
|
|
if retryAfter > 3600 {
|
|
retryAfter = 3600
|
|
}
|
|
|
|
// Calculate requests per minute for the limit header
|
|
limitPerMinute := int(math.Ceil(float64(cfg.Rate) * 60))
|
|
|
|
w.Header().Set("Retry-After", strconv.Itoa(retryAfter))
|
|
w.Header().Set("X-RateLimit-Limit", strconv.Itoa(limitPerMinute))
|
|
w.Header().Set("X-RateLimit-Remaining", "0")
|
|
writeError(w, http.StatusTooManyRequests, "rate_limited", "Too many requests. Please try again later.")
|
|
}
|