mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-19 14:56:27 +00:00
fix(api): add tiered rate limiting to prevent polling lockouts (#460)
* fix(api): add tiered rate limiting to prevent polling lockouts Replace the single global rate limiter (100 req/min/IP) with a tiered system that separates high-frequency polling endpoints from standard API traffic: - Polling tier (300/min): /stats, /system/stats, /stacks/statuses, /metrics/historical, /health, /meta, /auth/status, /auth/sso/providers, /license. Exempt from the global limiter but governed by their own safety net to prevent resource exhaustion. - Standard tier (200/min): All other endpoints, raised from 100. - Webhook tier (500/min): POST /webhooks/:id/trigger, dedicated limiter for CI/CD platforms sharing datacenter IPs. - Auth tier: Unchanged (5-10 attempts / 15 min). Enterprise adaptations: - Authenticated requests keyed by user session (JWT sub/username) instead of IP, preventing shared NAT/VPN environments from pooling budgets. - Internal node-to-node traffic (node_proxy tokens) bypasses all rate limiters entirely. Includes comprehensive stress tests (21 cases) validating tier separation, node proxy bypass, and per-user keying. * chore(deps): bump axios to 1.15.0 to fix SSRF vulnerability Addresses GHSA-3p68-rc4w-qgx5 (NO_PROXY hostname normalization bypass).
This commit is contained in:
Anso
@@ -0,0 +1,282 @@
|
||||
/**
|
||||
* Stress tests for the tiered rate limiting system.
|
||||
*
|
||||
* Validates that:
|
||||
* - Polling endpoints are exempt from the global limiter but have their own ceiling
|
||||
* - Standard endpoints are governed by the global limiter
|
||||
* - Webhook triggers have a dedicated limiter
|
||||
* - Node proxy tokens bypass all rate limiters
|
||||
* - Authenticated requests are keyed by user session, not IP
|
||||
*
|
||||
* These tests run against the real Express app with in-memory SQLite.
|
||||
* Rate limits in development mode are 10x production values (1000/3000/5000),
|
||||
* so we test header presence and tier separation rather than hitting ceilings.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { setupTestDb, cleanupTestDb, TEST_JWT_SECRET, TEST_USERNAME, TEST_PASSWORD } from './helpers/setupTestDb';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let authCookie: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
|
||||
// Log in to get an auth cookie for authenticated tests
|
||||
const loginRes = await request(app)
|
||||
.post('/api/auth/login')
|
||||
.send({ username: TEST_USERNAME, password: TEST_PASSWORD });
|
||||
const setCookie = loginRes.headers['set-cookie'];
|
||||
authCookie = Array.isArray(setCookie) ? setCookie[0] : setCookie;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
// ── Tier 0/1: Polling endpoints ──────────────────────────────────────────────
|
||||
|
||||
describe('Tier 0/1: Polling endpoints', () => {
|
||||
const pollingPaths = [
|
||||
'/api/health',
|
||||
'/api/meta',
|
||||
'/api/stats',
|
||||
'/api/system/stats',
|
||||
'/api/stacks/statuses',
|
||||
'/api/metrics/historical',
|
||||
'/api/auth/status',
|
||||
'/api/auth/sso/providers',
|
||||
'/api/license',
|
||||
];
|
||||
|
||||
it.each(pollingPaths)('%s returns polling-tier rate limit headers', async (path) => {
|
||||
const res = await request(app).get(path);
|
||||
const limit = parseInt(res.headers['ratelimit-limit'], 10);
|
||||
// Dev mode polling limit is 3000 (production would be 300)
|
||||
expect(limit).toBe(3000);
|
||||
});
|
||||
|
||||
it('polling endpoints do NOT count against the global limiter', async () => {
|
||||
// Snapshot the global limiter's remaining count before polling
|
||||
const before = await request(app).get('/api/stacks');
|
||||
const remainBefore = parseInt(before.headers['ratelimit-remaining'], 10);
|
||||
|
||||
// Hit 10 polling endpoints (none should decrement the global counter)
|
||||
await Promise.all(Array.from({ length: 10 }, () =>
|
||||
request(app).get('/api/health')
|
||||
));
|
||||
|
||||
// Check that the global limiter's remaining decreased by exactly 1
|
||||
// (the GET /api/stacks below, not the 10 polling requests)
|
||||
const after = await request(app).get('/api/stacks');
|
||||
const remainAfter = parseInt(after.headers['ratelimit-remaining'], 10);
|
||||
expect(remainAfter).toBe(remainBefore - 1);
|
||||
});
|
||||
|
||||
it('rapid polling does not trigger global rate limit', async () => {
|
||||
// Snapshot the global limiter budget before polling
|
||||
const before = await request(app).get('/api/stacks');
|
||||
const remainBefore = parseInt(before.headers['ratelimit-remaining'], 10);
|
||||
|
||||
// Fire 20 rapid polling requests
|
||||
const requests = Array.from({ length: 20 }, () =>
|
||||
request(app).get('/api/stats')
|
||||
);
|
||||
const results = await Promise.all(requests);
|
||||
|
||||
// All should succeed (no 429)
|
||||
for (const res of results) {
|
||||
expect(res.status).not.toBe(429);
|
||||
}
|
||||
|
||||
// Standard endpoint budget should only decrease by 1 (this request),
|
||||
// not by 21 (the 20 polling requests should not count)
|
||||
const after = await request(app).get('/api/stacks');
|
||||
const remainAfter = parseInt(after.headers['ratelimit-remaining'], 10);
|
||||
expect(remainAfter).toBe(remainBefore - 1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Tier 2: Standard endpoints ───────────────────────────────────────────────
|
||||
|
||||
describe('Tier 2: Standard endpoints', () => {
|
||||
it('standard endpoints return global-tier rate limit headers', async () => {
|
||||
const res = await request(app).get('/api/stacks');
|
||||
const limit = parseInt(res.headers['ratelimit-limit'], 10);
|
||||
// Dev mode global limit is 1000 (production would be 200)
|
||||
expect(limit).toBe(1000);
|
||||
});
|
||||
|
||||
it('standard endpoints do NOT get polling-tier headers', async () => {
|
||||
const res = await request(app).get('/api/containers');
|
||||
const limit = parseInt(res.headers['ratelimit-limit'], 10);
|
||||
// Should be 1000 (global), not 3000 (polling)
|
||||
expect(limit).not.toBe(3000);
|
||||
expect(limit).toBe(1000);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Node proxy bypass ────────────────────────────────────────────────────────
|
||||
|
||||
describe('Node proxy bypass', () => {
|
||||
it('requests with valid node_proxy token bypass all rate limiters', async () => {
|
||||
const nodeToken = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1h' });
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/stats')
|
||||
.set('Authorization', `Bearer ${nodeToken}`);
|
||||
|
||||
// Node proxy requests should NOT receive rate limit headers from either limiter
|
||||
// (both skip functions return true for node_proxy tokens)
|
||||
// When skipped, express-rate-limit does not set headers
|
||||
expect(res.headers['ratelimit-limit']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('requests with invalid Bearer token still get rate limited', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/stats')
|
||||
.set('Authorization', 'Bearer invalid-token-here');
|
||||
|
||||
// Should still get rate limit headers (polling tier for /stats)
|
||||
const limit = parseInt(res.headers['ratelimit-limit'], 10);
|
||||
expect(limit).toBe(3000);
|
||||
});
|
||||
|
||||
it('requests with non-node_proxy Bearer token still get rate limited', async () => {
|
||||
// A regular user JWT (not node_proxy scope) should still be rate limited
|
||||
const userToken = jwt.sign(
|
||||
{ username: 'testuser', role: 'admin' },
|
||||
TEST_JWT_SECRET,
|
||||
{ expiresIn: '1h' }
|
||||
);
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/stacks')
|
||||
.set('Authorization', `Bearer ${userToken}`);
|
||||
|
||||
const limit = parseInt(res.headers['ratelimit-limit'], 10);
|
||||
expect(limit).toBe(1000);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Hybrid key generator (per-user keying) ───────────────────────────────────
|
||||
|
||||
describe('Hybrid key generator', () => {
|
||||
it('authenticated requests are keyed by user, not IP', async () => {
|
||||
// Make requests with auth cookie
|
||||
const res1 = await request(app)
|
||||
.get('/api/stacks')
|
||||
.set('Cookie', authCookie);
|
||||
|
||||
// Make a request without auth (keyed by IP)
|
||||
const res2 = await request(app).get('/api/stacks');
|
||||
|
||||
// Both should succeed, and their remaining counts should be independent
|
||||
// (different rate limit buckets: user:testadmin vs IP)
|
||||
const remaining1 = parseInt(res1.headers['ratelimit-remaining'], 10);
|
||||
const remaining2 = parseInt(res2.headers['ratelimit-remaining'], 10);
|
||||
|
||||
// Each should have its own budget (999 for this test since each made 1 request)
|
||||
expect(remaining1).toBeGreaterThanOrEqual(990);
|
||||
expect(remaining2).toBeGreaterThanOrEqual(990);
|
||||
});
|
||||
|
||||
it('two different users get independent rate limit budgets', async () => {
|
||||
// User A (via cookie)
|
||||
const resA = await request(app)
|
||||
.get('/api/stacks')
|
||||
.set('Cookie', authCookie);
|
||||
|
||||
// User B (via Bearer with different username)
|
||||
const userBToken = jwt.sign(
|
||||
{ username: 'otheruser', role: 'viewer' },
|
||||
TEST_JWT_SECRET,
|
||||
{ expiresIn: '1h' }
|
||||
);
|
||||
const resB = await request(app)
|
||||
.get('/api/stacks')
|
||||
.set('Authorization', `Bearer ${userBToken}`);
|
||||
|
||||
// Both should have independent remaining counts
|
||||
const remainA = parseInt(resA.headers['ratelimit-remaining'], 10);
|
||||
const remainB = parseInt(resB.headers['ratelimit-remaining'], 10);
|
||||
|
||||
// Each user's budget should be near-full
|
||||
expect(remainA).toBeGreaterThanOrEqual(990);
|
||||
expect(remainB).toBeGreaterThanOrEqual(990);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Webhook trigger tier ─────────────────────────────────────────────────────
|
||||
|
||||
describe('Tier W: Webhook trigger', () => {
|
||||
it('webhook trigger route is exempt from the global limiter', async () => {
|
||||
// We can't easily test the webhook-specific limiter without creating a webhook,
|
||||
// but we can verify the global limiter skip logic by checking that rapid
|
||||
// POST requests to a webhook-trigger-shaped path don't decrement the global counter.
|
||||
|
||||
// Hit a standard endpoint first to establish a baseline
|
||||
const baseline = await request(app).get('/api/stacks');
|
||||
const baselineRemaining = parseInt(baseline.headers['ratelimit-remaining'], 10);
|
||||
|
||||
// POST to webhook trigger (will 404 since no webhook exists, but the rate
|
||||
// limiter skip runs before the route handler)
|
||||
await request(app).post('/api/webhooks/999/trigger').send({});
|
||||
|
||||
// Check that the global limiter's remaining hasn't decreased
|
||||
const after = await request(app).get('/api/stacks');
|
||||
const afterRemaining = parseInt(after.headers['ratelimit-remaining'], 10);
|
||||
|
||||
// Should have decreased by exactly 1 (the GET /api/stacks request itself)
|
||||
expect(afterRemaining).toBe(baselineRemaining - 1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Tier separation stress test ──────────────────────────────────────────────
|
||||
|
||||
describe('Tier separation under load', () => {
|
||||
it('50 polling requests do not affect standard endpoint budget', async () => {
|
||||
// Blast 50 polling requests in parallel
|
||||
const pollingRequests = Array.from({ length: 50 }, () =>
|
||||
request(app).get('/api/health')
|
||||
);
|
||||
await Promise.all(pollingRequests);
|
||||
|
||||
// Standard endpoint should still have a nearly full budget
|
||||
const stdRes = await request(app).get('/api/stacks');
|
||||
const remaining = parseInt(stdRes.headers['ratelimit-remaining'], 10);
|
||||
|
||||
// Should have lost very few from the standard budget (only from other tests)
|
||||
expect(remaining).toBeGreaterThanOrEqual(980);
|
||||
});
|
||||
|
||||
it('mixed polling and standard requests maintain independent counters', async () => {
|
||||
// Interleave 10 polling and 10 standard requests
|
||||
const mixed = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
mixed.push(request(app).get('/api/stats')); // polling
|
||||
mixed.push(request(app).get('/api/containers')); // standard
|
||||
}
|
||||
const results = await Promise.all(mixed);
|
||||
|
||||
// No 429s should occur (we're well within dev limits)
|
||||
for (const res of results) {
|
||||
expect(res.status).not.toBe(429);
|
||||
}
|
||||
|
||||
// Check that polling results have polling-tier headers
|
||||
for (let i = 0; i < results.length; i += 2) {
|
||||
const limit = parseInt(results[i].headers['ratelimit-limit'], 10);
|
||||
expect(limit).toBe(3000);
|
||||
}
|
||||
|
||||
// Check that standard results have global-tier headers
|
||||
for (let i = 1; i < results.length; i += 2) {
|
||||
const limit = parseInt(results[i].headers['ratelimit-limit'], 10);
|
||||
expect(limit).toBe(1000);
|
||||
}
|
||||
});
|
||||
});
|
||||
+125
-11
@@ -1,7 +1,7 @@
|
||||
import express, { Request, Response, NextFunction } from 'express';
|
||||
import cors from 'cors';
|
||||
import cookieParser from 'cookie-parser';
|
||||
import rateLimit from 'express-rate-limit';
|
||||
import rateLimit, { ipKeyGenerator } from 'express-rate-limit';
|
||||
import helmet from 'helmet';
|
||||
import WebSocket, { WebSocketServer } from 'ws';
|
||||
import jwt from 'jsonwebtoken';
|
||||
@@ -138,22 +138,137 @@ app.use(cors({
|
||||
credentials: true,
|
||||
}));
|
||||
|
||||
// Global API rate limiter — caps total requests per IP across all /api/ routes.
|
||||
// Auth-specific limiters (authRateLimiter, ssoRateLimiter) apply additional,
|
||||
// stricter limits on their respective routes and stack independently.
|
||||
const globalApiLimiter = rateLimit({
|
||||
// Cookie parser must run before rate limiters so the hybrid key generator
|
||||
// can read req.cookies for per-user rate limit bucketing.
|
||||
app.use(cookieParser());
|
||||
|
||||
// ── Rate Limiting ─────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Tiered rate limiting to prevent UX lockouts while maintaining security:
|
||||
// Tier 0/1 (Polling): High-frequency GET endpoints exempt from global limit,
|
||||
// with a 300/min safety net to prevent resource exhaustion.
|
||||
// Tier W (Webhooks): CI/CD webhook triggers at 500/min (shared datacenter IPs).
|
||||
// Tier 2 (Standard): All other endpoints at 200/min (raised from 100).
|
||||
// Tier 3 (Auth): Strict brute-force protection (5-10 attempts / 15min).
|
||||
//
|
||||
// Enterprise adaptations:
|
||||
// - Internal node-to-node traffic (node_proxy JWTs) bypasses all rate limiters.
|
||||
// - Authenticated requests are keyed by user ID (not IP) to prevent shared
|
||||
// NAT/VPN environments from pooling rate limit budgets.
|
||||
|
||||
/** Read-only GET endpoints polled at high frequency by the dashboard/fleet UI. */
|
||||
const POLLING_EXEMPT_PATHS = new Set([
|
||||
'/meta', '/health', '/stats', '/system/stats',
|
||||
'/stacks/statuses', '/metrics/historical',
|
||||
'/auth/status', '/auth/sso/providers', '/license',
|
||||
]);
|
||||
|
||||
const WEBHOOK_TRIGGER_RE = /^\/webhooks\/\d+\/trigger$/;
|
||||
|
||||
/**
|
||||
* Returns true if the request bears a node_proxy Bearer token.
|
||||
* Uses jwt.decode() (no signature verification) to avoid crypto overhead on the
|
||||
* hot path; authMiddleware performs full verification downstream. Worst case for
|
||||
* a forged token: it skips the rate limiter but is still rejected by auth.
|
||||
* Result is memoized on the request object so the two sequential limiters
|
||||
* don't repeat the work.
|
||||
*/
|
||||
function isNodeProxyRequest(req: Request): boolean {
|
||||
const cached = (req as any)._isNodeProxy;
|
||||
if (cached !== undefined) return cached;
|
||||
const auth = req.headers.authorization;
|
||||
if (!auth?.startsWith('Bearer ')) {
|
||||
(req as any)._isNodeProxy = false;
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const decoded = jwt.decode(auth.slice(7)) as { scope?: string } | null;
|
||||
const result = decoded?.scope === 'node_proxy';
|
||||
(req as any)._isNodeProxy = result;
|
||||
return result;
|
||||
} catch {
|
||||
(req as any)._isNodeProxy = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hybrid rate limit key: uses the JWT username/sub claim for authenticated
|
||||
* requests (per-user budgets) and falls back to IP for unauthenticated ones.
|
||||
* Uses jwt.decode() (no verification) to avoid double-verification cost;
|
||||
* authMiddleware handles signature checks downstream.
|
||||
*/
|
||||
function rateLimitKeyGenerator(req: Request): string {
|
||||
const cookie = req.cookies?.[COOKIE_NAME];
|
||||
if (cookie) {
|
||||
try {
|
||||
const decoded = jwt.decode(cookie) as { username?: string } | null;
|
||||
if (decoded?.username) return `user:${decoded.username}`;
|
||||
} catch { /* fall through to IP */ }
|
||||
}
|
||||
const auth = req.headers.authorization;
|
||||
if (auth?.startsWith('Bearer ')) {
|
||||
try {
|
||||
const decoded = jwt.decode(auth.slice(7)) as { username?: string; sub?: string } | null;
|
||||
if (decoded?.username) return `user:${decoded.username}`;
|
||||
if (decoded?.sub) return `user:${decoded.sub}`;
|
||||
} catch { /* fall through to IP */ }
|
||||
}
|
||||
return ipKeyGenerator(req.ip || 'unknown');
|
||||
}
|
||||
|
||||
/** Shared config for all rate limiters (1-minute window, standard headers). */
|
||||
const rateLimitBase = {
|
||||
windowMs: 60 * 1000,
|
||||
max: process.env.NODE_ENV === 'production'
|
||||
? parseInt(process.env.API_RATE_LIMIT || '100', 10)
|
||||
: 1000,
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
} as const;
|
||||
|
||||
// Tier 2: Global API rate limiter. Skips polling endpoints (Tier 0/1), webhook
|
||||
// triggers (Tier W), and internal node-to-node traffic (node_proxy).
|
||||
const globalApiLimiter = rateLimit({
|
||||
...rateLimitBase,
|
||||
max: process.env.NODE_ENV === 'production'
|
||||
? parseInt(process.env.API_RATE_LIMIT || '200', 10)
|
||||
: 1000,
|
||||
keyGenerator: rateLimitKeyGenerator,
|
||||
message: { error: 'Too many requests. Please try again shortly.' },
|
||||
skip: (req: Request) => req.path === '/meta' || req.path === '/health',
|
||||
skip: (req: Request) => {
|
||||
if (req.method === 'GET' && POLLING_EXEMPT_PATHS.has(req.path)) return true;
|
||||
if (req.method === 'POST' && WEBHOOK_TRIGGER_RE.test(req.path)) return true;
|
||||
if (isNodeProxyRequest(req)) return true;
|
||||
return false;
|
||||
},
|
||||
});
|
||||
|
||||
app.use('/api/', globalApiLimiter);
|
||||
|
||||
// Tier 0/1: Polling safety net. Applies only to polling-exempt endpoints to
|
||||
// prevent resource exhaustion from runaway or malicious polling.
|
||||
const pollingLimiter = rateLimit({
|
||||
...rateLimitBase,
|
||||
max: process.env.NODE_ENV === 'production'
|
||||
? parseInt(process.env.API_POLLING_RATE_LIMIT || '300', 10)
|
||||
: 3000,
|
||||
keyGenerator: rateLimitKeyGenerator,
|
||||
message: { error: 'Too many polling requests. Please try again shortly.' },
|
||||
skip: (req: Request) => {
|
||||
if (isNodeProxyRequest(req)) return true;
|
||||
return !(req.method === 'GET' && POLLING_EXEMPT_PATHS.has(req.path));
|
||||
},
|
||||
});
|
||||
|
||||
app.use('/api/', pollingLimiter);
|
||||
|
||||
// Tier W: Webhook trigger limiter. Applied inline on the trigger route handler.
|
||||
// CI/CD platforms (GitHub Actions, GitLab runners) often share datacenter IPs,
|
||||
// so a higher ceiling prevents dropped deployments during burst activity.
|
||||
const webhookTriggerLimiter = rateLimit({
|
||||
...rateLimitBase,
|
||||
max: process.env.NODE_ENV === 'production' ? 500 : 5000,
|
||||
message: { error: 'Too many webhook triggers. Please try again shortly.' },
|
||||
});
|
||||
|
||||
// JSON body parser that also captures the raw bytes for HMAC verification.
|
||||
const jsonParser = express.json({
|
||||
verify: (req, _res, buf) => {
|
||||
@@ -191,7 +306,6 @@ app.use((req: Request, res: Response, next: NextFunction): void => {
|
||||
}
|
||||
jsonParser(req, res, next);
|
||||
});
|
||||
app.use(cookieParser());
|
||||
|
||||
// Node Context Middleware
|
||||
const nodeContextMiddleware = (req: Request, res: Response, next: NextFunction) => {
|
||||
@@ -2190,7 +2304,7 @@ app.get('/api/webhooks/:id/history', authMiddleware, async (req: Request, res: R
|
||||
});
|
||||
|
||||
// Webhook trigger - public endpoint, authenticated via HMAC signature
|
||||
app.post('/api/webhooks/:id/trigger', async (req: Request, res: Response): Promise<void> => {
|
||||
app.post('/api/webhooks/:id/trigger', webhookTriggerLimiter, async (req: Request, res: Response): Promise<void> => {
|
||||
try {
|
||||
const id = parseInt(req.params.id as string, 10);
|
||||
const db = DatabaseService.getInstance();
|
||||
|
||||
Reference in New Issue
Block a user