mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-27 00:38:16 +00:00
feat(api): add opt-in per-bucket dimension to the S3 API rate limiter (#4949)
Follow-up to #4895 (backlog#1191 deferred sub-item). The client-IP dimension gives per-client fairness; this adds a collective per-bucket budget so one hot bucket cannot monopolize the server regardless of how many client IPs the traffic is spread across. - Generalize RateLimiter over its key type with a Borrow-based check() so &str lookups against String bucket keys stay allocation-free on the hit path; client-IP call sites are unchanged in behavior. - RateLimitLayer now carries optional client and bucket limiters; a request must pass every configured dimension, and rejections report which one tripped via a new 'dimension' metric label. - Bucket extraction mirrors s3s host routing: virtual-hosted-style resolves the Host/authority prefix against the same expanded domain set (with port variants) the s3s router uses; otherwise the first path segment. Admin and table-catalog namespaces are never buckets. - New env vars RUSTFS_API_RATE_LIMIT_BUCKET_RPM/_BURST (default 0 = dimension off) under the existing enable switch; bucket-only configurations (client RPM 0) are supported. - Bucket names are attacker-chosen, so the bounded-shards design (100k keys, lossless idle sweeps, most-idle eviction) is the memory defense; a test floods 10k random names and asserts the cap holds. - Unit tests for extraction, shared bucket budgets across IPs, both- dimensions interaction, and the extended env matrix; e2e test proves bucket-only throttling on the real server with an unrelated bucket unaffected.
This commit is contained in:
@@ -50,3 +50,26 @@ pub const ENV_API_RATE_LIMIT_BURST: &str = "RUSTFS_API_RATE_LIMIT_BURST";
|
||||
|
||||
/// Default for `RUSTFS_API_RATE_LIMIT_BURST` (`0` = same as RPM).
|
||||
pub const DEFAULT_API_RATE_LIMIT_BURST: u32 = 0;
|
||||
|
||||
/// Sustained S3 API request budget per addressed bucket, in requests per
|
||||
/// minute — a collective ceiling shared by all clients of that bucket.
|
||||
///
|
||||
/// Complements the per-client-IP dimension: it protects the server from one
|
||||
/// hot bucket regardless of how many client IPs the traffic comes from. `0`
|
||||
/// disables the bucket dimension. Requires `RUSTFS_API_RATE_LIMIT_ENABLE`.
|
||||
/// Environment variable: RUSTFS_API_RATE_LIMIT_BUCKET_RPM
|
||||
/// Example: RUSTFS_API_RATE_LIMIT_BUCKET_RPM=60000
|
||||
pub const ENV_API_RATE_LIMIT_BUCKET_RPM: &str = "RUSTFS_API_RATE_LIMIT_BUCKET_RPM";
|
||||
|
||||
/// Default for `RUSTFS_API_RATE_LIMIT_BUCKET_RPM` (`0` = dimension disabled).
|
||||
pub const DEFAULT_API_RATE_LIMIT_BUCKET_RPM: u32 = 0;
|
||||
|
||||
/// Burst capacity per bucket (maximum tokens in the bucket-dimension bucket).
|
||||
///
|
||||
/// `0` means "same as `RUSTFS_API_RATE_LIMIT_BUCKET_RPM`".
|
||||
/// Environment variable: RUSTFS_API_RATE_LIMIT_BUCKET_BURST
|
||||
/// Example: RUSTFS_API_RATE_LIMIT_BUCKET_BURST=2000
|
||||
pub const ENV_API_RATE_LIMIT_BUCKET_BURST: &str = "RUSTFS_API_RATE_LIMIT_BUCKET_BURST";
|
||||
|
||||
/// Default for `RUSTFS_API_RATE_LIMIT_BUCKET_BURST` (`0` = same as bucket RPM).
|
||||
pub const DEFAULT_API_RATE_LIMIT_BUCKET_BURST: u32 = 0;
|
||||
|
||||
@@ -86,6 +86,52 @@ async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResu
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn api_rate_limit_bucket_dimension_throttles_per_bucket() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
// Bucket dimension only: the readiness-poll ListBuckets calls hit "/"
|
||||
// (no bucket) and therefore do not consume any budget.
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_API_RATE_LIMIT_ENABLE", "true"),
|
||||
("RUSTFS_API_RATE_LIMIT_BUCKET_RPM", "60"),
|
||||
("RUSTFS_API_RATE_LIMIT_BUCKET_BURST", "5"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let client = local_http_client();
|
||||
|
||||
// Unauthenticated GETs are still counted arrivals (403, not 429, while
|
||||
// within budget); the sixth rapid hit on the same bucket must throttle.
|
||||
let mut throttled = false;
|
||||
for i in 0..6 {
|
||||
let response = client.get(format!("{}/hot-bucket/object-{i}", env.url)).send().await?;
|
||||
if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS {
|
||||
throttled = true;
|
||||
assert!(
|
||||
response.headers().contains_key(reqwest::header::RETRY_AFTER),
|
||||
"bucket-dimension 429 must carry Retry-After"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(throttled, "6 rapid requests against burst 5 must trip the bucket dimension");
|
||||
|
||||
// A different bucket has its own budget.
|
||||
let other = client.get(format!("{}/cold-bucket/object", env.url)).send().await?;
|
||||
assert_ne!(
|
||||
other.status(),
|
||||
reqwest::StatusCode::TOO_MANY_REQUESTS,
|
||||
"an unrelated bucket must not be throttled"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn api_rate_limit_stays_inert_by_default() -> TestResult {
|
||||
|
||||
@@ -17,8 +17,9 @@ use crate::admin::storage_api::access::RequestContext;
|
||||
use crate::license::has_valid_license;
|
||||
use crate::server::has_path_prefix;
|
||||
use crate::server::rate_limit::{
|
||||
LABEL_RATE_LIMIT_SCOPE, METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL, RATE_LIMIT_SCOPE_CONSOLE, RateLimitDecision,
|
||||
RateLimitQuota, RateLimiter, apply_throttle_headers, client_ip,
|
||||
LABEL_RATE_LIMIT_DIMENSION, LABEL_RATE_LIMIT_SCOPE, METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL,
|
||||
RATE_LIMIT_DIMENSION_CLIENT_IP, RATE_LIMIT_SCOPE_CONSOLE, RateLimitDecision, RateLimitQuota, RateLimiter,
|
||||
apply_throttle_headers, client_ip,
|
||||
};
|
||||
use crate::server::{
|
||||
CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HeaderMapCarrier, HealthProbe, LICENSE, RUSTFS_ADMIN_PREFIX,
|
||||
@@ -581,15 +582,16 @@ fn setup_console_middleware_stack(
|
||||
// without one fail open rather than trusting spoofable headers.
|
||||
if rate_limit_enable {
|
||||
if let Some(quota) = RateLimitQuota::per_minute(rate_limit_rpm, 0) {
|
||||
let limiter = Arc::new(RateLimiter::new(quota));
|
||||
let limiter: Arc<RateLimiter> = Arc::new(RateLimiter::new(quota));
|
||||
app = app.layer(middleware::from_fn(move |req: Request, next: middleware::Next| {
|
||||
let limiter = limiter.clone();
|
||||
async move {
|
||||
match client_ip(&req).map(|ip| limiter.check(ip)) {
|
||||
match client_ip(&req).map(|ip| limiter.check(&ip)) {
|
||||
Some(RateLimitDecision::Limited(throttle)) => {
|
||||
metrics::counter!(
|
||||
METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL,
|
||||
LABEL_RATE_LIMIT_SCOPE => RATE_LIMIT_SCOPE_CONSOLE
|
||||
LABEL_RATE_LIMIT_SCOPE => RATE_LIMIT_SCOPE_CONSOLE,
|
||||
LABEL_RATE_LIMIT_DIMENSION => RATE_LIMIT_DIMENSION_CLIENT_IP
|
||||
)
|
||||
.increment(1);
|
||||
debug!(retry_after_secs = throttle.retry_after_secs, "Console request rejected by rate limit");
|
||||
|
||||
+32
-25
@@ -609,6 +609,28 @@ pub async fn start_http_server(
|
||||
);
|
||||
}
|
||||
|
||||
// Expanded virtual-hosted-style domain set (with port variants); shared by
|
||||
// the s3s host router below and the rate limit layer's bucket extraction.
|
||||
let host_domain_sets = if !config.server_domains.is_empty() && !config.console_enable {
|
||||
MultiDomain::new(&config.server_domains).map_err(Error::other)?; // validate domains
|
||||
|
||||
// add the default port number to the given server domains
|
||||
let mut domain_sets = std::collections::HashSet::new();
|
||||
for domain in &config.server_domains {
|
||||
domain_sets.insert(domain.to_string());
|
||||
if let Some((host, _)) = domain.split_once(':') {
|
||||
domain_sets.insert(format!("{host}:{local_port}"));
|
||||
} else {
|
||||
domain_sets.insert(format!("{domain}:{local_port}"));
|
||||
}
|
||||
}
|
||||
|
||||
Some(domain_sets)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let rate_limit_vh_domains: Vec<String> = host_domain_sets.iter().flatten().cloned().collect();
|
||||
|
||||
// Setup S3 service
|
||||
// This project uses the S3S library to implement S3 services
|
||||
let s3_service = {
|
||||
@@ -620,24 +642,6 @@ pub async fn start_http_server(
|
||||
|
||||
let access_key = config.access_key.clone();
|
||||
let secret_key = config.secret_key.clone();
|
||||
let host_domain_sets = if !config.server_domains.is_empty() && !config.console_enable {
|
||||
MultiDomain::new(&config.server_domains).map_err(Error::other)?; // validate domains
|
||||
|
||||
// add the default port number to the given server domains
|
||||
let mut domain_sets = std::collections::HashSet::new();
|
||||
for domain in &config.server_domains {
|
||||
domain_sets.insert(domain.to_string());
|
||||
if let Some((host, _)) = domain.split_once(':') {
|
||||
domain_sets.insert(format!("{host}:{local_port}"));
|
||||
} else {
|
||||
domain_sets.insert(format!("{domain}:{local_port}"));
|
||||
}
|
||||
}
|
||||
|
||||
Some(domain_sets)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let metadata_route_host = host_domain_sets
|
||||
.as_ref()
|
||||
.map(MultiDomain::new)
|
||||
@@ -702,20 +706,23 @@ pub async fn start_http_server(
|
||||
);
|
||||
}
|
||||
|
||||
// Per-client S3 API rate limiting (backlog#1191). Built once so every
|
||||
// connection's stack shares the same limiter state; `None` (the default)
|
||||
// leaves the request path unchanged.
|
||||
let api_rate_limit_layer = api_rate_limit_layer_from_env();
|
||||
// Per-client / per-bucket S3 API rate limiting (backlog#1191). Built once
|
||||
// so every connection's stack shares the same limiter state; `None` (the
|
||||
// default) leaves the request path unchanged.
|
||||
let api_rate_limit_layer = api_rate_limit_layer_from_env(rate_limit_vh_domains);
|
||||
match &api_rate_limit_layer {
|
||||
Some(layer) => {
|
||||
let quota = layer.quota();
|
||||
let client_quota = layer.client_quota();
|
||||
let bucket_quota = layer.bucket_quota();
|
||||
info!(
|
||||
event = EVENT_API_RATE_LIMIT_STATE,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_HTTP,
|
||||
state = "enabled",
|
||||
requests_per_minute = quota.requests_per_minute,
|
||||
burst = quota.burst,
|
||||
client_rpm = client_quota.map(|q| q.requests_per_minute).unwrap_or(0),
|
||||
client_burst = client_quota.map(|q| q.burst).unwrap_or(0),
|
||||
bucket_rpm = bucket_quota.map(|q| q.requests_per_minute).unwrap_or(0),
|
||||
bucket_burst = bucket_quota.map(|q| q.burst).unwrap_or(0),
|
||||
"API rate limit state changed"
|
||||
);
|
||||
}
|
||||
|
||||
+428
-114
@@ -14,35 +14,48 @@
|
||||
|
||||
//! Per-client request rate limiting for the S3 API (backlog#1191).
|
||||
//!
|
||||
//! A token-bucket limiter keyed by client IP. Over-limit requests are rejected
|
||||
//! with `429 Too Many Requests`, a `Retry-After` header, and the
|
||||
//! `x-ratelimit-*` headers already used by the Swift protocol error mapping.
|
||||
//! A token-bucket limiter with two optional dimensions:
|
||||
//! - **client IP** (`RUSTFS_API_RATE_LIMIT_RPM`/`_BURST`): per-client fairness
|
||||
//! and abuse protection;
|
||||
//! - **bucket** (`RUSTFS_API_RATE_LIMIT_BUCKET_RPM`/`_BURST`): a collective
|
||||
//! budget per addressed bucket, protecting the server from one hot bucket
|
||||
//! regardless of how many client IPs the traffic comes from.
|
||||
//!
|
||||
//! Over-limit requests are rejected with `429 Too Many Requests`, a
|
||||
//! `Retry-After` header, and the `x-ratelimit-*` headers already used by the
|
||||
//! Swift protocol error mapping.
|
||||
//!
|
||||
//! Scope and invariants:
|
||||
//! - **Opt-in, default off.** [`api_rate_limit_layer_from_env`] returns `None`
|
||||
//! unless `RUSTFS_API_RATE_LIMIT_ENABLE=true` and `RUSTFS_API_RATE_LIMIT_RPM > 0`;
|
||||
//! the layer is then absent from the service stack and the request path is
|
||||
//! unchanged.
|
||||
//! unless `RUSTFS_API_RATE_LIMIT_ENABLE=true` and at least one dimension has
|
||||
//! a non-zero RPM; the layer is then absent from the service stack and the
|
||||
//! request path is unchanged.
|
||||
//! - **Client identity is never taken from request headers.** The key is the
|
||||
//! validated [`ClientInfo::real_ip`] inserted by the trusted-proxy layer
|
||||
//! (which only honors `X-Forwarded-For` from configured proxies) or, absent
|
||||
//! that, the socket peer address ([`RemoteAddr`]). A raw `X-Forwarded-For`
|
||||
//! header can therefore not be used to escape into an attacker-chosen bucket.
|
||||
//! - **Bucket extraction mirrors s3s host routing.** Virtual-hosted-style
|
||||
//! requests resolve the bucket from the Host/authority against the same
|
||||
//! expanded server-domain set the s3s router uses; everything else takes the
|
||||
//! first path segment. Admin and table-catalog namespaces are not buckets.
|
||||
//! - **Infra traffic is exempt** ([`is_rate_limit_exempt_path`]): health and
|
||||
//! profiling probes, internode RPC/gRPC, and the console (which has its own
|
||||
//! limiter, sharing this module's [`RateLimiter`] core).
|
||||
//! - This is client-facing abuse protection, not internal backpressure — it is
|
||||
//! unrelated to `workload_admission`, which schedules already-admitted work.
|
||||
//!
|
||||
//! Memory is bounded: buckets live in [`SHARD_COUNT`] independently locked
|
||||
//! shards, each capped and periodically swept. A bucket that has been idle
|
||||
//! long enough to have refilled completely is indistinguishable from a fresh
|
||||
//! one, so eviction never makes the limiter more permissive than a restart.
|
||||
//! Memory is bounded per dimension: buckets live in [`SHARD_COUNT`]
|
||||
//! independently locked shards, each capped and periodically swept. A bucket
|
||||
//! that has been idle long enough to have refilled completely is
|
||||
//! indistinguishable from a fresh one, so eviction never makes the limiter
|
||||
//! more permissive than a restart. The cap matters doubly for the bucket
|
||||
//! dimension, whose key space (bucket names) is attacker-chosen.
|
||||
|
||||
use crate::server::{
|
||||
CONSOLE_PREFIX, FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_HEALTH_CLUSTER_PATH,
|
||||
MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
|
||||
RPC_PREFIX, RemoteAddr, TONIC_PREFIX, has_path_prefix,
|
||||
RPC_PREFIX, RemoteAddr, TONIC_PREFIX, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::{Either, Ready, ready};
|
||||
@@ -50,8 +63,9 @@ use http::{HeaderMap, HeaderValue, Request, Response, StatusCode};
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use metrics::counter;
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::borrow::Borrow;
|
||||
use std::collections::HashMap;
|
||||
use std::hash::{BuildHasher, RandomState};
|
||||
use std::hash::{BuildHasher, Hash, RandomState};
|
||||
use std::net::IpAddr;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
@@ -66,6 +80,9 @@ pub(crate) const METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL: &str = "rustfs_
|
||||
pub(crate) const LABEL_RATE_LIMIT_SCOPE: &str = "scope";
|
||||
pub(crate) const RATE_LIMIT_SCOPE_S3_API: &str = "s3_api";
|
||||
pub(crate) const RATE_LIMIT_SCOPE_CONSOLE: &str = "console";
|
||||
pub(crate) const LABEL_RATE_LIMIT_DIMENSION: &str = "dimension";
|
||||
pub(crate) const RATE_LIMIT_DIMENSION_CLIENT_IP: &str = "client_ip";
|
||||
const RATE_LIMIT_DIMENSION_BUCKET: &str = "bucket";
|
||||
|
||||
// Header names shared with the Swift error mapping (crates/protocols swift/errors.rs).
|
||||
const X_RATE_LIMIT_LIMIT: &str = "x-ratelimit-limit";
|
||||
@@ -75,14 +92,15 @@ const X_REQUEST_ID: &str = "x-request-id";
|
||||
|
||||
/// Shards bound lock contention: each request locks 1/32 of the key space.
|
||||
const SHARD_COUNT: usize = 32;
|
||||
/// Upper bound on tracked client IPs across all shards (~100 bytes each, so
|
||||
/// worst case a few MiB). Real distinct-IP cardinality is bounded by actual
|
||||
/// TCP peers (or validated proxy clients), never by spoofable headers.
|
||||
/// Upper bound on tracked keys per limiter instance (~100 bytes each, so
|
||||
/// worst case a few MiB). For the client dimension real cardinality is
|
||||
/// bounded by actual TCP peers (or validated proxy clients); for the bucket
|
||||
/// dimension the cap is the defense, since names are attacker-chosen.
|
||||
const MAX_TRACKED_CLIENTS: usize = 100_000;
|
||||
/// Per-shard sweep cadence for dropping refilled-idle buckets.
|
||||
const CLEANUP_INTERVAL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Sustained-plus-burst request budget for one client.
|
||||
/// Sustained-plus-burst request budget for one key.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct RateLimitQuota {
|
||||
/// Sustained budget in requests per minute.
|
||||
@@ -120,8 +138,8 @@ pub enum RateLimitDecision {
|
||||
Limited(ThrottleInfo),
|
||||
}
|
||||
|
||||
/// One client's bucket. Capacity and refill rate are limiter-wide (single
|
||||
/// quota for all clients), so they are not duplicated per entry.
|
||||
/// One key's bucket. Capacity and refill rate are limiter-wide (single quota
|
||||
/// for all keys), so they are not duplicated per entry.
|
||||
#[derive(Debug)]
|
||||
struct TokenBucket {
|
||||
tokens: f64,
|
||||
@@ -163,16 +181,17 @@ impl TokenBucket {
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Shard {
|
||||
clients: HashMap<IpAddr, TokenBucket>,
|
||||
struct Shard<K> {
|
||||
clients: HashMap<K, TokenBucket>,
|
||||
next_cleanup: Instant,
|
||||
}
|
||||
|
||||
/// Sharded per-IP token-bucket limiter. Shared by the S3 API tower layer and
|
||||
/// the console middleware (each with its own instance and quota).
|
||||
/// Sharded token-bucket limiter over an arbitrary key dimension (client IP,
|
||||
/// bucket name, ...). Shared by the S3 API tower layer and the console
|
||||
/// middleware, each instance with its own quota.
|
||||
#[derive(Debug)]
|
||||
pub struct RateLimiter {
|
||||
shards: Vec<Mutex<Shard>>,
|
||||
pub struct RateLimiter<K = IpAddr> {
|
||||
shards: Vec<Mutex<Shard<K>>>,
|
||||
shard_hasher: RandomState,
|
||||
quota: RateLimitQuota,
|
||||
capacity: f64,
|
||||
@@ -180,7 +199,7 @@ pub struct RateLimiter {
|
||||
max_clients_per_shard: usize,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
impl<K: Hash + Eq + Clone> RateLimiter<K> {
|
||||
pub fn new(quota: RateLimitQuota) -> Self {
|
||||
Self::with_limits(quota, SHARD_COUNT, MAX_TRACKED_CLIENTS)
|
||||
}
|
||||
@@ -208,13 +227,23 @@ impl RateLimiter {
|
||||
self.quota
|
||||
}
|
||||
|
||||
/// Consume one token for `ip`, creating its bucket on first sight.
|
||||
pub fn check(&self, ip: IpAddr) -> RateLimitDecision {
|
||||
self.check_at(ip, Instant::now())
|
||||
/// Consume one token for `key`, creating its bucket on first sight. The
|
||||
/// borrowed-key form keeps the hit path allocation-free for owned key
|
||||
/// types (e.g. `&str` lookups against `String` keys).
|
||||
pub fn check<Q>(&self, key: &Q) -> RateLimitDecision
|
||||
where
|
||||
K: Borrow<Q>,
|
||||
Q: Hash + Eq + ToOwned<Owned = K> + ?Sized,
|
||||
{
|
||||
self.check_at(key, Instant::now())
|
||||
}
|
||||
|
||||
fn check_at(&self, ip: IpAddr, now: Instant) -> RateLimitDecision {
|
||||
let shard = &self.shards[self.shard_index(ip)];
|
||||
fn check_at<Q>(&self, key: &Q, now: Instant) -> RateLimitDecision
|
||||
where
|
||||
K: Borrow<Q>,
|
||||
Q: Hash + Eq + ToOwned<Owned = K> + ?Sized,
|
||||
{
|
||||
let shard = &self.shards[self.shard_index(key)];
|
||||
let mut shard = shard.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
|
||||
if now >= shard.next_cleanup {
|
||||
@@ -225,33 +254,35 @@ impl RateLimiter {
|
||||
shard.next_cleanup = now + CLEANUP_INTERVAL;
|
||||
}
|
||||
|
||||
if let Some(bucket) = shard.clients.get_mut(&ip) {
|
||||
if let Some(bucket) = shard.clients.get_mut(key) {
|
||||
return bucket.try_consume(self.capacity, self.refill_per_sec, now);
|
||||
}
|
||||
|
||||
if shard.clients.len() >= self.max_clients_per_shard {
|
||||
// At capacity: reclaim the most idle entry (closest to full, so the
|
||||
// least information is lost). O(shard len) but only under a
|
||||
// distinct-IP flood, where correctness beats micro-latency.
|
||||
// distinct-key flood, where correctness beats micro-latency.
|
||||
if let Some(stalest) = shard
|
||||
.clients
|
||||
.iter()
|
||||
.min_by_key(|(_, bucket)| bucket.last_refill)
|
||||
.map(|(ip, _)| *ip)
|
||||
.map(|(key, _)| key.clone())
|
||||
{
|
||||
shard.clients.remove(&stalest);
|
||||
// Turbofish: without it inference unifies the lookup type with
|
||||
// this method's `Q` instead of `K`.
|
||||
shard.clients.remove::<K>(&stalest);
|
||||
}
|
||||
}
|
||||
|
||||
shard
|
||||
.clients
|
||||
.entry(ip)
|
||||
.entry(key.to_owned())
|
||||
.or_insert_with(|| TokenBucket::full(self.capacity, now))
|
||||
.try_consume(self.capacity, self.refill_per_sec, now)
|
||||
}
|
||||
|
||||
fn shard_index(&self, ip: IpAddr) -> usize {
|
||||
(self.shard_hasher.hash_one(ip) as usize) % self.shards.len()
|
||||
fn shard_index<Q: Hash + ?Sized>(&self, key: &Q) -> usize {
|
||||
(self.shard_hasher.hash_one(key) as usize) % self.shards.len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -276,6 +307,62 @@ pub(crate) fn client_ip<B>(req: &Request<B>) -> Option<IpAddr> {
|
||||
req.extensions().get::<RemoteAddr>().map(|remote| remote.0.ip())
|
||||
}
|
||||
|
||||
/// Resolve the host a request addresses, for virtual-hosted-style bucket
|
||||
/// extraction (HTTP/2 carries it in the URI authority, HTTP/1.1 in `Host`).
|
||||
fn request_host<B>(req: &Request<B>) -> Option<&str> {
|
||||
if let Some(authority) = req.uri().authority() {
|
||||
return Some(authority.as_str());
|
||||
}
|
||||
req.headers().get(http::header::HOST).and_then(|value| value.to_str().ok())
|
||||
}
|
||||
|
||||
/// `bucket.domain` → `bucket` when `host` is a subdomain of `domain`
|
||||
/// (ASCII-case-insensitive, port-inclusive — the configured domain set
|
||||
/// already carries port variants).
|
||||
fn strip_vh_prefix<'a>(host: &'a str, domain: &str) -> Option<&'a str> {
|
||||
let (host_len, domain_len) = (host.len(), domain.len());
|
||||
if host_len > domain_len + 1
|
||||
&& host.as_bytes()[host_len - domain_len - 1] == b'.'
|
||||
&& host[host_len - domain_len..].eq_ignore_ascii_case(domain)
|
||||
{
|
||||
Some(&host[..host_len - domain_len - 1])
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the bucket a request addresses, if any.
|
||||
///
|
||||
/// Mirrors s3s host routing: on a configured virtual-hosted-style domain the
|
||||
/// bucket is the host prefix (an exact domain match means path-style on that
|
||||
/// domain); otherwise the first path segment. Admin and table-catalog
|
||||
/// namespaces are not buckets. Best-effort by design — a request this cannot
|
||||
/// classify simply skips the bucket dimension (the client dimension still
|
||||
/// applies).
|
||||
fn request_bucket<'a, B>(req: &'a Request<B>, vh_domains: &[String]) -> Option<&'a str> {
|
||||
let path = req.uri().path();
|
||||
if is_admin_path(path) || is_table_catalog_path(path) {
|
||||
return None;
|
||||
}
|
||||
|
||||
if !vh_domains.is_empty()
|
||||
&& let Some(host) = request_host(req)
|
||||
{
|
||||
for domain in vh_domains {
|
||||
if host.eq_ignore_ascii_case(domain) {
|
||||
// Path-style request on the API domain itself.
|
||||
break;
|
||||
}
|
||||
if let Some(bucket) = strip_vh_prefix(host, domain) {
|
||||
return Some(bucket);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let bucket = path.trim_start_matches('/').split('/').next().unwrap_or("");
|
||||
(!bucket.is_empty()).then_some(bucket)
|
||||
}
|
||||
|
||||
/// Paths exempt from S3 API rate limiting.
|
||||
///
|
||||
/// - Health/profiling probes: kubelet and load-balancer checks must never be
|
||||
@@ -357,45 +444,70 @@ fn s3_too_many_requests_response(request_id: Option<&HeaderValue>, limit_rpm: u3
|
||||
|
||||
/// Build the S3 API rate limit layer from `RUSTFS_API_RATE_LIMIT_*`.
|
||||
///
|
||||
/// Returns `None` (no layer in the stack, zero request-path change) unless
|
||||
/// explicitly enabled with a non-zero RPM.
|
||||
pub fn api_rate_limit_layer_from_env() -> Option<RateLimitLayer> {
|
||||
/// `vh_domains` is the expanded virtual-hosted-style domain set (with port
|
||||
/// variants) the s3s host router uses; pass an empty vec when virtual-hosted
|
||||
/// routing is not configured. Returns `None` (no layer in the stack, zero
|
||||
/// request-path change) unless explicitly enabled with a non-zero RPM on at
|
||||
/// least one dimension.
|
||||
pub fn api_rate_limit_layer_from_env(vh_domains: Vec<String>) -> Option<RateLimitLayer> {
|
||||
if !rustfs_utils::get_env_bool(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, rustfs_config::DEFAULT_API_RATE_LIMIT_ENABLE) {
|
||||
return None;
|
||||
}
|
||||
let rpm = rustfs_utils::get_env_u32(rustfs_config::ENV_API_RATE_LIMIT_RPM, rustfs_config::DEFAULT_API_RATE_LIMIT_RPM);
|
||||
let burst = rustfs_utils::get_env_u32(rustfs_config::ENV_API_RATE_LIMIT_BURST, rustfs_config::DEFAULT_API_RATE_LIMIT_BURST);
|
||||
let Some(quota) = RateLimitQuota::per_minute(rpm, burst) else {
|
||||
let client_quota = RateLimitQuota::per_minute(
|
||||
rustfs_utils::get_env_u32(rustfs_config::ENV_API_RATE_LIMIT_RPM, rustfs_config::DEFAULT_API_RATE_LIMIT_RPM),
|
||||
rustfs_utils::get_env_u32(rustfs_config::ENV_API_RATE_LIMIT_BURST, rustfs_config::DEFAULT_API_RATE_LIMIT_BURST),
|
||||
);
|
||||
let bucket_quota = RateLimitQuota::per_minute(
|
||||
rustfs_utils::get_env_u32(
|
||||
rustfs_config::ENV_API_RATE_LIMIT_BUCKET_RPM,
|
||||
rustfs_config::DEFAULT_API_RATE_LIMIT_BUCKET_RPM,
|
||||
),
|
||||
rustfs_utils::get_env_u32(
|
||||
rustfs_config::ENV_API_RATE_LIMIT_BUCKET_BURST,
|
||||
rustfs_config::DEFAULT_API_RATE_LIMIT_BUCKET_BURST,
|
||||
),
|
||||
);
|
||||
if client_quota.is_none() && bucket_quota.is_none() {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_RATE_LIMIT,
|
||||
"{} is enabled but {} is 0; API rate limiting stays inactive",
|
||||
"{} is enabled but {} and {} are both 0; API rate limiting stays inactive",
|
||||
rustfs_config::ENV_API_RATE_LIMIT_ENABLE,
|
||||
rustfs_config::ENV_API_RATE_LIMIT_RPM
|
||||
rustfs_config::ENV_API_RATE_LIMIT_RPM,
|
||||
rustfs_config::ENV_API_RATE_LIMIT_BUCKET_RPM
|
||||
);
|
||||
return None;
|
||||
};
|
||||
Some(RateLimitLayer::new(quota))
|
||||
}
|
||||
Some(RateLimitLayer::new(client_quota, bucket_quota, vh_domains))
|
||||
}
|
||||
|
||||
/// Tower layer enforcing the per-IP quota on the external S3 service stack.
|
||||
/// Tower layer enforcing the per-client-IP and/or per-bucket quotas on the
|
||||
/// external S3 service stack.
|
||||
///
|
||||
/// All clones share one [`RateLimiter`], so per-connection stack construction
|
||||
/// keeps a single global view of client budgets.
|
||||
/// All clones share the same [`RateLimiter`] instances, so per-connection
|
||||
/// stack construction keeps a single global view of budgets.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RateLimitLayer {
|
||||
limiter: Arc<RateLimiter>,
|
||||
client_limiter: Option<Arc<RateLimiter<IpAddr>>>,
|
||||
bucket_limiter: Option<Arc<RateLimiter<String>>>,
|
||||
vh_domains: Arc<[String]>,
|
||||
}
|
||||
|
||||
impl RateLimitLayer {
|
||||
pub fn new(quota: RateLimitQuota) -> Self {
|
||||
pub fn new(client_quota: Option<RateLimitQuota>, bucket_quota: Option<RateLimitQuota>, vh_domains: Vec<String>) -> Self {
|
||||
Self {
|
||||
limiter: Arc::new(RateLimiter::new(quota)),
|
||||
client_limiter: client_quota.map(|quota| Arc::new(RateLimiter::new(quota))),
|
||||
bucket_limiter: bucket_quota.map(|quota| Arc::new(RateLimiter::new(quota))),
|
||||
vh_domains: vh_domains.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn quota(&self) -> RateLimitQuota {
|
||||
self.limiter.quota()
|
||||
pub fn client_quota(&self) -> Option<RateLimitQuota> {
|
||||
self.client_limiter.as_ref().map(|limiter| limiter.quota())
|
||||
}
|
||||
|
||||
pub fn bucket_quota(&self) -> Option<RateLimitQuota> {
|
||||
self.bucket_limiter.as_ref().map(|limiter| limiter.quota())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -405,7 +517,9 @@ impl<S> Layer<S> for RateLimitLayer {
|
||||
fn layer(&self, inner: S) -> Self::Service {
|
||||
RateLimitService {
|
||||
inner,
|
||||
limiter: self.limiter.clone(),
|
||||
client_limiter: self.client_limiter.clone(),
|
||||
bucket_limiter: self.bucket_limiter.clone(),
|
||||
vh_domains: self.vh_domains.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -416,7 +530,39 @@ impl<S> Layer<S> for RateLimitLayer {
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RateLimitService<S> {
|
||||
inner: S,
|
||||
limiter: Arc<RateLimiter>,
|
||||
client_limiter: Option<Arc<RateLimiter<IpAddr>>>,
|
||||
bucket_limiter: Option<Arc<RateLimiter<String>>>,
|
||||
vh_domains: Arc<[String]>,
|
||||
}
|
||||
|
||||
fn rejected_response<F, E, ReqBody>(
|
||||
req: &Request<ReqBody>,
|
||||
dimension: &'static str,
|
||||
key: &dyn std::fmt::Display,
|
||||
limit_rpm: u32,
|
||||
throttle: &ThrottleInfo,
|
||||
) -> Either<F, Ready<Result<Response<BoxBody>, E>>> {
|
||||
counter!(
|
||||
METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL,
|
||||
LABEL_RATE_LIMIT_SCOPE => RATE_LIMIT_SCOPE_S3_API,
|
||||
LABEL_RATE_LIMIT_DIMENSION => dimension
|
||||
)
|
||||
.increment(1);
|
||||
debug!(
|
||||
event = EVENT_REQUEST_RATE_LIMITED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_RATE_LIMIT,
|
||||
scope = RATE_LIMIT_SCOPE_S3_API,
|
||||
dimension,
|
||||
key = %key,
|
||||
retry_after_secs = throttle.retry_after_secs,
|
||||
"Request rejected by API rate limit"
|
||||
);
|
||||
Either::Right(ready(Ok(s3_too_many_requests_response(
|
||||
req.headers().get(X_REQUEST_ID),
|
||||
limit_rpm,
|
||||
throttle,
|
||||
))))
|
||||
}
|
||||
|
||||
impl<S, ReqBody> Service<Request<ReqBody>> for RateLimitService<S>
|
||||
@@ -435,38 +581,34 @@ where
|
||||
if is_rate_limit_exempt_path(req.uri().path()) {
|
||||
return Either::Left(self.inner.call(req));
|
||||
}
|
||||
let Some(ip) = client_ip(&req) else {
|
||||
// Fail open: without a socket peer address there is no trustworthy
|
||||
// client identity, and deriving one from spoofable headers would
|
||||
// let clients pick their own bucket.
|
||||
return Either::Left(self.inner.call(req));
|
||||
};
|
||||
|
||||
match self.limiter.check(ip) {
|
||||
RateLimitDecision::Allowed => Either::Left(self.inner.call(req)),
|
||||
RateLimitDecision::Limited(throttle) => {
|
||||
counter!(
|
||||
METRIC_HTTP_SERVER_REQUESTS_RATE_LIMITED_TOTAL,
|
||||
LABEL_RATE_LIMIT_SCOPE => RATE_LIMIT_SCOPE_S3_API
|
||||
)
|
||||
.increment(1);
|
||||
debug!(
|
||||
event = EVENT_REQUEST_RATE_LIMITED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_RATE_LIMIT,
|
||||
scope = RATE_LIMIT_SCOPE_S3_API,
|
||||
client_ip = %ip,
|
||||
retry_after_secs = throttle.retry_after_secs,
|
||||
"Request rejected by API rate limit"
|
||||
);
|
||||
let limit_rpm = self.limiter.quota().requests_per_minute;
|
||||
Either::Right(ready(Ok(s3_too_many_requests_response(
|
||||
req.headers().get(X_REQUEST_ID),
|
||||
limit_rpm,
|
||||
if let Some(limiter) = &self.client_limiter {
|
||||
// A request without a socket peer address fails open: there is no
|
||||
// trustworthy client identity, and deriving one from spoofable
|
||||
// headers would let clients pick their own budget.
|
||||
if let Some(ip) = client_ip(&req)
|
||||
&& let RateLimitDecision::Limited(throttle) = limiter.check(&ip)
|
||||
{
|
||||
return rejected_response(
|
||||
&req,
|
||||
RATE_LIMIT_DIMENSION_CLIENT_IP,
|
||||
&ip,
|
||||
limiter.quota().requests_per_minute,
|
||||
&throttle,
|
||||
))))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(limiter) = &self.bucket_limiter
|
||||
&& let Some(bucket) = request_bucket(&req, &self.vh_domains)
|
||||
&& let RateLimitDecision::Limited(throttle) = limiter.check(bucket)
|
||||
{
|
||||
let limit_rpm = limiter.quota().requests_per_minute;
|
||||
let bucket = bucket.to_owned();
|
||||
return rejected_response(&req, RATE_LIMIT_DIMENSION_BUCKET, &bucket, limit_rpm, &throttle);
|
||||
}
|
||||
|
||||
Either::Left(self.inner.call(req))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -496,48 +638,58 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn bucket_exhaustion_returns_429_hints_and_window_recovers() {
|
||||
let limiter = RateLimiter::new(quota(60, 5)); // 1 token/sec, burst 5
|
||||
let limiter: RateLimiter = RateLimiter::new(quota(60, 5)); // 1 token/sec, burst 5
|
||||
let start = Instant::now();
|
||||
|
||||
for _ in 0..5 {
|
||||
assert_eq!(limiter.check_at(ip(1), start), RateLimitDecision::Allowed);
|
||||
assert_eq!(limiter.check_at(&ip(1), start), RateLimitDecision::Allowed);
|
||||
}
|
||||
let RateLimitDecision::Limited(throttle) = limiter.check_at(ip(1), start) else {
|
||||
let RateLimitDecision::Limited(throttle) = limiter.check_at(&ip(1), start) else {
|
||||
panic!("sixth request within the same instant must be limited");
|
||||
};
|
||||
assert_eq!(throttle.retry_after_secs, 1);
|
||||
assert_eq!(throttle.reset_after_secs, 5);
|
||||
|
||||
// One refill interval restores exactly one token.
|
||||
assert_eq!(limiter.check_at(ip(1), start + Duration::from_secs(1)), RateLimitDecision::Allowed);
|
||||
assert_eq!(limiter.check_at(&ip(1), start + Duration::from_secs(1)), RateLimitDecision::Allowed);
|
||||
assert!(matches!(
|
||||
limiter.check_at(ip(1), start + Duration::from_secs(1)),
|
||||
limiter.check_at(&ip(1), start + Duration::from_secs(1)),
|
||||
RateLimitDecision::Limited(_)
|
||||
));
|
||||
|
||||
// A full window restores the full burst.
|
||||
let later = start + Duration::from_secs(61);
|
||||
for _ in 0..5 {
|
||||
assert_eq!(limiter.check_at(ip(1), later), RateLimitDecision::Allowed);
|
||||
assert_eq!(limiter.check_at(&ip(1), later), RateLimitDecision::Allowed);
|
||||
}
|
||||
assert!(matches!(limiter.check_at(ip(1), later), RateLimitDecision::Limited(_)));
|
||||
assert!(matches!(limiter.check_at(&ip(1), later), RateLimitDecision::Limited(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clients_have_independent_buckets() {
|
||||
let limiter = RateLimiter::new(quota(60, 1));
|
||||
let limiter: RateLimiter = RateLimiter::new(quota(60, 1));
|
||||
let now = Instant::now();
|
||||
assert_eq!(limiter.check_at(ip(1), now), RateLimitDecision::Allowed);
|
||||
assert!(matches!(limiter.check_at(ip(1), now), RateLimitDecision::Limited(_)));
|
||||
assert_eq!(limiter.check_at(ip(2), now), RateLimitDecision::Allowed);
|
||||
assert_eq!(limiter.check_at(&ip(1), now), RateLimitDecision::Allowed);
|
||||
assert!(matches!(limiter.check_at(&ip(1), now), RateLimitDecision::Limited(_)));
|
||||
assert_eq!(limiter.check_at(&ip(2), now), RateLimitDecision::Allowed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_keys_share_budget_across_borrowed_lookups() {
|
||||
let limiter: RateLimiter<String> = RateLimiter::new(quota(60, 2));
|
||||
let now = Instant::now();
|
||||
assert_eq!(limiter.check_at("photos", now), RateLimitDecision::Allowed);
|
||||
assert_eq!(limiter.check_at("photos", now), RateLimitDecision::Allowed);
|
||||
assert!(matches!(limiter.check_at("photos", now), RateLimitDecision::Limited(_)));
|
||||
assert_eq!(limiter.check_at("logs", now), RateLimitDecision::Allowed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_clients_stay_bounded_under_distinct_ip_flood() {
|
||||
let limiter = RateLimiter::with_limits(quota(60, 1), 4, 64);
|
||||
let limiter: RateLimiter = RateLimiter::with_limits(quota(60, 1), 4, 64);
|
||||
let now = Instant::now();
|
||||
for i in 0..10_000u32 {
|
||||
limiter.check_at(IpAddr::V4(Ipv4Addr::from(i)), now);
|
||||
limiter.check_at(&IpAddr::V4(Ipv4Addr::from(i)), now);
|
||||
}
|
||||
assert!(
|
||||
limiter.tracked_clients() <= 64,
|
||||
@@ -546,23 +698,38 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracked_buckets_stay_bounded_under_random_name_flood() {
|
||||
// Bucket names are attacker-chosen strings; the cap is the defense.
|
||||
let limiter: RateLimiter<String> = RateLimiter::with_limits(quota(60, 1), 4, 64);
|
||||
let now = Instant::now();
|
||||
for i in 0..10_000u32 {
|
||||
limiter.check_at(format!("bucket-{i}").as_str(), now);
|
||||
}
|
||||
assert!(
|
||||
limiter.tracked_clients() <= 64,
|
||||
"tracked {} buckets, cap is 64",
|
||||
limiter.tracked_clients()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refilled_idle_buckets_are_swept() {
|
||||
let limiter = RateLimiter::with_limits(quota(60, 5), 1, 100);
|
||||
let limiter: RateLimiter = RateLimiter::with_limits(quota(60, 5), 1, 100);
|
||||
let start = Instant::now();
|
||||
limiter.check_at(ip(1), start);
|
||||
limiter.check_at(&ip(1), start);
|
||||
assert_eq!(limiter.tracked_clients(), 1);
|
||||
|
||||
// After burst/rate = 5s the bucket is full again; the next request past
|
||||
// the cleanup deadline sweeps it.
|
||||
let after_sweep = start + CLEANUP_INTERVAL + Duration::from_secs(6);
|
||||
limiter.check_at(ip(2), after_sweep);
|
||||
limiter.check_at(&ip(2), after_sweep);
|
||||
assert_eq!(limiter.tracked_clients(), 1, "idle refilled bucket must be dropped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_hammering_never_exceeds_burst() {
|
||||
let limiter = Arc::new(RateLimiter::new(quota(60, 100)));
|
||||
let limiter: Arc<RateLimiter> = Arc::new(RateLimiter::new(quota(60, 100)));
|
||||
let now = Instant::now();
|
||||
let allowed = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
|
||||
@@ -572,7 +739,7 @@ mod tests {
|
||||
let allowed = Arc::clone(&allowed);
|
||||
scope.spawn(move || {
|
||||
for _ in 0..50 {
|
||||
if limiter.check_at(ip(9), now) == RateLimitDecision::Allowed {
|
||||
if limiter.check_at(&ip(9), now) == RateLimitDecision::Allowed {
|
||||
allowed.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
@@ -612,6 +779,56 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- bucket extraction ----
|
||||
|
||||
fn request_with_host(host: &str, path: &str) -> Request<()> {
|
||||
let mut req = Request::builder().uri(path).body(()).expect("request");
|
||||
req.headers_mut()
|
||||
.insert(http::header::HOST, HeaderValue::from_str(host).expect("host"));
|
||||
req
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_bucket_takes_first_path_segment_for_path_style() {
|
||||
let domains: Vec<String> = vec![];
|
||||
let req = request_with_host("s3.example.com", "/photos/2024/cat.jpg");
|
||||
assert_eq!(request_bucket(&req, &domains), Some("photos"));
|
||||
|
||||
let root = request_with_host("s3.example.com", "/");
|
||||
assert_eq!(request_bucket(&root, &domains), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_bucket_resolves_virtual_hosted_style_against_domains() {
|
||||
let domains = vec!["s3.example.com".to_string(), "s3.example.com:9000".to_string()];
|
||||
|
||||
let vh = request_with_host("photos.s3.example.com", "/2024/cat.jpg");
|
||||
assert_eq!(request_bucket(&vh, &domains), Some("photos"));
|
||||
|
||||
let vh_port = request_with_host("photos.s3.example.com:9000", "/2024/cat.jpg");
|
||||
assert_eq!(request_bucket(&vh_port, &domains), Some("photos"));
|
||||
|
||||
let vh_case = request_with_host("Photos.S3.Example.COM", "/2024/cat.jpg");
|
||||
assert_eq!(request_bucket(&vh_case, &domains), Some("Photos"));
|
||||
|
||||
// Exact domain match means path-style on the API domain.
|
||||
let path_style = request_with_host("s3.example.com", "/photos/cat.jpg");
|
||||
assert_eq!(request_bucket(&path_style, &domains), Some("photos"));
|
||||
|
||||
// Unrelated hosts fall back to path-style extraction.
|
||||
let other = request_with_host("cdn.other.net", "/photos/cat.jpg");
|
||||
assert_eq!(request_bucket(&other, &domains), Some("photos"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_bucket_skips_admin_and_catalog_namespaces() {
|
||||
let domains: Vec<String> = vec![];
|
||||
for path in ["/rustfs/admin/v3/info", "/minio/admin/v3/info", "/iceberg/v1/config"] {
|
||||
let req = request_with_host("s3.example.com", path);
|
||||
assert_eq!(request_bucket(&req, &domains), None, "{path} must not be a bucket");
|
||||
}
|
||||
}
|
||||
|
||||
// ---- service-level tests ----
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -635,7 +852,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn service_with_quota(rpm: u32, burst: u32) -> RateLimitService<OkService> {
|
||||
RateLimitLayer::new(quota(rpm, burst)).layer(OkService)
|
||||
RateLimitLayer::new(Some(quota(rpm, burst)), None, vec![]).layer(OkService)
|
||||
}
|
||||
|
||||
fn request_from(peer: IpAddr, path: &str) -> Request<()> {
|
||||
@@ -744,6 +961,77 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_dimension_budget_is_shared_across_client_ips() {
|
||||
// Bucket dimension only (client dimension off): two IPs, one bucket.
|
||||
let mut service = RateLimitLayer::new(None, Some(quota(60, 2)), vec![]).layer(OkService);
|
||||
|
||||
assert_eq!(
|
||||
service.call(request_from(ip(1), "/photos/a.jpg")).await.expect("ok").status(),
|
||||
StatusCode::OK
|
||||
);
|
||||
assert_eq!(
|
||||
service.call(request_from(ip(2), "/photos/b.jpg")).await.expect("ok").status(),
|
||||
StatusCode::OK
|
||||
);
|
||||
// Third hit on the same bucket is rejected regardless of the new IP.
|
||||
assert_eq!(
|
||||
service.call(request_from(ip(3), "/photos/c.jpg")).await.expect("ok").status(),
|
||||
StatusCode::TOO_MANY_REQUESTS
|
||||
);
|
||||
// A different bucket has its own budget.
|
||||
assert_eq!(
|
||||
service.call(request_from(ip(3), "/logs/d.log")).await.expect("ok").status(),
|
||||
StatusCode::OK
|
||||
);
|
||||
// Requests without a bucket (e.g. ListBuckets) skip the dimension.
|
||||
assert_eq!(service.call(request_from(ip(3), "/")).await.expect("ok").status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_dimension_resolves_virtual_hosted_requests() {
|
||||
let domains = vec!["s3.example.com".to_string()];
|
||||
let mut service = RateLimitLayer::new(None, Some(quota(60, 1)), domains).layer(OkService);
|
||||
|
||||
let mut vh = request_from(ip(1), "/a.jpg");
|
||||
vh.headers_mut()
|
||||
.insert(http::header::HOST, HeaderValue::from_static("photos.s3.example.com"));
|
||||
assert_eq!(service.call(vh).await.expect("ok").status(), StatusCode::OK);
|
||||
|
||||
// Path-style hit on the same bucket shares the budget.
|
||||
let path_style = request_from(ip(2), "/photos/b.jpg");
|
||||
assert_eq!(service.call(path_style).await.expect("ok").status(), StatusCode::TOO_MANY_REQUESTS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn both_dimensions_apply_when_configured() {
|
||||
// Client burst 2, bucket burst 1: the bucket dimension trips first for
|
||||
// one bucket, the client dimension caps the client across buckets.
|
||||
let mut service = RateLimitLayer::new(Some(quota(60, 2)), Some(quota(60, 1)), vec![]).layer(OkService);
|
||||
|
||||
assert_eq!(
|
||||
service.call(request_from(ip(1), "/photos/a.jpg")).await.expect("ok").status(),
|
||||
StatusCode::OK
|
||||
);
|
||||
assert_eq!(
|
||||
service.call(request_from(ip(1), "/photos/b.jpg")).await.expect("ok").status(),
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"bucket budget exhausted"
|
||||
);
|
||||
// Client budget: the first request consumed 1, the rejected one also
|
||||
// consumed 1 (arrivals are counted), so the client is now exhausted too.
|
||||
assert_eq!(
|
||||
service.call(request_from(ip(1), "/logs/c.log")).await.expect("ok").status(),
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"client budget exhausted"
|
||||
);
|
||||
// A different client can still reach a fresh bucket.
|
||||
assert_eq!(
|
||||
service.call(request_from(ip(2), "/logs/c.log")).await.expect("ok").status(),
|
||||
StatusCode::OK
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn env_constructor_defaults_to_disabled() {
|
||||
@@ -752,24 +1040,29 @@ mod tests {
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, None::<&str>),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_RPM, None),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BURST, None),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BUCKET_RPM, None),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BUCKET_BURST, None),
|
||||
],
|
||||
|| {
|
||||
assert!(api_rate_limit_layer_from_env().is_none());
|
||||
assert!(api_rate_limit_layer_from_env(vec![]).is_none());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn env_constructor_requires_enable_and_nonzero_rpm() {
|
||||
fn env_constructor_requires_enable_and_a_nonzero_dimension() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_RPM, None),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BURST, None),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BUCKET_RPM, None),
|
||||
],
|
||||
|| {
|
||||
assert!(api_rate_limit_layer_from_env().is_none(), "enabled with rpm=0 stays inactive");
|
||||
assert!(
|
||||
api_rate_limit_layer_from_env(vec![]).is_none(),
|
||||
"enabled with all dimensions at 0 stays inactive"
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -777,10 +1070,9 @@ mod tests {
|
||||
[
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, Some("false")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_RPM, Some("600")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BURST, None),
|
||||
],
|
||||
|| {
|
||||
assert!(api_rate_limit_layer_from_env().is_none(), "rpm without enable stays inactive");
|
||||
assert!(api_rate_limit_layer_from_env(vec![]).is_none(), "rpm without enable stays inactive");
|
||||
},
|
||||
);
|
||||
|
||||
@@ -789,15 +1081,37 @@ mod tests {
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_RPM, Some("600")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BURST, Some("50")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BUCKET_RPM, None),
|
||||
],
|
||||
|| {
|
||||
let layer = api_rate_limit_layer_from_env().expect("enabled with rpm");
|
||||
let layer = api_rate_limit_layer_from_env(vec![]).expect("enabled with client rpm");
|
||||
assert_eq!(
|
||||
layer.quota(),
|
||||
RateLimitQuota {
|
||||
layer.client_quota(),
|
||||
Some(RateLimitQuota {
|
||||
requests_per_minute: 600,
|
||||
burst: 50
|
||||
}
|
||||
})
|
||||
);
|
||||
assert_eq!(layer.bucket_quota(), None);
|
||||
},
|
||||
);
|
||||
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_RPM, None),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BUCKET_RPM, Some("6000")),
|
||||
(rustfs_config::ENV_API_RATE_LIMIT_BUCKET_BURST, Some("100")),
|
||||
],
|
||||
|| {
|
||||
let layer = api_rate_limit_layer_from_env(vec![]).expect("enabled with bucket rpm only");
|
||||
assert_eq!(layer.client_quota(), None);
|
||||
assert_eq!(
|
||||
layer.bucket_quota(),
|
||||
Some(RateLimitQuota {
|
||||
requests_per_minute: 6000,
|
||||
burst: 100
|
||||
})
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user