mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
c82ee6be58
feat(api): wire opt-in per-client S3 API rate limiting (backlog#1191) RustFS shipped three rate-limiter implementations and none was wired to any request path: the tower layer never returned 429 (its over-limit branch passed requests through) and was never instantiated, the console env switches only logged, and the Swift token bucket was never called. Replace them with one working, default-off implementation: - Rewrite rustfs/src/server/rate_limit.rs as a sharded per-client-IP token-bucket limiter (32 mutex shards instead of one global RwLock write per request), bounded at 100k tracked IPs with lossless refilled-idle sweeps, returning 429 + Retry-After + x-ratelimit-* headers and an S3-style XML body. - Key on trusted-proxy-validated ClientInfo.real_ip, else the socket peer address; never read spoofable X-Forwarded-For/X-Real-IP headers. Requests without a resolvable identity fail open. The echoed request id is charset-gated to prevent reflected XML injection. - Wire the layer once at startup via option_layer between CatchPanicLayer and ReadinessGateLayer (external stack only), gated by new RUSTFS_API_RATE_LIMIT_ENABLE/_RPM/_BURST constants; health and profiling probes, internode RPC/gRPC, and the console are exempt. - Make RUSTFS_CONSOLE_RATE_LIMIT_ENABLE/_RPM actually enforce by reusing the same limiter core through an axum middleware. - Delete the dead Swift ratelimit module, its isolated tests, and the stale logging-guardrail entry; keep the live SwiftError 429 mapping. - Add unit tests (exhaustion/recovery with injected time, concurrency, cap eviction, spoofed-header and fail-open behavior, env matrix) and e2e tests proving 429 + Retry-After on the real server and zero behavior change with default configuration.
124 lines
4.0 KiB
Rust
124 lines
4.0 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//! Simple integration tests for Swift API that verify module interactions
|
|
|
|
#![cfg(feature = "swift")]
|
|
|
|
use rustfs_protocols::swift::{quota, slo, symlink, sync, tempurl, versioning};
|
|
use std::collections::HashMap;
|
|
|
|
/// Test sync configuration parsing
|
|
#[test]
|
|
fn test_sync_config_parsing() {
|
|
let mut metadata = HashMap::new();
|
|
metadata.insert("x-container-sync-to".to_string(), "https://remote/v1/AUTH_test/backup".to_string());
|
|
metadata.insert("x-container-sync-key".to_string(), "secret123".to_string());
|
|
|
|
let config = sync::SyncConfig::from_metadata(&metadata).unwrap().unwrap();
|
|
assert_eq!(config.sync_to, "https://remote/v1/AUTH_test/backup");
|
|
assert!(config.enabled);
|
|
}
|
|
|
|
/// Test sync signature generation
|
|
#[test]
|
|
fn test_sync_signatures() {
|
|
let path = "/v1/AUTH_test/container/object.txt";
|
|
let key = "sharedsecret";
|
|
|
|
let sig1 = sync::generate_sync_signature(path, key).expect("failed to generate sync signature");
|
|
let sig2 = sync::generate_sync_signature(path, key).expect("failed to generate sync signature");
|
|
|
|
assert_eq!(sig1, sig2);
|
|
assert_eq!(sig1.len(), 40); // HMAC-SHA1 = 40 hex chars
|
|
assert!(sync::verify_sync_signature(path, key, &sig1));
|
|
}
|
|
|
|
/// Test SLO manifest ETag calculation
|
|
#[test]
|
|
fn test_slo_etag() {
|
|
let manifest = slo::SLOManifest {
|
|
segments: vec![slo::SLOSegment {
|
|
path: "/c/seg1".to_string(),
|
|
size_bytes: 1024,
|
|
etag: "abc".to_string(),
|
|
range: None,
|
|
}],
|
|
created_at: None,
|
|
};
|
|
|
|
let etag = manifest.calculate_etag();
|
|
assert!(!etag.is_empty());
|
|
assert_eq!(manifest.total_size(), 1024);
|
|
}
|
|
|
|
/// Test TempURL signature generation
|
|
#[test]
|
|
fn test_tempurl_signature() {
|
|
let tempurl = tempurl::TempURL::new("secret".to_string());
|
|
let sig = tempurl.generate_signature("GET", 1735689600, "/v1/AUTH_test/c/o").unwrap();
|
|
assert_eq!(sig.len(), 40); // HMAC-SHA1
|
|
}
|
|
|
|
/// Test versioning name generation
|
|
#[test]
|
|
fn test_versioning_names() {
|
|
let name1 = versioning::generate_version_name("container", "file.txt");
|
|
let name2 = versioning::generate_version_name("container", "other.txt");
|
|
|
|
assert!(name1.contains("file.txt"));
|
|
assert!(name2.contains("other.txt"));
|
|
assert_ne!(name1, name2);
|
|
}
|
|
|
|
/// Test symlink detection
|
|
#[test]
|
|
fn test_symlink_detection() {
|
|
let mut metadata = HashMap::new();
|
|
metadata.insert("x-symlink-target".to_string(), "container/object".to_string());
|
|
|
|
// Just verify the function works - may require specific metadata format
|
|
let _is_symlink = symlink::is_symlink(&metadata);
|
|
}
|
|
|
|
/// Test quota structure
|
|
#[test]
|
|
fn test_quota_structure() {
|
|
let quota = quota::QuotaConfig {
|
|
quota_bytes: Some(1048576),
|
|
quota_count: Some(100),
|
|
};
|
|
assert_eq!(quota.quota_bytes, Some(1048576));
|
|
}
|
|
|
|
/// Test conflict resolution
|
|
#[test]
|
|
fn test_conflict_resolution() {
|
|
assert!(sync::resolve_conflict(2000, 1000, sync::ConflictResolution::LastWriteWins));
|
|
assert!(!sync::resolve_conflict(1000, 2000, sync::ConflictResolution::LastWriteWins));
|
|
assert!(sync::resolve_conflict(1500, 1500, sync::ConflictResolution::LastWriteWins));
|
|
}
|
|
|
|
/// Test sync retry queue
|
|
#[test]
|
|
fn test_sync_retry_queue() {
|
|
let mut entry = sync::SyncQueueEntry::new("file.txt".to_string(), "abc".to_string(), 1000);
|
|
entry.schedule_retry(2000);
|
|
|
|
assert_eq!(entry.retry_count, 1);
|
|
assert_eq!(entry.next_retry, 2060);
|
|
assert!(!entry.ready_for_retry(2000));
|
|
assert!(entry.ready_for_retry(2060));
|
|
}
|