mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-05 12:57:42 +00:00
feat: Implement whitelist-based HTTP response compression configuration (#1136)
Signed-off-by: 唐小鸭 <tangtang1251@qq.com> Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: loverustfs <hello@rustfs.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,485 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! HTTP Response Compression Module
|
||||
//!
|
||||
//! This module provides configurable HTTP response compression functionality
|
||||
//! using a whitelist-based approach. Unlike traditional blacklist approaches,
|
||||
//! this design only compresses explicitly configured content types, which:
|
||||
//!
|
||||
//! 1. Preserves Content-Length for all other responses (better browser UX)
|
||||
//! 2. Aligns with MinIO's opt-in compression behavior
|
||||
//! 3. Provides fine-grained control over what gets compressed
|
||||
//!
|
||||
//! # Configuration
|
||||
//!
|
||||
//! Compression can be configured via environment variables or command line options:
|
||||
//!
|
||||
//! - `RUSTFS_COMPRESS_ENABLE` - Enable/disable compression (default: off)
|
||||
//! - `RUSTFS_COMPRESS_EXTENSIONS` - File extensions to compress (e.g., `.txt,.log,.csv`)
|
||||
//! - `RUSTFS_COMPRESS_MIME_TYPES` - MIME types to compress (e.g., `text/*,application/json`)
|
||||
//! - `RUSTFS_COMPRESS_MIN_SIZE` - Minimum file size for compression (default: 1000 bytes)
|
||||
//!
|
||||
//! # Example
|
||||
//!
|
||||
//! ```bash
|
||||
//! RUSTFS_COMPRESS_ENABLE=on \
|
||||
//! RUSTFS_COMPRESS_EXTENSIONS=.txt,.log,.csv \
|
||||
//! RUSTFS_COMPRESS_MIME_TYPES=text/*,application/json \
|
||||
//! RUSTFS_COMPRESS_MIN_SIZE=1000 \
|
||||
//! rustfs /data
|
||||
//! ```
|
||||
|
||||
use http::Response;
|
||||
use rustfs_config::{
|
||||
DEFAULT_COMPRESS_ENABLE, DEFAULT_COMPRESS_EXTENSIONS, DEFAULT_COMPRESS_MIME_TYPES, DEFAULT_COMPRESS_MIN_SIZE,
|
||||
ENV_COMPRESS_ENABLE, ENV_COMPRESS_EXTENSIONS, ENV_COMPRESS_MIME_TYPES, ENV_COMPRESS_MIN_SIZE, EnableState,
|
||||
};
|
||||
use std::str::FromStr;
|
||||
use tower_http::compression::predicate::Predicate;
|
||||
use tracing::debug;
|
||||
|
||||
/// Configuration for HTTP response compression.
|
||||
///
|
||||
/// This structure holds the whitelist-based compression settings:
|
||||
/// - File extensions that should be compressed (checked via Content-Disposition header)
|
||||
/// - MIME types that should be compressed (supports wildcards like `text/*`)
|
||||
/// - Minimum file size threshold for compression
|
||||
///
|
||||
/// When compression is enabled, only responses matching these criteria will be compressed.
|
||||
/// This approach aligns with MinIO's behavior where compression is opt-in rather than default.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CompressionConfig {
|
||||
/// Whether compression is enabled
|
||||
pub enabled: bool,
|
||||
/// File extensions to compress (normalized to lowercase with leading dot)
|
||||
pub extensions: Vec<String>,
|
||||
/// MIME type patterns to compress (supports wildcards like `text/*`)
|
||||
pub mime_patterns: Vec<String>,
|
||||
/// Minimum file size (in bytes) for compression
|
||||
pub min_size: u64,
|
||||
}
|
||||
|
||||
impl CompressionConfig {
|
||||
/// Create a new compression configuration from environment variables
|
||||
///
|
||||
/// Reads the following environment variables:
|
||||
/// - `RUSTFS_COMPRESS_ENABLE` - Enable/disable compression (default: false)
|
||||
/// - `RUSTFS_COMPRESS_EXTENSIONS` - File extensions to compress (default: "")
|
||||
/// - `RUSTFS_COMPRESS_MIME_TYPES` - MIME types to compress (default: "text/*,application/json,...")
|
||||
/// - `RUSTFS_COMPRESS_MIN_SIZE` - Minimum file size for compression (default: 1000)
|
||||
pub fn from_env() -> Self {
|
||||
// Read compression enable state
|
||||
let enabled = std::env::var(ENV_COMPRESS_ENABLE)
|
||||
.ok()
|
||||
.and_then(|v| EnableState::from_str(&v).ok())
|
||||
.map(|state| state.is_enabled())
|
||||
.unwrap_or(DEFAULT_COMPRESS_ENABLE);
|
||||
|
||||
// Read file extensions
|
||||
let extensions_str = std::env::var(ENV_COMPRESS_EXTENSIONS).unwrap_or_else(|_| DEFAULT_COMPRESS_EXTENSIONS.to_string());
|
||||
let extensions: Vec<String> = if extensions_str.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
extensions_str
|
||||
.split(',')
|
||||
.map(|s| {
|
||||
let s = s.trim().to_lowercase();
|
||||
if s.starts_with('.') { s } else { format!(".{s}") }
|
||||
})
|
||||
.filter(|s| s.len() > 1)
|
||||
.collect()
|
||||
};
|
||||
|
||||
// Read MIME type patterns
|
||||
let mime_types_str = std::env::var(ENV_COMPRESS_MIME_TYPES).unwrap_or_else(|_| DEFAULT_COMPRESS_MIME_TYPES.to_string());
|
||||
let mime_patterns: Vec<String> = if mime_types_str.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
mime_types_str
|
||||
.split(',')
|
||||
.map(|s| s.trim().to_lowercase())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect()
|
||||
};
|
||||
|
||||
// Read minimum file size
|
||||
let min_size = std::env::var(ENV_COMPRESS_MIN_SIZE)
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(DEFAULT_COMPRESS_MIN_SIZE);
|
||||
|
||||
Self {
|
||||
enabled,
|
||||
extensions,
|
||||
mime_patterns,
|
||||
min_size,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a MIME type matches any of the configured patterns
|
||||
pub(crate) fn matches_mime_type(&self, content_type: &str) -> bool {
|
||||
let ct_lower = content_type.to_lowercase();
|
||||
// Extract the main MIME type (before any parameters like charset)
|
||||
let main_type = ct_lower.split(';').next().unwrap_or(&ct_lower).trim();
|
||||
|
||||
for pattern in &self.mime_patterns {
|
||||
if pattern.ends_with("/*") {
|
||||
// Wildcard pattern like "text/*"
|
||||
let prefix = &pattern[..pattern.len() - 1]; // "text/"
|
||||
if main_type.starts_with(prefix) {
|
||||
return true;
|
||||
}
|
||||
} else if main_type == pattern {
|
||||
// Exact match
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if a filename matches any of the configured extensions
|
||||
/// The filename is extracted from Content-Disposition header
|
||||
pub(crate) fn matches_extension(&self, filename: &str) -> bool {
|
||||
if self.extensions.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let filename_lower = filename.to_lowercase();
|
||||
for ext in &self.extensions {
|
||||
if filename_lower.ends_with(ext) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Extract filename from Content-Disposition header
|
||||
/// Format: attachment; filename="example.txt" or attachment; filename=example.txt
|
||||
pub(crate) fn extract_filename_from_content_disposition(header_value: &str) -> Option<String> {
|
||||
// Look for filename= or filename*= parameter
|
||||
let lower = header_value.to_lowercase();
|
||||
|
||||
// Try to find filename="..." or filename=...
|
||||
if let Some(idx) = lower.find("filename=") {
|
||||
let start = idx + "filename=".len();
|
||||
let rest = &header_value[start..];
|
||||
|
||||
// Check if it's quoted
|
||||
if let Some(stripped) = rest.strip_prefix('"') {
|
||||
// Find closing quote
|
||||
if let Some(end_quote) = stripped.find('"') {
|
||||
return Some(stripped[..end_quote].to_string());
|
||||
}
|
||||
} else {
|
||||
// Unquoted - take until semicolon or end
|
||||
let end = rest.find(';').unwrap_or(rest.len());
|
||||
return Some(rest[..end].trim().to_string());
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CompressionConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: rustfs_config::DEFAULT_COMPRESS_ENABLE,
|
||||
extensions: rustfs_config::DEFAULT_COMPRESS_EXTENSIONS
|
||||
.split(',')
|
||||
.filter_map(|s| {
|
||||
let s = s.trim().to_lowercase();
|
||||
if s.is_empty() {
|
||||
None
|
||||
} else if s.starts_with('.') {
|
||||
Some(s)
|
||||
} else {
|
||||
Some(format!(".{s}"))
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
mime_patterns: rustfs_config::DEFAULT_COMPRESS_MIME_TYPES
|
||||
.split(',')
|
||||
.map(|s| s.trim().to_lowercase())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect(),
|
||||
min_size: rustfs_config::DEFAULT_COMPRESS_MIN_SIZE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Predicate to determine if a response should be compressed.
|
||||
///
|
||||
/// This predicate implements a whitelist-based compression approach:
|
||||
/// - Only compresses responses that match configured file extensions OR MIME types
|
||||
/// - Respects minimum file size threshold
|
||||
/// - Always skips error responses (4xx, 5xx) to avoid Content-Length issues
|
||||
/// - Skips already encoded responses (Content-Encoding header present)
|
||||
///
|
||||
/// # Design Philosophy
|
||||
/// Unlike the previous blacklist approach, this whitelist approach:
|
||||
/// 1. Only compresses explicitly configured content types
|
||||
/// 2. Preserves Content-Length for all other responses (better browser UX)
|
||||
/// 3. Aligns with MinIO's opt-in compression behavior
|
||||
/// 4. Avoids double compression by checking Content-Encoding header
|
||||
///
|
||||
/// # Extension Matching
|
||||
/// File extension matching works by extracting the filename from the
|
||||
/// `Content-Disposition` response header (e.g., `attachment; filename="file.txt"`).
|
||||
///
|
||||
/// # Performance
|
||||
/// This predicate is evaluated per-response and has O(n) complexity where n is
|
||||
/// the number of configured extensions/MIME patterns.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CompressionPredicate {
|
||||
config: CompressionConfig,
|
||||
}
|
||||
|
||||
impl CompressionPredicate {
|
||||
/// Create a new compression predicate with the given configuration
|
||||
pub fn new(config: CompressionConfig) -> Self {
|
||||
Self { config }
|
||||
}
|
||||
}
|
||||
|
||||
impl Predicate for CompressionPredicate {
|
||||
fn should_compress<B>(&self, response: &Response<B>) -> bool
|
||||
where
|
||||
B: http_body::Body,
|
||||
{
|
||||
// If compression is disabled, never compress
|
||||
if !self.config.enabled {
|
||||
return false;
|
||||
}
|
||||
|
||||
let status = response.status();
|
||||
|
||||
// Never compress error responses (4xx and 5xx status codes)
|
||||
// This prevents Content-Length mismatch issues with error responses
|
||||
if status.is_client_error() || status.is_server_error() {
|
||||
debug!("Skipping compression for error response: status={}", status.as_u16());
|
||||
return false;
|
||||
}
|
||||
|
||||
// Skip if content is already encoded (e.g., gzip, br, deflate, zstd)
|
||||
// Re-compressing already compressed content provides no benefit and may cause issues
|
||||
if let Some(content_encoding) = response.headers().get(http::header::CONTENT_ENCODING) {
|
||||
if let Ok(encoding) = content_encoding.to_str() {
|
||||
let encoding_lower = encoding.to_lowercase();
|
||||
// Check for common compression encodings
|
||||
// "identity" means no encoding, so we can still compress
|
||||
if encoding_lower != "identity" && !encoding_lower.is_empty() {
|
||||
debug!("Skipping compression for already encoded response: Content-Encoding={}", encoding);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check Content-Length header for minimum size threshold
|
||||
if let Some(content_length) = response.headers().get(http::header::CONTENT_LENGTH) {
|
||||
if let Ok(length_str) = content_length.to_str() {
|
||||
if let Ok(length) = length_str.parse::<u64>() {
|
||||
if length < self.config.min_size {
|
||||
debug!(
|
||||
"Skipping compression for small response: size={} bytes, min_size={}",
|
||||
length, self.config.min_size
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the response matches configured extension via Content-Disposition
|
||||
if let Some(content_disposition) = response.headers().get(http::header::CONTENT_DISPOSITION) {
|
||||
if let Ok(cd) = content_disposition.to_str() {
|
||||
if let Some(filename) = CompressionConfig::extract_filename_from_content_disposition(cd) {
|
||||
if self.config.matches_extension(&filename) {
|
||||
debug!("Compressing response: filename '{}' matches configured extension", filename);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the response matches configured MIME type
|
||||
if let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE) {
|
||||
if let Ok(ct) = content_type.to_str() {
|
||||
if self.config.matches_mime_type(ct) {
|
||||
debug!("Compressing response: Content-Type '{}' matches configured MIME pattern", ct);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Default: don't compress (whitelist approach)
|
||||
debug!("Skipping compression: response does not match any configured extension or MIME type");
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_compression_config_default() {
|
||||
let config = CompressionConfig::default();
|
||||
assert!(!config.enabled);
|
||||
assert!(config.extensions.is_empty());
|
||||
assert!(!config.mime_patterns.is_empty());
|
||||
assert_eq!(config.min_size, 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_config_mime_matching() {
|
||||
let config = CompressionConfig {
|
||||
enabled: true,
|
||||
extensions: vec![],
|
||||
mime_patterns: vec!["text/*".to_string(), "application/json".to_string()],
|
||||
min_size: 1000,
|
||||
};
|
||||
|
||||
// Test wildcard matching
|
||||
assert!(config.matches_mime_type("text/plain"));
|
||||
assert!(config.matches_mime_type("text/html"));
|
||||
assert!(config.matches_mime_type("text/css"));
|
||||
assert!(config.matches_mime_type("TEXT/PLAIN")); // case insensitive
|
||||
|
||||
// Test exact matching
|
||||
assert!(config.matches_mime_type("application/json"));
|
||||
assert!(config.matches_mime_type("application/json; charset=utf-8"));
|
||||
|
||||
// Test non-matching types
|
||||
assert!(!config.matches_mime_type("image/png"));
|
||||
assert!(!config.matches_mime_type("application/octet-stream"));
|
||||
assert!(!config.matches_mime_type("video/mp4"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_config_extension_matching() {
|
||||
let config = CompressionConfig {
|
||||
enabled: true,
|
||||
extensions: vec![".txt".to_string(), ".log".to_string(), ".csv".to_string()],
|
||||
mime_patterns: vec![],
|
||||
min_size: 1000,
|
||||
};
|
||||
|
||||
// Test matching extensions
|
||||
assert!(config.matches_extension("file.txt"));
|
||||
assert!(config.matches_extension("path/to/file.log"));
|
||||
assert!(config.matches_extension("data.csv"));
|
||||
assert!(config.matches_extension("FILE.TXT")); // case insensitive
|
||||
|
||||
// Test non-matching extensions
|
||||
assert!(!config.matches_extension("image.png"));
|
||||
assert!(!config.matches_extension("archive.zip"));
|
||||
assert!(!config.matches_extension("document.pdf"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_filename_from_content_disposition() {
|
||||
// Quoted filename
|
||||
assert_eq!(
|
||||
CompressionConfig::extract_filename_from_content_disposition(r#"attachment; filename="example.txt""#),
|
||||
Some("example.txt".to_string())
|
||||
);
|
||||
|
||||
// Unquoted filename
|
||||
assert_eq!(
|
||||
CompressionConfig::extract_filename_from_content_disposition("attachment; filename=example.log"),
|
||||
Some("example.log".to_string())
|
||||
);
|
||||
|
||||
// Filename with path
|
||||
assert_eq!(
|
||||
CompressionConfig::extract_filename_from_content_disposition(r#"attachment; filename="path/to/file.csv""#),
|
||||
Some("path/to/file.csv".to_string())
|
||||
);
|
||||
|
||||
// Mixed case
|
||||
assert_eq!(
|
||||
CompressionConfig::extract_filename_from_content_disposition(r#"Attachment; FILENAME="test.json""#),
|
||||
Some("test.json".to_string())
|
||||
);
|
||||
|
||||
// No filename
|
||||
assert_eq!(CompressionConfig::extract_filename_from_content_disposition("inline"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_config_from_empty_strings() {
|
||||
// Simulate config with empty extension and mime strings
|
||||
let config = CompressionConfig {
|
||||
enabled: true,
|
||||
extensions: ""
|
||||
.split(',')
|
||||
.map(|s| s.trim().to_lowercase())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect(),
|
||||
mime_patterns: ""
|
||||
.split(',')
|
||||
.map(|s| s.trim().to_lowercase())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect(),
|
||||
min_size: 1000,
|
||||
};
|
||||
|
||||
assert!(config.extensions.is_empty());
|
||||
assert!(config.mime_patterns.is_empty());
|
||||
assert!(!config.matches_extension("file.txt"));
|
||||
assert!(!config.matches_mime_type("text/plain"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_config_extension_normalization() {
|
||||
// Extensions should be normalized with leading dot
|
||||
let extensions: Vec<String> = "txt,.log,csv"
|
||||
.split(',')
|
||||
.map(|s| {
|
||||
let s = s.trim().to_lowercase();
|
||||
if s.starts_with('.') { s } else { format!(".{s}") }
|
||||
})
|
||||
.filter(|s| s.len() > 1)
|
||||
.collect();
|
||||
|
||||
assert_eq!(extensions, vec![".txt", ".log", ".csv"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_predicate_creation() {
|
||||
// Test that CompressionPredicate can be created with various configs
|
||||
let config_disabled = CompressionConfig {
|
||||
enabled: false,
|
||||
extensions: vec![".txt".to_string()],
|
||||
mime_patterns: vec!["text/*".to_string()],
|
||||
min_size: 0,
|
||||
};
|
||||
let predicate = CompressionPredicate::new(config_disabled.clone());
|
||||
assert!(!predicate.config.enabled);
|
||||
|
||||
let config_enabled = CompressionConfig {
|
||||
enabled: true,
|
||||
extensions: vec![".txt".to_string(), ".log".to_string()],
|
||||
mime_patterns: vec!["text/*".to_string(), "application/json".to_string()],
|
||||
min_size: 1000,
|
||||
};
|
||||
let predicate = CompressionPredicate::new(config_enabled.clone());
|
||||
assert!(predicate.config.enabled);
|
||||
assert_eq!(predicate.config.extensions.len(), 2);
|
||||
assert_eq!(predicate.config.mime_patterns.len(), 2);
|
||||
assert_eq!(predicate.config.min_size, 1000);
|
||||
}
|
||||
}
|
||||
+42
-69
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
// Ensure the correct path for parse_license is imported
|
||||
use super::compress::{CompressionConfig, CompressionPredicate};
|
||||
use crate::admin;
|
||||
use crate::auth::IAMAuth;
|
||||
use crate::config;
|
||||
@@ -43,7 +44,7 @@ use tokio_rustls::TlsAcceptor;
|
||||
use tonic::{Request, Status, metadata::MetadataValue};
|
||||
use tower::ServiceBuilder;
|
||||
use tower_http::catch_panic::CatchPanicLayer;
|
||||
use tower_http::compression::{CompressionLayer, predicate::Predicate};
|
||||
use tower_http::compression::CompressionLayer;
|
||||
use tower_http::cors::{AllowOrigin, Any, CorsLayer};
|
||||
use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
|
||||
use tower_http::trace::TraceLayer;
|
||||
@@ -108,60 +109,6 @@ fn get_cors_allowed_origins() -> String {
|
||||
.unwrap_or(rustfs_config::DEFAULT_CONSOLE_CORS_ALLOWED_ORIGINS.to_string())
|
||||
}
|
||||
|
||||
/// Predicate to determine if a response should be compressed.
|
||||
///
|
||||
/// This predicate implements intelligent compression selection to avoid issues
|
||||
/// with error responses and small payloads. It excludes:
|
||||
/// - Client error responses (4xx status codes) - typically small XML/JSON error messages
|
||||
/// - Server error responses (5xx status codes) - ensures error details are preserved
|
||||
/// - Very small responses (< 256 bytes) - compression overhead outweighs benefits
|
||||
///
|
||||
/// # Rationale
|
||||
/// The CompressionLayer can cause Content-Length header mismatches with error responses,
|
||||
/// particularly when the s3s library generates XML error responses (~119 bytes for NoSuchKey).
|
||||
/// By excluding these responses from compression, we ensure:
|
||||
/// 1. Error responses are sent with accurate Content-Length headers
|
||||
/// 2. Clients receive complete error bodies without truncation
|
||||
/// 3. Small responses avoid compression overhead
|
||||
///
|
||||
/// # Performance
|
||||
/// This predicate is evaluated per-response and has O(1) complexity.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ShouldCompress;
|
||||
|
||||
impl Predicate for ShouldCompress {
|
||||
fn should_compress<B>(&self, response: &Response<B>) -> bool
|
||||
where
|
||||
B: http_body::Body,
|
||||
{
|
||||
let status = response.status();
|
||||
|
||||
// Never compress error responses (4xx and 5xx status codes)
|
||||
// This prevents Content-Length mismatch issues with error responses
|
||||
if status.is_client_error() || status.is_server_error() {
|
||||
debug!("Skipping compression for error response: status={}", status.as_u16());
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check Content-Length header to avoid compressing very small responses
|
||||
// Responses smaller than 256 bytes typically don't benefit from compression
|
||||
// and may actually increase in size due to compression overhead
|
||||
if let Some(content_length) = response.headers().get(http::header::CONTENT_LENGTH) {
|
||||
if let Ok(length_str) = content_length.to_str() {
|
||||
if let Ok(length) = length_str.parse::<u64>() {
|
||||
if length < 256 {
|
||||
debug!("Skipping compression for small response: size={} bytes", length);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compress successful responses with sufficient size
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn start_http_server(
|
||||
opt: &config::Opt,
|
||||
worker_state_manager: ServiceStateManager,
|
||||
@@ -290,6 +237,17 @@ pub async fn start_http_server(
|
||||
Some(cors_allowed_origins)
|
||||
};
|
||||
|
||||
// Create compression configuration from environment variables
|
||||
let compression_config = CompressionConfig::from_env();
|
||||
if compression_config.enabled {
|
||||
info!(
|
||||
"HTTP response compression enabled: extensions={:?}, mime_patterns={:?}, min_size={} bytes",
|
||||
compression_config.extensions, compression_config.mime_patterns, compression_config.min_size
|
||||
);
|
||||
} else {
|
||||
debug!("HTTP response compression is disabled");
|
||||
}
|
||||
|
||||
let is_console = opt.console_enable;
|
||||
tokio::spawn(async move {
|
||||
// Create CORS layer inside the server loop closure
|
||||
@@ -395,15 +353,15 @@ pub async fn start_http_server(
|
||||
warn!(?err, "Failed to set set_send_buffer_size");
|
||||
}
|
||||
|
||||
process_connection(
|
||||
socket,
|
||||
tls_acceptor.clone(),
|
||||
http_server.clone(),
|
||||
s3_service.clone(),
|
||||
graceful.clone(),
|
||||
cors_layer.clone(),
|
||||
let connection_ctx = ConnectionContext {
|
||||
http_server: http_server.clone(),
|
||||
s3_service: s3_service.clone(),
|
||||
cors_layer: cors_layer.clone(),
|
||||
compression_config: compression_config.clone(),
|
||||
is_console,
|
||||
);
|
||||
};
|
||||
|
||||
process_connection(socket, tls_acceptor.clone(), connection_ctx, graceful.clone());
|
||||
}
|
||||
|
||||
worker_state_manager.update(ServiceState::Stopping);
|
||||
@@ -496,6 +454,15 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ConnectionContext {
|
||||
http_server: Arc<ConnBuilder<TokioExecutor>>,
|
||||
s3_service: S3Service,
|
||||
cors_layer: CorsLayer,
|
||||
compression_config: CompressionConfig,
|
||||
is_console: bool,
|
||||
}
|
||||
|
||||
/// Process a single incoming TCP connection.
|
||||
///
|
||||
/// This function is executed in a new Tokio task and it will:
|
||||
@@ -507,13 +474,18 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
|
||||
fn process_connection(
|
||||
socket: TcpStream,
|
||||
tls_acceptor: Option<Arc<TlsAcceptor>>,
|
||||
http_server: Arc<ConnBuilder<TokioExecutor>>,
|
||||
s3_service: S3Service,
|
||||
context: ConnectionContext,
|
||||
graceful: Arc<GracefulShutdown>,
|
||||
cors_layer: CorsLayer,
|
||||
is_console: bool,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
let ConnectionContext {
|
||||
http_server,
|
||||
s3_service,
|
||||
cors_layer,
|
||||
compression_config,
|
||||
is_console,
|
||||
} = context;
|
||||
|
||||
// Build services inside each connected task to avoid passing complex service types across tasks,
|
||||
// It also ensures that each connection has an independent service instance.
|
||||
let rpc_service = NodeServiceServer::with_interceptor(make_server(), check_auth);
|
||||
@@ -577,8 +549,9 @@ fn process_connection(
|
||||
)
|
||||
.layer(PropagateRequestIdLayer::x_request_id())
|
||||
.layer(cors_layer)
|
||||
// Compress responses, but exclude error responses to avoid Content-Length mismatch issues
|
||||
.layer(CompressionLayer::new().compress_when(ShouldCompress))
|
||||
// Compress responses based on whitelist configuration
|
||||
// Only compresses when enabled and matches configured extensions/MIME types
|
||||
.layer(CompressionLayer::new().compress_when(CompressionPredicate::new(compression_config)))
|
||||
.option_layer(if is_console { Some(RedirectLayer) } else { None })
|
||||
.service(service);
|
||||
|
||||
|
||||
@@ -13,13 +13,13 @@
|
||||
// limitations under the License.
|
||||
|
||||
mod audit;
|
||||
mod compress;
|
||||
mod event;
|
||||
mod http;
|
||||
mod hybrid;
|
||||
mod layer;
|
||||
mod service_state;
|
||||
|
||||
mod event;
|
||||
mod runtime;
|
||||
mod service_state;
|
||||
|
||||
pub(crate) use audit::{start_audit_system, stop_audit_system};
|
||||
pub(crate) use event::{init_event_notifier, shutdown_event_notifier};
|
||||
|
||||
Reference in New Issue
Block a user