diff --git a/crates/ecstore/src/compress.rs b/crates/ecstore/src/compress.rs index 99c91b7fd..0aa5623b7 100644 --- a/crates/ecstore/src/compress.rs +++ b/crates/ecstore/src/compress.rs @@ -12,55 +12,111 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_utils::string::{has_pattern, has_string_suffix_in_slice}; +use rustfs_utils::string::{has_pattern, has_string_suffix_in_slice, match_simple}; use std::env; use std::sync::OnceLock; use tracing::debug; -pub const MIN_COMPRESSIBLE_SIZE: usize = 4096; +pub const MIN_DISK_COMPRESSIBLE_SIZE: usize = 4096; -// Environment variable name to control whether compression is enabled -pub const ENV_COMPRESSION_ENABLED: &str = "RUSTFS_COMPRESSION_ENABLED"; +// Environment variable name to control whether object disk compression is enabled. +pub const ENV_DISK_COMPRESSION_ENABLED: &str = "RUSTFS_COMPRESSION_ENABLED"; + +// Environment variable for file extensions to include in object disk compression. +pub const ENV_DISK_COMPRESSION_EXTENSIONS: &str = "RUSTFS_COMPRESSION_EXTENSIONS"; + +// Environment variable for MIME types to include in object disk compression. +pub const ENV_DISK_COMPRESSION_MIME_TYPES: &str = "RUSTFS_COMPRESSION_MIME_TYPES"; // Environment variable for additional extensions to exclude from compression (comma-separated, e.g. ".foo,.bar") pub const ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS: &str = "RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS"; +pub const DEFAULT_DISK_COMPRESS_EXTENSIONS: &str = ".txt,.log,.csv,.json,.tar,.xml,.bin"; +pub const DEFAULT_DISK_COMPRESS_MIME_TYPES: &str = "text/*,application/json,application/xml,binary/octet-stream"; + +#[derive(Debug)] +struct DiskCompressionConfig { + enabled: bool, + extensions: Vec, + mime_types: Vec, + added_exclude_extensions: Vec, +} + /// Parses RUSTFS_COMPRESSION_ENABLED. Called once at first use via OnceLock. -fn parse_compression_enabled() -> bool { - env::var(ENV_COMPRESSION_ENABLED) - .map(|s| s.to_lowercase() == "true") +fn parse_disk_compression_enabled() -> bool { + env::var(ENV_DISK_COMPRESSION_ENABLED) + .map(|s| matches!(s.to_ascii_lowercase().as_str(), "true" | "on" | "1")) .unwrap_or(false) } +fn parse_csv(value: &str) -> Vec { + value + .split(',') + .map(|v| v.trim().to_ascii_lowercase()) + .filter(|v| !v.is_empty()) + .collect() +} + +fn normalize_extensions(extensions: Vec) -> Vec { + extensions + .into_iter() + .map(|v| if v.starts_with('.') { v } else { format!(".{v}") }) + .collect() +} + +fn has_config_pattern(patterns: &[String], match_str: &str) -> bool { + patterns.iter().any(|pattern| match_simple(pattern, match_str)) +} + +fn has_object_content_encoding(headers: &http::HeaderMap) -> bool { + for value in headers.get_all(http::header::CONTENT_ENCODING) { + let Ok(content_encoding) = value.to_str() else { + return true; + }; + + for encoding in content_encoding.split(',').map(str::trim) { + if !encoding.is_empty() && !encoding.eq_ignore_ascii_case("identity") && !encoding.eq_ignore_ascii_case("aws-chunked") + { + return true; + } + } + } + false +} + +fn parse_disk_compression_extensions() -> Vec { + let extensions = env::var(ENV_DISK_COMPRESSION_EXTENSIONS).unwrap_or_else(|_| DEFAULT_DISK_COMPRESS_EXTENSIONS.to_string()); + normalize_extensions(parse_csv(&extensions)) +} + +fn parse_disk_compression_mime_types() -> Vec { + let mime_types = env::var(ENV_DISK_COMPRESSION_MIME_TYPES).unwrap_or_else(|_| DEFAULT_DISK_COMPRESS_MIME_TYPES.to_string()); + parse_csv(&mime_types) +} + /// Parses RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS (comma-separated). Called once at first use via OnceLock. pub(crate) fn parse_added_exclude_extensions() -> Vec { env::var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS) .ok() - .map(|s| { - s.split(',') - .map(|v| { - let v = v.trim().to_lowercase(); - if v.is_empty() { - String::new() - } else if v.starts_with('.') { - v - } else { - format!(".{v}") - } - }) - .filter(|v| !v.is_empty()) - .collect() - }) + .map(|s| normalize_extensions(parse_csv(&s))) .unwrap_or_default() } -// Parsed once at first use, then reused for all is_compressible() checks. -static COMPRESSION_ENABLED: OnceLock = OnceLock::new(); -static ADDED_EXCLUDE_COMPRESS_EXTENSIONS: OnceLock> = OnceLock::new(); +fn parse_disk_compression_config() -> DiskCompressionConfig { + DiskCompressionConfig { + enabled: parse_disk_compression_enabled(), + extensions: parse_disk_compression_extensions(), + mime_types: parse_disk_compression_mime_types(), + added_exclude_extensions: parse_added_exclude_extensions(), + } +} + +// Parsed once at first use, then reused for all disk compression checks. +static DISK_COMPRESSION_CONFIG: OnceLock = OnceLock::new(); // Some standard object extensions which we strictly dis-allow for compression. #[rustfmt::skip] -pub const STANDARD_EXCLUDE_COMPRESS_EXTENSIONS: &[&str] = &[ +pub const DISK_COMPRESSION_EXCLUDED_EXTENSIONS: &[&str] = &[ // Compressed archives ".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".zst", ".lz4", ".br", ".lzo", ".sz", ".tgz", // Images @@ -78,7 +134,7 @@ pub const STANDARD_EXCLUDE_COMPRESS_EXTENSIONS: &[&str] = &[ ]; // Some standard content-types which we strictly dis-allow for compression. -pub const STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES: &[&str] = &[ +pub const DISK_COMPRESSION_EXCLUDED_CONTENT_TYPES: &[&str] = &[ "video/*", "audio/*", "image/*", @@ -107,35 +163,53 @@ pub const STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES: &[&str] = &[ "font/*", ]; -pub fn is_compressible(headers: &http::HeaderMap, object_name: &str) -> bool { - // Check if compression is enabled (read once at first use, then fixed for process lifetime) - if !*COMPRESSION_ENABLED.get_or_init(parse_compression_enabled) { - debug!("Compression is disabled by environment variable"); +pub fn is_disk_compressible(headers: &http::HeaderMap, object_name: &str) -> bool { + is_disk_compressible_with_config(headers, object_name, DISK_COMPRESSION_CONFIG.get_or_init(parse_disk_compression_config)) +} + +fn is_disk_compressible_with_config(headers: &http::HeaderMap, object_name: &str, config: &DiskCompressionConfig) -> bool { + // Check if disk compression is enabled (read once at first use, then fixed for process lifetime) + if !config.enabled { + debug!("Disk compression is disabled by environment variable"); return false; } let content_type = headers.get("content-type").and_then(|s| s.to_str().ok()).unwrap_or(""); - // TODO: crypto request return false - - if has_string_suffix_in_slice(object_name, STANDARD_EXCLUDE_COMPRESS_EXTENSIONS) { - debug!("object_name: {} is not compressible", object_name); + if has_object_content_encoding(headers) { + debug!("object_name: {} is already content-encoded; skipping disk compression", object_name); return false; } - let added = ADDED_EXCLUDE_COMPRESS_EXTENSIONS.get_or_init(parse_added_exclude_extensions); - if !added.is_empty() && has_string_suffix_in_slice(object_name, added) { - debug!("object_name: {} is not compressible (added exclusion)", object_name); + if has_string_suffix_in_slice(object_name, DISK_COMPRESSION_EXCLUDED_EXTENSIONS) { + debug!("object_name: {} is not disk-compressible", object_name); return false; } - if !content_type.is_empty() && has_pattern(STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES, content_type) { - debug!("content_type: {} is not compressible", content_type); + if !config.added_exclude_extensions.is_empty() && has_string_suffix_in_slice(object_name, &config.added_exclude_extensions) { + debug!("object_name: {} is not disk-compressible (added exclusion)", object_name); return false; } - true - // TODO: check from config + if !content_type.is_empty() && has_pattern(DISK_COMPRESSION_EXCLUDED_CONTENT_TYPES, content_type) { + debug!("content_type: {} is not disk-compressible", content_type); + return false; + } + + if config.extensions.is_empty() && config.mime_types.is_empty() { + return true; + } + + if !config.extensions.is_empty() && has_string_suffix_in_slice(object_name, &config.extensions) { + return true; + } + + if !content_type.is_empty() && !config.mime_types.is_empty() && has_config_pattern(&config.mime_types, content_type) { + return true; + } + + debug!("object_name: {} does not match disk compression include filters", object_name); + false } #[cfg(test)] @@ -144,117 +218,169 @@ mod tests { use temp_env; #[test] - fn test_parse_compression_enabled() { - temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || { - assert!(parse_compression_enabled()); + fn test_parse_disk_compression_enabled() { + temp_env::with_var(ENV_DISK_COMPRESSION_ENABLED, Some("true"), || { + assert!(parse_disk_compression_enabled()); }); - temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("false"), || { - assert!(!parse_compression_enabled()); + temp_env::with_var(ENV_DISK_COMPRESSION_ENABLED, Some("on"), || { + assert!(parse_disk_compression_enabled()); }); - temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("FALSE"), || { - assert!(!parse_compression_enabled()); + temp_env::with_var(ENV_DISK_COMPRESSION_ENABLED, Some("false"), || { + assert!(!parse_disk_compression_enabled()); }); - temp_env::with_var_unset(ENV_COMPRESSION_ENABLED, || { - assert!(!parse_compression_enabled()); + temp_env::with_var(ENV_DISK_COMPRESSION_ENABLED, Some("FALSE"), || { + assert!(!parse_disk_compression_enabled()); + }); + temp_env::with_var_unset(ENV_DISK_COMPRESSION_ENABLED, || { + assert!(!parse_disk_compression_enabled()); }); } #[test] - fn test_is_compressible() { + fn test_parse_disk_compression_includes() { + temp_env::with_var(ENV_DISK_COMPRESSION_EXTENSIONS, Some(".txt,log, .json"), || { + assert_eq!(parse_disk_compression_extensions(), [".txt", ".log", ".json"]); + }); + temp_env::with_var(ENV_DISK_COMPRESSION_MIME_TYPES, Some("text/*, application/json"), || { + assert_eq!(parse_disk_compression_mime_types(), ["text/*", "application/json"]); + }); + temp_env::with_var_unset(ENV_DISK_COMPRESSION_EXTENSIONS, || { + assert_eq!( + parse_disk_compression_extensions(), + [".txt", ".log", ".csv", ".json", ".tar", ".xml", ".bin"] + ); + }); + temp_env::with_var_unset(ENV_DISK_COMPRESSION_MIME_TYPES, || { + assert_eq!( + parse_disk_compression_mime_types(), + ["text/*", "application/json", "application/xml", "binary/octet-stream"] + ); + }); + } + + fn test_config() -> DiskCompressionConfig { + DiskCompressionConfig { + enabled: true, + extensions: normalize_extensions(parse_csv(DEFAULT_DISK_COMPRESS_EXTENSIONS)), + mime_types: parse_csv(DEFAULT_DISK_COMPRESS_MIME_TYPES), + added_exclude_extensions: Vec::new(), + } + } + + #[test] + fn test_is_disk_compressible() { use http::HeaderMap; - let headers = HeaderMap::new(); + let config = test_config(); + let mut headers = HeaderMap::new(); + // Test non-compressible extensions - compressed archives + headers.insert("content-type", "text/plain".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.gz", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.zip", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.7z", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.zst", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.br", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.tgz", &config)); - // COMPRESSION_ENABLED is cached in OnceLock at first use; test with "true" so extension/content-type logic is exercised. - // For env false/unset behavior see test_parse_compression_enabled. - temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || { - assert!(is_compressible(&headers, "file.txt")); - }); + // Test non-compressible extensions - images + assert!(!is_disk_compressible_with_config(&headers, "file.jpg", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.jpeg", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.png", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.gif", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.webp", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.avif", &config)); - temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || { - let mut headers = HeaderMap::new(); - // Test non-compressible extensions - compressed archives - headers.insert("content-type", "text/plain".parse().unwrap()); - assert!(!is_compressible(&headers, "file.gz")); - assert!(!is_compressible(&headers, "file.zip")); - assert!(!is_compressible(&headers, "file.7z")); - assert!(!is_compressible(&headers, "file.zst")); - assert!(!is_compressible(&headers, "file.br")); - assert!(!is_compressible(&headers, "file.tgz")); + // Test non-compressible extensions - video + assert!(!is_disk_compressible_with_config(&headers, "file.mp4", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.mkv", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.webm", &config)); - // Test non-compressible extensions - images - assert!(!is_compressible(&headers, "file.jpg")); - assert!(!is_compressible(&headers, "file.jpeg")); - assert!(!is_compressible(&headers, "file.png")); - assert!(!is_compressible(&headers, "file.gif")); - assert!(!is_compressible(&headers, "file.webp")); - assert!(!is_compressible(&headers, "file.avif")); + // Test non-compressible extensions - audio + assert!(!is_disk_compressible_with_config(&headers, "file.mp3", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.aac", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.ogg", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.flac", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.opus", &config)); - // Test non-compressible extensions - video - assert!(!is_compressible(&headers, "file.mp4")); - assert!(!is_compressible(&headers, "file.mkv")); - assert!(!is_compressible(&headers, "file.webm")); + // Test non-compressible extensions - documents + assert!(!is_disk_compressible_with_config(&headers, "file.pdf", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.docx", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.xlsx", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.pptx", &config)); - // Test non-compressible extensions - audio - assert!(!is_compressible(&headers, "file.mp3")); - assert!(!is_compressible(&headers, "file.aac")); - assert!(!is_compressible(&headers, "file.ogg")); - assert!(!is_compressible(&headers, "file.flac")); - assert!(!is_compressible(&headers, "file.opus")); + // Test non-compressible extensions - packages + assert!(!is_disk_compressible_with_config(&headers, "file.deb", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.rpm", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.jar", &config)); - // Test non-compressible extensions - documents - assert!(!is_compressible(&headers, "file.pdf")); - assert!(!is_compressible(&headers, "file.docx")); - assert!(!is_compressible(&headers, "file.xlsx")); - assert!(!is_compressible(&headers, "file.pptx")); + // Test non-compressible extensions - web fonts + assert!(!is_disk_compressible_with_config(&headers, "file.woff2", &config)); - // Test non-compressible extensions - packages - assert!(!is_compressible(&headers, "file.deb")); - assert!(!is_compressible(&headers, "file.rpm")); - assert!(!is_compressible(&headers, "file.jar")); + // Test non-compressible content types + headers.insert("content-type", "video/mp4".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); - // Test non-compressible extensions - web fonts - assert!(!is_compressible(&headers, "file.woff2")); + headers.insert("content-type", "audio/mpeg".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); - // Test non-compressible content types - headers.insert("content-type", "video/mp4".parse().unwrap()); - assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "image/png".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); - headers.insert("content-type", "audio/mpeg".parse().unwrap()); - assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "image/webp".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); - headers.insert("content-type", "image/png".parse().unwrap()); - assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "application/zip".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); - headers.insert("content-type", "image/webp".parse().unwrap()); - assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "application/x-gzip".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); - headers.insert("content-type", "application/zip".parse().unwrap()); - assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "application/x-7z-compressed".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); - headers.insert("content-type", "application/x-gzip".parse().unwrap()); - assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "application/pdf".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); - headers.insert("content-type", "application/x-7z-compressed".parse().unwrap()); - assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "font/woff2".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); - headers.insert("content-type", "application/pdf".parse().unwrap()); - assert!(!is_compressible(&headers, "file.txt")); + // Test compressible cases + headers.insert("content-type", "text/plain".parse().expect("valid content type")); + assert!(is_disk_compressible_with_config(&headers, "file.txt", &config)); + assert!(is_disk_compressible_with_config(&headers, "file.log", &config)); - headers.insert("content-type", "font/woff2".parse().unwrap()); - assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "text/html".parse().expect("valid content type")); + assert!(is_disk_compressible_with_config(&headers, "file.html", &config)); - // Test compressible cases - headers.insert("content-type", "text/plain".parse().unwrap()); - assert!(is_compressible(&headers, "file.txt")); - assert!(is_compressible(&headers, "file.log")); + headers.insert("content-type", "application/json".parse().expect("valid content type")); + assert!(is_disk_compressible_with_config(&headers, "file.json", &config)); - headers.insert("content-type", "text/html".parse().unwrap()); - assert!(is_compressible(&headers, "file.html")); + headers.insert("content-type", "application/octet-stream".parse().expect("valid content type")); + assert!(!is_disk_compressible_with_config(&headers, "file.data", &config)); + } - headers.insert("content-type", "application/json".parse().unwrap()); - assert!(is_compressible(&headers, "file.json")); - }); + #[test] + fn test_content_encoding_skips_disk_compression() { + use http::{HeaderMap, HeaderValue}; + + let config = test_config(); + let mut headers = HeaderMap::new(); + headers.insert("content-type", "text/plain".parse().expect("valid content type")); + + headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("zstd")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); + + headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("gzip")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); + + headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("aws-chunked,gzip")); + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); + + headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("aws-chunked")); + assert!(is_disk_compressible_with_config(&headers, "file.txt", &config)); + + headers.insert(http::header::CONTENT_ENCODING, HeaderValue::from_static("identity")); + assert!(is_disk_compressible_with_config(&headers, "file.txt", &config)); } #[test] @@ -274,16 +400,32 @@ mod tests { } #[test] - fn test_added_exclude_compress_extensions_default_compressible() { + fn test_added_exclude_compress_extensions_excludes_included_extension() { use http::HeaderMap; let mut headers = HeaderMap::new(); - headers.insert("content-type", "text/plain".parse().unwrap()); - temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || { - temp_env::with_var_unset(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, || { - assert!(is_compressible(&headers, "file.foo")); - assert!(is_compressible(&headers, "file.bar")); - }); - }); + headers.insert("content-type", "text/plain".parse().expect("valid content type")); + let mut config = test_config(); + config.added_exclude_extensions = vec![".txt".to_string()]; + + assert!(!is_disk_compressible_with_config(&headers, "file.txt", &config)); + assert!(is_disk_compressible_with_config(&headers, "file.log", &config)); + } + + #[test] + fn test_empty_includes_compress_everything_except_exclusions() { + use http::HeaderMap; + + let mut headers = HeaderMap::new(); + headers.insert("content-type", "application/octet-stream".parse().expect("valid content type")); + let config = DiskCompressionConfig { + enabled: true, + extensions: Vec::new(), + mime_types: Vec::new(), + added_exclude_extensions: Vec::new(), + }; + + assert!(is_disk_compressible_with_config(&headers, "file.data", &config)); + assert!(!is_disk_compressible_with_config(&headers, "file.zip", &config)); } } diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index b27259581..5b6453dc9 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -46,7 +46,7 @@ use rustfs_ecstore::bucket::{ versioning_sys::BucketVersioningSys, }; use rustfs_ecstore::client::object_api_utils::to_s3s_etag; -use rustfs_ecstore::compress::is_compressible; +use rustfs_ecstore::compress::is_disk_compressible; use rustfs_ecstore::error::{StorageError, is_err_object_not_found, is_err_version_not_found}; use rustfs_ecstore::new_object_layer_fn; #[cfg(test)] @@ -606,7 +606,7 @@ impl DefaultMultipartUsecase { None => (None, None), }; - if is_compressible(&req.headers, &key) { + if is_disk_compressible(&req.headers, &key) { rustfs_utils::http::insert_str( &mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, @@ -752,7 +752,7 @@ impl DefaultMultipartUsecase { StreamReader::new(body_stream.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), ); - let is_compressible = rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); + let is_disk_compressed = rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); let actual_size = size; @@ -768,7 +768,7 @@ impl DefaultMultipartUsecase { let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); let mut write_plan = WritePlan::new(); - let mut reader = if is_compressible { + let mut reader = if is_disk_compressed { let algorithm = CompressionAlgorithm::default(); let mut hrd = HashReader::from_stream(body, size, actual_size, md5hex.take(), sha256hex.take(), false) .map_err(ApiError::from)?; @@ -1121,12 +1121,13 @@ impl DefaultMultipartUsecase { .map_err(map_get_object_reader_error)?; let src_stream = src_reader.stream; - let is_compressible = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); + let is_disk_compressed = + rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); let actual_size = length; let mut write_plan = WritePlan::new(); - let mut reader = if is_compressible { + let mut reader = if is_disk_compressed { let algorithm = CompressionAlgorithm::default(); let hrd = HashReader::from_stream(src_stream, length, actual_size, None, None, false).map_err(ApiError::from)?; write_plan = write_plan.with_compression(algorithm); diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 65ee5fb5d..942514954 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -69,7 +69,7 @@ use rustfs_ecstore::bucket::{ versioning_sys::BucketVersioningSys, }; use rustfs_ecstore::client::object_api_utils::to_s3s_etag; -use rustfs_ecstore::compress::{MIN_COMPRESSIBLE_SIZE, is_compressible}; +use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; use rustfs_ecstore::config::storageclass; use rustfs_ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found}; use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found}; @@ -1928,7 +1928,7 @@ impl DefaultObjectUsecase { let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); - let should_compress = is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64; + let should_compress = is_disk_compressible(&req.headers, &key) && size > MIN_DISK_COMPRESSIBLE_SIZE as i64; let mut write_plan = WritePlan::new(); let mut reader = if should_compress { let algorithm = CompressionAlgorithm::default(); @@ -2902,7 +2902,7 @@ impl DefaultObjectUsecase { let mut compress_metadata = HashMap::new(); - let should_compress = is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64; + let should_compress = is_disk_compressible(&req.headers, &key) && actual_size > MIN_DISK_COMPRESSIBLE_SIZE as i64; if should_compress { insert_str( @@ -4411,7 +4411,8 @@ impl DefaultObjectUsecase { let actual_size = size; - let should_compress = !is_dir && is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64; + let should_compress = + !is_dir && is_disk_compressible(&HeaderMap::new(), &fpath) && size > MIN_DISK_COMPRESSIBLE_SIZE as i64; let mut write_plan = WritePlan::new(); let mut hrd = if is_dir { diff --git a/rustfs/src/server/compress.rs b/rustfs/src/server/compress.rs index 64ef6dae3..7404ed85b 100644 --- a/rustfs/src/server/compress.rs +++ b/rustfs/src/server/compress.rs @@ -46,12 +46,50 @@ use rustfs_config::{ DEFAULT_COMPRESS_ENABLE, DEFAULT_COMPRESS_EXTENSIONS, DEFAULT_COMPRESS_MIME_TYPES, DEFAULT_COMPRESS_MIN_SIZE, ENV_COMPRESS_ENABLE, ENV_COMPRESS_EXTENSIONS, ENV_COMPRESS_MIME_TYPES, ENV_COMPRESS_MIN_SIZE, EnableState, }; -use rustfs_ecstore::compress::{STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES, STANDARD_EXCLUDE_COMPRESS_EXTENSIONS}; use rustfs_utils::string::{has_pattern, has_string_suffix_in_slice}; use std::str::FromStr; use tower_http::compression::predicate::Predicate; use tracing::debug; +#[rustfmt::skip] +const HTTP_COMPRESSION_EXCLUDED_EXTENSIONS: &[&str] = &[ + // Encoded archives + ".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".zst", ".lz4", ".br", ".lzo", ".sz", ".tgz", + // Media + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".heic", ".heif", ".jxl", + ".mp4", ".mkv", ".mov", ".avi", ".wmv", ".flv", ".webm", ".m4v", ".mpeg", ".mpg", + ".mp3", ".aac", ".ogg", ".flac", ".wma", ".m4a", ".opus", + // Internally compressed documents and packages + ".pdf", ".docx", ".xlsx", ".pptx", ".deb", ".rpm", ".jar", ".war", ".apk", ".woff", ".woff2", +]; + +const HTTP_COMPRESSION_EXCLUDED_CONTENT_TYPES: &[&str] = &[ + "video/*", + "audio/*", + "image/*", + "application/zip", + "application/gzip", + "application/x-gzip", + "application/x-zip-compressed", + "application/x-compress", + "application/x-spoon", + "application/x-rar-compressed", + "application/x-7z-compressed", + "application/x-bzip", + "application/x-bzip2", + "application/x-xz", + "application/x-lzip", + "application/x-lzma", + "application/x-lzop", + "application/zstd", + "application/x-zstd", + "application/x-tar", + "application/tar", + "application/pdf", + "application/wasm", + "font/*", +]; + /// Response extension key for storing the request path category. /// Set by `PathCategoryInjectionLayer` before the compression predicate evaluates. #[derive(Debug, Clone, Copy)] @@ -67,7 +105,7 @@ pub(crate) struct RequestPathCategory(pub(crate) PathCategory); /// When compression is enabled, only responses matching these criteria will be compressed. /// This approach aligns with MinIO's behavior where compression is opt-in rather than default. #[derive(Clone, Debug)] -pub struct CompressionConfig { +pub struct HttpCompressionConfig { /// Whether compression is enabled pub enabled: bool, /// File extensions to compress (normalized to lowercase with leading dot) @@ -78,7 +116,7 @@ pub struct CompressionConfig { pub min_size: u64, } -impl CompressionConfig { +impl HttpCompressionConfig { /// Create a new compression configuration from environment variables /// /// Reads the following environment variables: @@ -200,7 +238,7 @@ impl CompressionConfig { } pub(crate) fn is_excluded_filename(filename: &str) -> bool { - has_string_suffix_in_slice(&filename.to_ascii_lowercase(), STANDARD_EXCLUDE_COMPRESS_EXTENSIONS) + has_string_suffix_in_slice(&filename.to_ascii_lowercase(), HTTP_COMPRESSION_EXCLUDED_EXTENSIONS) } pub(crate) fn is_excluded_mime_type(content_type: &str) -> bool { @@ -210,11 +248,11 @@ impl CompressionConfig { .unwrap_or(content_type) .trim() .to_ascii_lowercase(); - !main_type.is_empty() && has_pattern(STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES, &main_type) + !main_type.is_empty() && has_pattern(HTTP_COMPRESSION_EXCLUDED_CONTENT_TYPES, &main_type) } } -impl Default for CompressionConfig { +impl Default for HttpCompressionConfig { fn default() -> Self { Self { enabled: rustfs_config::DEFAULT_COMPRESS_ENABLE, @@ -269,18 +307,18 @@ impl Default for CompressionConfig { /// This predicate is evaluated per-response and has O(n) complexity where n is /// the number of configured extensions/MIME patterns. #[derive(Clone, Debug)] -pub struct CompressionPredicate { - config: CompressionConfig, +pub struct HttpCompressionPredicate { + config: HttpCompressionConfig, } -impl CompressionPredicate { +impl HttpCompressionPredicate { /// Create a new compression predicate with the given configuration - pub fn new(config: CompressionConfig) -> Self { + pub fn new(config: HttpCompressionConfig) -> Self { Self { config } } } -impl Predicate for CompressionPredicate { +impl Predicate for HttpCompressionPredicate { fn should_compress(&self, response: &Response) -> bool where B: http_body::Body, @@ -321,7 +359,7 @@ impl Predicate for CompressionPredicate { if let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE) && let Ok(ct) = content_type.to_str() { - if CompressionConfig::is_excluded_mime_type(ct) { + if HttpCompressionConfig::is_excluded_mime_type(ct) { debug!("Skipping compression for excluded Content-Type '{}'", ct); return false; } @@ -335,9 +373,9 @@ impl Predicate for CompressionPredicate { // Hard-stop archive-like attachment downloads even if the whitelist matches. if let Some(content_disposition) = response.headers().get(http::header::CONTENT_DISPOSITION) && let Ok(cd) = content_disposition.to_str() - && let Some(filename) = CompressionConfig::extract_filename_from_content_disposition(cd) + && let Some(filename) = HttpCompressionConfig::extract_filename_from_content_disposition(cd) { - if CompressionConfig::is_excluded_filename(&filename) { + if HttpCompressionConfig::is_excluded_filename(&filename) { debug!("Skipping compression for excluded filename '{}'", filename); return false; } @@ -401,19 +439,19 @@ impl PathCategory { /// This avoids running MIME type / extension matching for admin, RPC, console, /// and health probe paths where compression is never beneficial. #[derive(Clone, Debug)] -pub(crate) struct PathAwareCompressionPredicate { - inner: CompressionPredicate, +pub(crate) struct PathAwareHttpCompressionPredicate { + inner: HttpCompressionPredicate, } -impl PathAwareCompressionPredicate { - pub(crate) fn new(config: CompressionConfig) -> Self { +impl PathAwareHttpCompressionPredicate { + pub(crate) fn new(config: HttpCompressionConfig) -> Self { Self { - inner: CompressionPredicate::new(config), + inner: HttpCompressionPredicate::new(config), } } } -impl Predicate for PathAwareCompressionPredicate { +impl Predicate for PathAwareHttpCompressionPredicate { fn should_compress(&self, response: &Response) -> bool where B: http_body::Body, @@ -514,7 +552,7 @@ mod tests { #[test] fn test_compression_config_default() { - let config = CompressionConfig::default(); + let config = HttpCompressionConfig::default(); assert!(!config.enabled); assert!(config.extensions.is_empty()); assert!(!config.mime_patterns.is_empty()); @@ -523,7 +561,7 @@ mod tests { #[test] fn test_compression_config_mime_matching() { - let config = CompressionConfig { + let config = HttpCompressionConfig { enabled: true, extensions: vec![], mime_patterns: vec!["text/*".to_string(), "application/json".to_string()], @@ -548,7 +586,7 @@ mod tests { #[test] fn test_compression_config_extension_matching() { - let config = CompressionConfig { + let config = HttpCompressionConfig { enabled: true, extensions: vec![".txt".to_string(), ".log".to_string(), ".csv".to_string()], mime_patterns: vec![], @@ -571,36 +609,36 @@ mod tests { fn test_extract_filename_from_content_disposition() { // Quoted filename assert_eq!( - CompressionConfig::extract_filename_from_content_disposition(r#"attachment; filename="example.txt""#), + HttpCompressionConfig::extract_filename_from_content_disposition(r#"attachment; filename="example.txt""#), Some("example.txt".to_string()) ); // Unquoted filename assert_eq!( - CompressionConfig::extract_filename_from_content_disposition("attachment; filename=example.log"), + HttpCompressionConfig::extract_filename_from_content_disposition("attachment; filename=example.log"), Some("example.log".to_string()) ); // Filename with path assert_eq!( - CompressionConfig::extract_filename_from_content_disposition(r#"attachment; filename="path/to/file.csv""#), + HttpCompressionConfig::extract_filename_from_content_disposition(r#"attachment; filename="path/to/file.csv""#), Some("path/to/file.csv".to_string()) ); // Mixed case assert_eq!( - CompressionConfig::extract_filename_from_content_disposition(r#"Attachment; FILENAME="test.json""#), + HttpCompressionConfig::extract_filename_from_content_disposition(r#"Attachment; FILENAME="test.json""#), Some("test.json".to_string()) ); // No filename - assert_eq!(CompressionConfig::extract_filename_from_content_disposition("inline"), None); + assert_eq!(HttpCompressionConfig::extract_filename_from_content_disposition("inline"), None); } #[test] fn test_compression_config_from_empty_strings() { // Simulate config with empty extension and mime strings - let config = CompressionConfig { + let config = HttpCompressionConfig { enabled: true, extensions: "" .split(',') @@ -638,23 +676,23 @@ mod tests { #[test] fn test_compression_predicate_creation() { - // Test that CompressionPredicate can be created with various configs - let config_disabled = CompressionConfig { + // Test that HttpCompressionPredicate can be created with various configs + let config_disabled = HttpCompressionConfig { enabled: false, extensions: vec![".txt".to_string()], mime_patterns: vec!["text/*".to_string()], min_size: 0, }; - let predicate = CompressionPredicate::new(config_disabled); + let predicate = HttpCompressionPredicate::new(config_disabled); assert!(!predicate.config.enabled); - let config_enabled = CompressionConfig { + let config_enabled = HttpCompressionConfig { enabled: true, extensions: vec![".txt".to_string(), ".log".to_string()], mime_patterns: vec!["text/*".to_string(), "application/json".to_string()], min_size: 1000, }; - let predicate = CompressionPredicate::new(config_enabled); + let predicate = HttpCompressionPredicate::new(config_enabled); assert!(predicate.config.enabled); assert_eq!(predicate.config.extensions.len(), 2); assert_eq!(predicate.config.mime_patterns.len(), 2); @@ -663,7 +701,7 @@ mod tests { #[test] fn test_compression_predicate_skips_archive_mime_type_even_when_whitelisted() { - let predicate = CompressionPredicate::new(CompressionConfig { + let predicate = HttpCompressionPredicate::new(HttpCompressionConfig { enabled: true, extensions: vec![], mime_patterns: vec!["application/zip".to_string()], @@ -681,7 +719,7 @@ mod tests { #[test] fn test_compression_predicate_skips_archive_filename_even_when_whitelisted() { - let predicate = CompressionPredicate::new(CompressionConfig { + let predicate = HttpCompressionPredicate::new(HttpCompressionConfig { enabled: true, extensions: vec![".zip".to_string()], mime_patterns: vec![], diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index e33e77315..772b2c990 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -19,7 +19,7 @@ use crate::auth_keystone; use crate::config; use crate::server::{ ReadinessGateLayer, RemoteAddr, ShutdownHandle, - compress::{CompressionConfig, PathAwareCompressionPredicate, PathCategoryInjectionLayer}, + compress::{HttpCompressionConfig, PathAwareHttpCompressionPredicate, PathCategoryInjectionLayer}, hybrid::hybrid, layer::{ BodylessStatusFixLayer, ConditionalCorsLayer, EmptyBodyContentLengthCompatLayer, HeadRequestBodyFixLayer, @@ -409,7 +409,7 @@ pub async fn start_http_server(config: &config::Config, readiness: Arc>, s3_service: S3Service, - compression_config: CompressionConfig, + compression_config: HttpCompressionConfig, is_console: bool, readiness: Arc, /// Pre-computed Keystone auth provider (avoids per-connection OnceLock read). @@ -939,7 +939,7 @@ fn process_connection( .layer(PropagateRequestIdLayer::x_request_id()) // Compress responses based on whitelist configuration // Only compresses when enabled and matches configured extensions/MIME types - .layer(CompressionLayer::new().compress_when(PathAwareCompressionPredicate::new(compression_config))) + .layer(CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(compression_config))) .layer(PathCategoryInjectionLayer) .layer(S3ErrorMessageCompatLayer) .layer(ObjectAttributesEtagFixLayer)