From 4413878739746ef3ecc044e55629f30fc156d44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 16 Feb 2026 03:36:22 +0800 Subject: [PATCH] fix(compress): downgrade non-compressible log level and expand excluded file types (#1780) --- crates/ecstore/src/compress.rs | 197 +++++++++++++++++++++++++++++---- crates/utils/src/string.rs | 5 +- 2 files changed, 177 insertions(+), 25 deletions(-) diff --git a/crates/ecstore/src/compress.rs b/crates/ecstore/src/compress.rs index 29e0baa69..0886c84be 100644 --- a/crates/ecstore/src/compress.rs +++ b/crates/ecstore/src/compress.rs @@ -14,38 +14,93 @@ use rustfs_utils::string::{has_pattern, has_string_suffix_in_slice}; use std::env; -use tracing::error; +use std::sync::OnceLock; +use tracing::debug; pub const MIN_COMPRESSIBLE_SIZE: usize = 4096; // Environment variable name to control whether compression is enabled pub const ENV_COMPRESSION_ENABLED: &str = "RUSTFS_COMPRESSION_ENABLED"; +// Environment variable for additional extensions to exclude from compression (comma-separated, e.g. ".foo,.bar") +pub const ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS: &str = "RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS"; + +/// Parses RUSTFS_COMPRESSION_ENABLED. Called once at first use via OnceLock. +fn parse_compression_enabled() -> bool { + env::var(ENV_COMPRESSION_ENABLED) + .map(|s| s.to_lowercase() == "true") + .unwrap_or(false) +} + +/// Parses RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS (comma-separated). Called once at first use via OnceLock. +pub(crate) fn parse_added_exclude_extensions() -> Vec { + env::var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS) + .ok() + .map(|s| { + s.split(',') + .map(|v| { + let v = v.trim().to_lowercase(); + if v.is_empty() { + String::new() + } else if v.starts_with('.') { + v + } else { + format!(".{v}") + } + }) + .filter(|v| !v.is_empty()) + .collect() + }) + .unwrap_or_default() +} + +// Parsed once at first use, then reused for all is_compressible() checks. +static COMPRESSION_ENABLED: OnceLock = OnceLock::new(); +static ADDED_EXCLUDE_COMPRESS_EXTENSIONS: OnceLock> = OnceLock::new(); + // Some standard object extensions which we strictly dis-allow for compression. +#[rustfmt::skip] pub const STANDARD_EXCLUDE_COMPRESS_EXTENSIONS: &[&str] = &[ - ".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".mp4", ".mkv", ".mov", ".jpg", ".png", ".gif", + // Compressed archives + ".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".zst", ".lz4", ".br", ".lzo", ".sz", ".tgz", + // Images + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".heic", ".heif", ".jxl", + // Video + ".mp4", ".mkv", ".mov", ".avi", ".wmv", ".flv", ".webm", ".m4v", ".mpeg", ".mpg", + // Audio + ".mp3", ".aac", ".ogg", ".flac", ".wma", ".m4a", ".opus", + // Documents (internally compressed) + ".pdf", ".docx", ".xlsx", ".pptx", + // Package formats + ".deb", ".rpm", ".jar", ".war", ".apk", + // Web fonts + ".woff", ".woff2", ]; // Some standard content-types which we strictly dis-allow for compression. pub const STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES: &[&str] = &[ "video/*", "audio/*", + "image/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", "application/x-compress", "application/x-spoon", + "application/x-rar-compressed", + "application/x-7z-compressed", + "application/x-bzip2", + "application/x-xz", + "application/zstd", + "application/pdf", + "application/wasm", + "font/*", ]; pub fn is_compressible(headers: &http::HeaderMap, object_name: &str) -> bool { - // Check if compression is enabled via environment variable, default disabled - if let Ok(compression_enabled) = env::var(ENV_COMPRESSION_ENABLED) { - if compression_enabled.to_lowercase() != "true" { - error!("Compression is disabled by environment variable"); - return false; - } - } else { - // Default disabled when environment variable is not set + // Check if compression is enabled (read once at first use, then fixed for process lifetime) + if !*COMPRESSION_ENABLED.get_or_init(parse_compression_enabled) { + debug!("Compression is disabled by environment variable"); return false; } @@ -54,12 +109,18 @@ pub fn is_compressible(headers: &http::HeaderMap, object_name: &str) -> bool { // TODO: crypto request return false if has_string_suffix_in_slice(object_name, STANDARD_EXCLUDE_COMPRESS_EXTENSIONS) { - error!("object_name: {} is not compressible", object_name); + debug!("object_name: {} is not compressible", object_name); + return false; + } + + let added = ADDED_EXCLUDE_COMPRESS_EXTENSIONS.get_or_init(parse_added_exclude_extensions); + if !added.is_empty() && has_string_suffix_in_slice(object_name, added) { + debug!("object_name: {} is not compressible (added exclusion)", object_name); return false; } if !content_type.is_empty() && has_pattern(STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES, content_type) { - error!("content_type: {} is not compressible", content_type); + debug!("content_type: {} is not compressible", content_type); return false; } true @@ -72,33 +133,78 @@ mod tests { use super::*; use temp_env; + #[test] + fn test_parse_compression_enabled() { + temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || { + assert!(parse_compression_enabled()); + }); + temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("false"), || { + assert!(!parse_compression_enabled()); + }); + temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("FALSE"), || { + assert!(!parse_compression_enabled()); + }); + temp_env::with_var_unset(ENV_COMPRESSION_ENABLED, || { + assert!(!parse_compression_enabled()); + }); + } + #[test] fn test_is_compressible() { use http::HeaderMap; let headers = HeaderMap::new(); - // Test environment variable control - temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("false"), || { - assert!(!is_compressible(&headers, "file.txt")); - }); - + // COMPRESSION_ENABLED is cached in OnceLock at first use; test with "true" so extension/content-type logic is exercised. + // For env false/unset behavior see test_parse_compression_enabled. temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || { assert!(is_compressible(&headers, "file.txt")); }); - temp_env::with_var_unset(ENV_COMPRESSION_ENABLED, || { - assert!(!is_compressible(&headers, "file.txt")); - }); - temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || { let mut headers = HeaderMap::new(); - // Test non-compressible extensions + // Test non-compressible extensions - compressed archives headers.insert("content-type", "text/plain".parse().unwrap()); assert!(!is_compressible(&headers, "file.gz")); assert!(!is_compressible(&headers, "file.zip")); - assert!(!is_compressible(&headers, "file.mp4")); + assert!(!is_compressible(&headers, "file.7z")); + assert!(!is_compressible(&headers, "file.zst")); + assert!(!is_compressible(&headers, "file.br")); + assert!(!is_compressible(&headers, "file.tgz")); + + // Test non-compressible extensions - images assert!(!is_compressible(&headers, "file.jpg")); + assert!(!is_compressible(&headers, "file.jpeg")); + assert!(!is_compressible(&headers, "file.png")); + assert!(!is_compressible(&headers, "file.gif")); + assert!(!is_compressible(&headers, "file.webp")); + assert!(!is_compressible(&headers, "file.avif")); + + // Test non-compressible extensions - video + assert!(!is_compressible(&headers, "file.mp4")); + assert!(!is_compressible(&headers, "file.mkv")); + assert!(!is_compressible(&headers, "file.webm")); + + // Test non-compressible extensions - audio + assert!(!is_compressible(&headers, "file.mp3")); + assert!(!is_compressible(&headers, "file.aac")); + assert!(!is_compressible(&headers, "file.ogg")); + assert!(!is_compressible(&headers, "file.flac")); + assert!(!is_compressible(&headers, "file.opus")); + + // Test non-compressible extensions - documents + assert!(!is_compressible(&headers, "file.pdf")); + assert!(!is_compressible(&headers, "file.docx")); + assert!(!is_compressible(&headers, "file.xlsx")); + assert!(!is_compressible(&headers, "file.pptx")); + + // Test non-compressible extensions - packages + assert!(!is_compressible(&headers, "file.deb")); + assert!(!is_compressible(&headers, "file.rpm")); + assert!(!is_compressible(&headers, "file.jar")); + + // Test non-compressible extensions - web fonts + assert!(!is_compressible(&headers, "file.woff2")); // Test non-compressible content types headers.insert("content-type", "video/mp4".parse().unwrap()); @@ -107,12 +213,27 @@ mod tests { headers.insert("content-type", "audio/mpeg".parse().unwrap()); assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "image/png".parse().unwrap()); + assert!(!is_compressible(&headers, "file.txt")); + + headers.insert("content-type", "image/webp".parse().unwrap()); + assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "application/zip".parse().unwrap()); assert!(!is_compressible(&headers, "file.txt")); headers.insert("content-type", "application/x-gzip".parse().unwrap()); assert!(!is_compressible(&headers, "file.txt")); + headers.insert("content-type", "application/x-7z-compressed".parse().unwrap()); + assert!(!is_compressible(&headers, "file.txt")); + + headers.insert("content-type", "application/pdf".parse().unwrap()); + assert!(!is_compressible(&headers, "file.txt")); + + headers.insert("content-type", "font/woff2".parse().unwrap()); + assert!(!is_compressible(&headers, "file.txt")); + // Test compressible cases headers.insert("content-type", "text/plain".parse().unwrap()); assert!(is_compressible(&headers, "file.txt")); @@ -125,4 +246,34 @@ mod tests { assert!(is_compressible(&headers, "file.json")); }); } + + #[test] + fn test_added_exclude_compress_extensions_parsing() { + temp_env::with_var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, Some(".foo,.bar"), || { + let added = parse_added_exclude_extensions(); + assert_eq!(added, [".foo", ".bar"]); + }); + temp_env::with_var(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, Some("baz, .qux"), || { + let added = parse_added_exclude_extensions(); + assert_eq!(added, [".baz", ".qux"]); + }); + temp_env::with_var_unset(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, || { + let added = parse_added_exclude_extensions(); + assert!(added.is_empty()); + }); + } + + #[test] + fn test_added_exclude_compress_extensions_default_compressible() { + use http::HeaderMap; + + let mut headers = HeaderMap::new(); + headers.insert("content-type", "text/plain".parse().unwrap()); + temp_env::with_var(ENV_COMPRESSION_ENABLED, Some("true"), || { + temp_env::with_var_unset(ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS, || { + assert!(is_compressible(&headers, "file.foo")); + assert!(is_compressible(&headers, "file.bar")); + }); + }); + } } diff --git a/crates/utils/src/string.rs b/crates/utils/src/string.rs index 24675db67..79ef5746e 100644 --- a/crates/utils/src/string.rs +++ b/crates/utils/src/string.rs @@ -157,10 +157,11 @@ pub fn has_pattern(patterns: &[&str], match_str: &str) -> bool { /// assert!(has_string_suffix_in_slice("document.TXT", &suffixes)); /// assert!(!has_string_suffix_in_slice("image.png", &suffixes)); /// ``` -pub fn has_string_suffix_in_slice(str: &str, list: &[&str]) -> bool { +pub fn has_string_suffix_in_slice(str: &str, list: &[impl AsRef]) -> bool { let str = str.to_lowercase(); for v in list { - if *v == "*" { + let v = v.as_ref(); + if v == "*" { return true; }