mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-16 09:58:21 +00:00
feat(storage): stage multipart compression behind RUSTFS_COMPRESSION_MULTIPART_ENABLED
Review follow-up: a rolling-upgrade window must not create new compressed multipart objects while pre-fix nodes (whose decompressor is not resumable) may still serve reads. The session marker is now additionally gated on RUSTFS_COMPRESSION_MULTIPART_ENABLED, default off, so the restored capability stays dark until the operator confirms fleet convergence. The default flips per the multipart-compression-default-off-window entry in docs/architecture/compat-cleanup-register.md once the minimum supported direct-upgrade release ships the resumable decoder.
This commit is contained in:
@@ -72,6 +72,7 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
||||
command
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
||||
.args([
|
||||
"--address",
|
||||
&env.address,
|
||||
@@ -652,6 +653,7 @@ async fn start_rustfs_with_compression_and_sse(
|
||||
let process = Command::new(&binary_path)
|
||||
.env("RUSTFS_CONSOLE_ENABLE", "false")
|
||||
.env("RUSTFS_COMPRESSION_ENABLED", "true")
|
||||
.env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true")
|
||||
.env("RUSTFS_SSE_S3_MASTER_KEY", master_key)
|
||||
.env("RUST_LOG", "rustfs=info,rustfs_ecstore=info")
|
||||
.stdout(std::process::Stdio::from(server_log))
|
||||
|
||||
@@ -1839,6 +1839,7 @@ async fn four_node_multipart_disk_compression_roundtrip() -> TestResult {
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
configure_reader_metric_cluster(&mut cluster, &collector);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "inline-multipart-compression-roundtrip";
|
||||
@@ -1873,6 +1874,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
|
||||
let sse_master_key = base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]);
|
||||
cluster.set_env("RUSTFS_SSE_S3_MASTER_KEY", sse_master_key);
|
||||
cluster.set_env("RUSTFS_COMPRESSION_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true");
|
||||
configure_mixed_msgpack_cluster(&mut cluster, &collector)?;
|
||||
cluster.start().await?;
|
||||
|
||||
|
||||
@@ -278,7 +278,9 @@ pub mod cluster {
|
||||
}
|
||||
|
||||
pub mod compression {
|
||||
pub use crate::io_support::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled};
|
||||
pub use crate::io_support::compress::{
|
||||
MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible, is_disk_compression_enabled, is_multipart_disk_compression_enabled,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod config {
|
||||
|
||||
@@ -31,6 +31,13 @@ pub const ENV_DISK_COMPRESSION_MIME_TYPES: &str = "RUSTFS_COMPRESSION_MIME_TYPES
|
||||
// Environment variable for additional extensions to exclude from compression (comma-separated, e.g. ".foo,.bar")
|
||||
pub const ENV_ADDED_EXCLUDE_COMPRESS_EXTENSIONS: &str = "RUSTFS_ADDED_EXCLUDE_COMPRESS_EXTENSIONS";
|
||||
|
||||
// Environment variable to additionally enable disk compression for multipart uploads.
|
||||
// Default off: nodes from before the resumable decompressor fix fail transient reads of
|
||||
// compressed objects, so multipart compression stays dark until the operator confirms the
|
||||
// fleet has converged on a fixed build (see docs/architecture/compat-cleanup-register.md,
|
||||
// `multipart-compression-default-off-window`, for the default-flip condition).
|
||||
pub const ENV_DISK_COMPRESSION_MULTIPART_ENABLED: &str = "RUSTFS_COMPRESSION_MULTIPART_ENABLED";
|
||||
|
||||
pub const DEFAULT_DISK_COMPRESS_EXTENSIONS: &str = ".txt,.log,.csv,.json,.tar,.xml,.bin";
|
||||
pub const DEFAULT_DISK_COMPRESS_MIME_TYPES: &str = "text/*,application/json,application/xml,binary/octet-stream";
|
||||
|
||||
@@ -171,6 +178,21 @@ pub fn is_disk_compression_enabled() -> bool {
|
||||
DISK_COMPRESSION_CONFIG.get_or_init(parse_disk_compression_config).enabled
|
||||
}
|
||||
|
||||
// Parsed once at first use, mirroring DISK_COMPRESSION_CONFIG.
|
||||
static MULTIPART_DISK_COMPRESSION_ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
|
||||
/// Whether multipart uploads may advertise disk compression. Requires the
|
||||
/// regular disk-compression gates to pass as well; this is the staged-rollout
|
||||
/// switch that keeps multipart compression dark during rolling upgrades from
|
||||
/// builds whose decompressor was not yet resumable.
|
||||
pub fn is_multipart_disk_compression_enabled() -> bool {
|
||||
*MULTIPART_DISK_COMPRESSION_ENABLED.get_or_init(|| {
|
||||
env::var(ENV_DISK_COMPRESSION_MULTIPART_ENABLED)
|
||||
.map(|s| matches!(s.to_ascii_lowercase().as_str(), "true" | "on" | "1"))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
}
|
||||
|
||||
fn is_disk_compressible_with_config(headers: &http::HeaderMap, object_name: &str, config: &DiskCompressionConfig) -> bool {
|
||||
// Check if disk compression is enabled (read once at first use, then fixed for process lifetime)
|
||||
if !config.enabled {
|
||||
|
||||
Reference in New Issue
Block a user