fix(download): preserve archive download integrity (#2415)

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
2026-04-07 16:25:34 +08:00
committed by GitHub
parent d6158c0481
commit c9c2428373
12 changed files with 699 additions and 18 deletions
Generated
+1
View File
@@ -3235,6 +3235,7 @@ dependencies = [
"urlencoding",
"uuid",
"walkdir",
"zip",
"zstd",
]
+1
View File
@@ -80,3 +80,4 @@ suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] }
rcgen.workspace = true
anyhow.workspace = true
rustls.workspace = true
zip.workspace = true
@@ -0,0 +1,329 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use http::header::{CONTENT_TYPE, HOST};
use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::{pre_sign_v4, sign_v4};
use s3s::Body;
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::error::Error;
use std::io::{Cursor, Write};
use std::process::Command;
use time::OffsetDateTime;
use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
const ARCHIVE_TEST_BUCKET: &str = "archive-download-integrity";
const MULTIPART_ARCHIVE_TEST_BUCKET: &str = "archive-multipart-integrity";
const MULTIPART_PART_SIZE: usize = 5 * 1024 * 1024;
fn build_zip_bytes(files: &[(&str, &[u8])]) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
let cursor = Cursor::new(Vec::new());
let mut zip = ZipWriter::new(cursor);
let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
for (name, content) in files {
zip.start_file(*name, options)?;
zip.write_all(content)?;
}
Ok(zip.finish()?.into_inner())
}
fn random_bytes(size: usize) -> Vec<u8> {
(0..size).map(|idx| (idx % 251) as u8).collect()
}
async fn start_rustfs_server_with_env(
env: &mut RustFSTestEnvironment,
extra_env: &[(&str, &str)],
) -> Result<(), Box<dyn Error + Send + Sync>> {
let binary_path = rustfs_binary_path();
let mut command = Command::new(&binary_path);
command.env("RUST_LOG", "rustfs=info,rustfs_notify=debug");
for (key, value) in extra_env {
command.env(key, value);
}
let process = command
.args([
"--address",
&env.address,
"--access-key",
&env.access_key,
"--secret-key",
&env.secret_key,
&env.temp_dir,
])
.spawn()?;
env.process = Some(process);
env.wait_for_server_ready().await?;
Ok(())
}
async fn presigned_get_request_with_accept_encoding(
url: &str,
access_key: &str,
secret_key: &str,
accept_encoding: &str,
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let signed = pre_sign_v4(
http::Request::builder()
.method(http::Method::GET)
.uri(uri)
.header(HOST, authority)
.body(Body::empty())?,
access_key,
secret_key,
"",
"us-east-1",
600,
OffsetDateTime::now_utc(),
);
let client = reqwest::Client::builder()
.no_proxy()
.no_gzip()
.no_brotli()
.no_zstd()
.no_deflate()
.build()?;
Ok(client
.get(signed.uri().to_string())
.header("Accept-Encoding", accept_encoding)
.send()
.await?)
}
async fn signed_put_request_with_headers(
url: &str,
access_key: &str,
secret_key: &str,
body: Vec<u8>,
content_type: &str,
content_encoding: &str,
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let request = http::Request::builder()
.method(http::Method::PUT)
.uri(uri)
.header(HOST, authority)
.header(CONTENT_TYPE, content_type)
.header("content-encoding", content_encoding)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
.body(Body::empty())?;
let signed = sign_v4(request, body.len() as i64, access_key, secret_key, "", "us-east-1");
let client = reqwest::Client::builder().no_proxy().build()?;
let mut builder = client.put(url).body(body);
for (name, value) in signed.headers() {
builder = builder.header(name, value);
}
Ok(builder.send().await?)
}
#[tokio::test]
#[serial]
async fn test_archive_put_rejects_content_encoding_by_default() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_without_cleanup(vec![]).await?;
env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?;
let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
let response =
signed_put_request_with_headers(&object_url, &env.access_key, &env.secret_key, zip_bytes, "application/zip", "gzip")
.await?;
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = response.text().await?;
assert!(
body.contains("InvalidArgument") || body.contains("Content-Encoding"),
"unexpected error body: {body}"
);
env.stop_server();
Ok(())
}
#[tokio::test]
#[serial]
async fn test_archive_download_roundtrip_with_http_compression_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_server_with_env(
&mut env,
&[
("RUSTFS_COMPRESS_ENABLE", "on"),
("RUSTFS_COMPRESS_MIME_TYPES", "text/*,application/json,application/zip"),
("RUSTFS_COMPRESS_MIN_SIZE", "1"),
],
)
.await?;
env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?;
let client = env.create_s3_client();
let zip_bytes = build_zip_bytes(&[
("docs/readme.txt", b"archive-download-integrity"),
("docs/notes.txt", b"response-compression-must-not-alter-zip-bytes"),
])?;
let expected_sha256 = Sha256::digest(&zip_bytes);
client
.put_object()
.bucket(ARCHIVE_TEST_BUCKET)
.key("bundle.zip")
.content_type("application/zip")
.body(ByteStream::from(zip_bytes.clone()))
.send()
.await?;
let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip");
let response =
presigned_get_request_with_accept_encoding(&object_url, &env.access_key, &env.secret_key, "gzip, br, zstd").await?;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.headers()
.get("content-encoding")
.and_then(|value| value.to_str().ok()),
None,
"archive download must not be HTTP-compressed"
);
let downloaded = response.bytes().await?;
assert_eq!(
downloaded.as_ref(),
zip_bytes.as_slice(),
"downloaded archive bytes must match uploaded bytes"
);
assert_eq!(
Sha256::digest(downloaded.as_ref()).as_slice(),
expected_sha256.as_slice(),
"archive SHA256 mismatch"
);
env.stop_server();
Ok(())
}
#[tokio::test]
#[serial]
async fn test_archive_multipart_roundtrip_preserves_bytes() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_without_cleanup(vec![]).await?;
env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
let client = env.create_s3_client();
let payload = random_bytes(MULTIPART_PART_SIZE + 512 * 1024);
let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
let expected_sha256 = Sha256::digest(&zip_bytes);
let create_output = client
.create_multipart_upload()
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
.key("multipart-bundle.zip")
.content_type("application/zip")
.send()
.await?;
let upload_id = create_output.upload_id().expect("multipart upload id");
let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
let upload_part_1 = client
.upload_part()
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
.key("multipart-bundle.zip")
.upload_id(upload_id)
.part_number(1)
.body(ByteStream::from(first_part))
.send()
.await?;
let upload_part_2 = client
.upload_part()
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
.key("multipart-bundle.zip")
.upload_id(upload_id)
.part_number(2)
.body(ByteStream::from(second_part))
.send()
.await?;
let completed_upload = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(upload_part_1.e_tag().unwrap_or_default())
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(upload_part_2.e_tag().unwrap_or_default())
.build(),
)
.build();
client
.complete_multipart_upload()
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
.key("multipart-bundle.zip")
.upload_id(upload_id)
.multipart_upload(completed_upload)
.send()
.await?;
let downloaded = client
.get_object()
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
.key("multipart-bundle.zip")
.send()
.await?
.body
.collect()
.await?
.into_bytes();
assert_eq!(
downloaded.as_ref(),
zip_bytes.as_slice(),
"multipart archive bytes must match uploaded bytes"
);
assert_eq!(
Sha256::digest(downloaded.as_ref()).as_slice(),
expected_sha256.as_slice(),
"multipart archive SHA256 mismatch"
);
env.stop_server();
Ok(())
}
}
+3
View File
@@ -56,6 +56,9 @@ mod special_chars_test;
#[cfg(test)]
mod content_encoding_test;
#[cfg(test)]
mod archive_download_integrity_test;
// ListObjectsV2 pagination test (Issue #1596)
#[cfg(test)]
mod list_objects_v2_pagination_test;
+10
View File
@@ -82,16 +82,26 @@ pub const STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES: &[&str] = &[
"video/*",
"audio/*",
"image/*",
// Archive formats (compressed)
"application/zip",
"application/gzip",
"application/x-gzip",
"application/x-zip-compressed",
"application/x-compress",
"application/x-spoon",
"application/x-rar-compressed",
"application/x-7z-compressed",
"application/x-bzip",
"application/x-bzip2",
"application/x-xz",
"application/x-lzip",
"application/x-lzma",
"application/x-lzop",
"application/zstd",
"application/x-zstd",
// Archive formats (uncompressed containers that are typically not further compressible)
"application/x-tar",
"application/tar",
"application/pdf",
"application/wasm",
"font/*",
+56
View File
@@ -170,6 +170,7 @@ pub trait CachedGetObjectSource {
fn content_type(&self) -> Option<&str>;
fn e_tag(&self) -> Option<&str>;
fn last_modified(&self) -> Option<&str>;
fn expires(&self) -> Option<&str>;
fn cache_control(&self) -> Option<&str>;
fn content_disposition(&self) -> Option<&str>;
fn content_encoding(&self) -> Option<&str>;
@@ -405,6 +406,10 @@ where
.last_modified()
.and_then(|s| OffsetDateTime::parse(s, &Rfc3339).ok())
.map(Timestamp::from);
let expires = cached
.expires()
.and_then(|s| OffsetDateTime::parse(s, &Rfc3339).ok())
.map(Timestamp::from);
let content_type = cached.content_type().and_then(|ct| ContentType::from_str(ct).ok());
@@ -416,6 +421,7 @@ where
accept_ranges: Some("bytes".to_string()),
e_tag: cached.e_tag().map(to_s3s_etag),
last_modified,
expires,
content_type,
cache_control: cached.cache_control().map(str::to_string),
content_disposition: cached.content_disposition().map(str::to_string),
@@ -890,8 +896,12 @@ pub fn build_get_object_output(
body,
content_length: Some(response_content_length),
last_modified,
expires: info.expires.map(Timestamp::from),
content_type,
cache_control: info.user_defined.get(CACHE_CONTROL.as_str()).cloned(),
content_disposition: info.user_defined.get(CONTENT_DISPOSITION.as_str()).cloned(),
content_encoding: info.content_encoding.clone(),
content_language: info.user_defined.get(CONTENT_LANGUAGE.as_str()).cloned(),
accept_ranges: Some("bytes".to_string()),
content_range,
e_tag: info.etag.as_ref().map(|etag| to_s3s_etag(etag)),
@@ -1052,6 +1062,7 @@ mod tests {
content_type: Option<String>,
e_tag: Option<String>,
last_modified: Option<String>,
expires: Option<String>,
cache_control: Option<String>,
content_disposition: Option<String>,
content_encoding: Option<String>,
@@ -1090,6 +1101,10 @@ mod tests {
self.last_modified.as_deref()
}
fn expires(&self) -> Option<&str> {
self.expires.as_deref()
}
fn cache_control(&self) -> Option<&str> {
self.cache_control.as_deref()
}
@@ -1624,6 +1639,7 @@ mod tests {
content_type: None,
e_tag: None,
last_modified: None,
expires: None,
cache_control: None,
content_disposition: None,
content_encoding: None,
@@ -1651,6 +1667,46 @@ mod tests {
assert_eq!(result.output.checksum_type, Some(ChecksumType::from_static(ChecksumType::FULL_OBJECT)));
}
#[test]
fn build_get_object_output_preserves_http_metadata_like_cached_path() {
let mut info = ObjectInfo {
content_type: Some("application/octet-stream".to_string()),
content_encoding: Some("zstd".to_string()),
etag: Some("etag".to_string()),
expires: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
info.user_defined
.insert("cache-control".to_string(), "max-age=3600".to_string());
info.user_defined
.insert("content-disposition".to_string(), "attachment; filename=\"bundle.zip\"".to_string());
info.user_defined.insert("content-language".to_string(), "en-US".to_string());
let output = build_get_object_output(
None,
&info,
info.content_type
.as_ref()
.and_then(|content_type| ContentType::from_str(content_type).ok()),
info.mod_time.map(Timestamp::from),
8,
None,
None,
None,
None,
None,
&GetObjectChecksums::default(),
None,
None,
);
assert_eq!(output.cache_control.as_deref(), Some("max-age=3600"));
assert_eq!(output.content_disposition.as_deref(), Some("attachment; filename=\"bundle.zip\""));
assert_eq!(output.content_language.as_deref(), Some("en-US"));
assert_eq!(output.content_encoding.as_deref(), Some("zstd"));
assert_eq!(output.expires, Some(Timestamp::from(OffsetDateTime::UNIX_EPOCH)));
}
#[test]
fn build_cors_wrapped_get_object_flow_result_uses_wrapped_mode() {
let result = build_cors_wrapped_get_object_flow_result(
+7 -1
View File
@@ -23,7 +23,7 @@ use crate::storage::entity;
use crate::storage::helper::OperationHelper;
use crate::storage::options::{
copy_src_opts, extract_metadata, get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts,
parse_copy_source_range, put_opts,
parse_copy_source_range, put_opts, validate_archive_content_encoding,
};
use crate::storage::request_context::spawn_traced;
use crate::storage::s3_api::multipart::build_list_parts_output;
@@ -559,6 +559,12 @@ impl DefaultMultipartUsecase {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
validate_archive_content_encoding(
&key,
req.headers.get("content-type").and_then(|value| value.to_str().ok()),
req.headers.get("content-encoding").and_then(|value| value.to_str().ok()),
)?;
let mut metadata = extract_metadata(&req.headers);
if let Some(tags) = tagging {
+15
View File
@@ -38,6 +38,7 @@ use crate::storage::helper::OperationHelper;
use crate::storage::options::{
copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name,
filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts,
validate_archive_content_encoding,
};
use crate::storage::s3_api::multipart::parse_list_parts_params;
use crate::storage::s3_api::{acl, restore, select};
@@ -315,6 +316,20 @@ fn apply_put_request_metadata(
tagging: Option<TaggingHeader>,
storage_class: Option<StorageClass>,
) -> S3Result<()> {
let request_content_type = content_type.as_ref().map(ToString::to_string).or_else(|| {
headers
.get("content-type")
.and_then(|value| value.to_str().ok())
.map(ToOwned::to_owned)
});
let request_content_encoding = content_encoding.as_ref().map(ToString::to_string).or_else(|| {
headers
.get("content-encoding")
.and_then(|value| value.to_str().ok())
.map(ToOwned::to_owned)
});
validate_archive_content_encoding(object_name, request_content_type.as_deref(), request_content_encoding.as_deref())?;
if let Some(cache_control) = cache_control {
metadata.insert("cache-control".to_string(), cache_control.to_string());
}
@@ -104,6 +104,10 @@ impl ObjectIoCachedGetObjectSource for CachedGetObject {
self.last_modified.as_deref()
}
fn expires(&self) -> Option<&str> {
self.expires.as_deref()
}
fn cache_control(&self) -> Option<&str> {
self.cache_control.as_deref()
}
+83 -13
View File
@@ -46,6 +46,8 @@ use rustfs_config::{
DEFAULT_COMPRESS_ENABLE, DEFAULT_COMPRESS_EXTENSIONS, DEFAULT_COMPRESS_MIME_TYPES, DEFAULT_COMPRESS_MIN_SIZE,
ENV_COMPRESS_ENABLE, ENV_COMPRESS_EXTENSIONS, ENV_COMPRESS_MIME_TYPES, ENV_COMPRESS_MIN_SIZE, EnableState,
};
use rustfs_ecstore::compress::{STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES, STANDARD_EXCLUDE_COMPRESS_EXTENSIONS};
use rustfs_utils::string::{has_pattern, has_string_suffix_in_slice};
use std::str::FromStr;
use tower_http::compression::predicate::Predicate;
use tracing::debug;
@@ -196,6 +198,20 @@ impl CompressionConfig {
None
}
pub(crate) fn is_excluded_filename(filename: &str) -> bool {
has_string_suffix_in_slice(&filename.to_ascii_lowercase(), STANDARD_EXCLUDE_COMPRESS_EXTENSIONS)
}
pub(crate) fn is_excluded_mime_type(content_type: &str) -> bool {
let main_type = content_type
.split(';')
.next()
.unwrap_or(content_type)
.trim()
.to_ascii_lowercase();
!main_type.is_empty() && has_pattern(STANDARD_EXCLUDE_COMPRESS_CONTENT_TYPES, &main_type)
}
}
impl Default for CompressionConfig {
@@ -299,23 +315,37 @@ impl Predicate for CompressionPredicate {
return false;
}
// Check if the response matches configured extension via Content-Disposition
// Hard-stop archive/media/package MIME types even if the whitelist matches.
// This includes tar, gzip, bzip2, xz, zstd, zip, rar, 7z, lzip, lzma, lzop variants,
// plus video/*, audio/*, image/*, font/*, application/pdf, and application/wasm.
if let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE)
&& let Ok(ct) = content_type.to_str()
{
if CompressionConfig::is_excluded_mime_type(ct) {
debug!("Skipping compression for excluded Content-Type '{}'", ct);
return false;
}
if self.config.matches_mime_type(ct) {
debug!("Compressing response: Content-Type '{}' matches configured MIME pattern", ct);
return true;
}
}
// Hard-stop archive-like attachment downloads even if the whitelist matches.
if let Some(content_disposition) = response.headers().get(http::header::CONTENT_DISPOSITION)
&& let Ok(cd) = content_disposition.to_str()
&& let Some(filename) = CompressionConfig::extract_filename_from_content_disposition(cd)
&& self.config.matches_extension(&filename)
{
debug!("Compressing response: filename '{}' matches configured extension", filename);
return true;
}
if CompressionConfig::is_excluded_filename(&filename) {
debug!("Skipping compression for excluded filename '{}'", filename);
return false;
}
// Check if the response matches configured MIME type
if let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE)
&& let Ok(ct) = content_type.to_str()
&& self.config.matches_mime_type(ct)
{
debug!("Compressing response: Content-Type '{}' matches configured MIME pattern", ct);
return true;
if self.config.matches_extension(&filename) {
debug!("Compressing response: filename '{}' matches configured extension", filename);
return true;
}
}
// Default: don't compress (whitelist approach)
@@ -405,7 +435,11 @@ use std::task::{Context, Poll};
use tower::{Layer, Service};
/// Tower layer that injects `RequestPathCategory` into each response's extensions
/// based on the incoming request URI path. Must be placed before `CompressionLayer`.
/// based on the incoming request URI path.
///
/// It must be placed inside `CompressionLayer` so the category is available when
/// the outer compression middleware evaluates its response predicate. With
/// `tower::ServiceBuilder`, that means adding this layer after `CompressionLayer`.
#[derive(Clone, Copy, Debug)]
pub(crate) struct PathCategoryInjectionLayer;
@@ -627,6 +661,42 @@ mod tests {
assert_eq!(predicate.config.min_size, 1000);
}
#[test]
fn test_compression_predicate_skips_archive_mime_type_even_when_whitelisted() {
let predicate = CompressionPredicate::new(CompressionConfig {
enabled: true,
extensions: vec![],
mime_patterns: vec!["application/zip".to_string()],
min_size: 0,
});
let response = Response::builder()
.header(http::header::CONTENT_TYPE, "application/zip")
.header(http::header::CONTENT_LENGTH, "4096")
.body(http_body_util::Empty::<bytes::Bytes>::new())
.expect("response");
assert!(!predicate.should_compress(&response));
}
#[test]
fn test_compression_predicate_skips_archive_filename_even_when_whitelisted() {
let predicate = CompressionPredicate::new(CompressionConfig {
enabled: true,
extensions: vec![".zip".to_string()],
mime_patterns: vec![],
min_size: 0,
});
let response = Response::builder()
.header(http::header::CONTENT_DISPOSITION, r#"attachment; filename="bundle.zip""#)
.header(http::header::CONTENT_LENGTH, "4096")
.body(http_body_util::Empty::<bytes::Bytes>::new())
.expect("response");
assert!(!predicate.should_compress(&response));
}
#[test]
fn test_path_category_classify_s3() {
assert_eq!(PathCategory::classify("/"), PathCategory::S3DataPlane);
+96 -3
View File
@@ -603,8 +603,8 @@ fn process_connection(
// 9. KeystoneAuthLayer — X-Auth-Token validation
// 10. TraceLayer — request/response tracing + metrics
// 11. PropagateRequestIdLayer — X-Request-ID → response
// 12. PathCategoryInjectionLayer — injects path category for compression
// 13. CompressionLayer — response compression (whitelist, path-aware)
// 12. CompressionLayer — response compression (whitelist, path-aware)
// 13. PathCategoryInjectionLayer — injects path category for compression predicate
// 14. ObjectAttributesEtagFixLayer — ETag fix for GetObjectAttributes
// 15. ConditionalCorsLayer — S3 API CORS
// 16. RedirectLayer — console redirect (conditional)
@@ -739,8 +739,8 @@ fn process_connection(
.layer(PropagateRequestIdLayer::x_request_id())
// Compress responses based on whitelist configuration
// Only compresses when enabled and matches configured extensions/MIME types
.layer(PathCategoryInjectionLayer)
.layer(CompressionLayer::new().compress_when(PathAwareCompressionPredicate::new(compression_config)))
.layer(PathCategoryInjectionLayer)
.layer(ObjectAttributesEtagFixLayer)
// Conditional CORS layer: only applies to S3 API requests (not Admin, not Console)
// Admin has its own CORS handling in router.rs
@@ -927,8 +927,16 @@ fn get_default_tcp_keepalive() -> TcpKeepalive {
#[cfg(test)]
mod tests {
use super::*;
use crate::server::compress::RequestPathCategory;
use bytes::Bytes;
use http::HeaderMap;
use http::Request as HttpRequest;
use http_body_util::Empty;
use opentelemetry::propagation::Extractor;
use std::convert::Infallible;
use std::future::Ready;
use std::task::{Context, Poll};
use tower::{Layer, Service, ServiceBuilder};
/// Baseline constants — reference the authoritative config defaults.
/// If a config default changes, tests automatically follow.
@@ -1065,4 +1073,89 @@ mod tests {
assert_eq!(carrier.get("Content-Type"), Some("application/json"));
assert_eq!(carrier.get("CONTENT-TYPE"), Some("application/json"));
}
#[derive(Clone, Copy)]
struct ObserveCategoryLayer;
#[derive(Clone)]
struct ObserveCategoryService<S> {
inner: S,
}
impl<S> Layer<S> for ObserveCategoryLayer {
type Service = ObserveCategoryService<S>;
fn layer(&self, inner: S) -> Self::Service {
ObserveCategoryService { inner }
}
}
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ObserveCategoryService<S>
where
S: Service<HttpRequest<ReqBody>, Response = Response<ResBody>, Error = Infallible>,
{
type Response = Response<ResBody>;
type Error = Infallible;
type Future = Ready<std::result::Result<Response<ResBody>, Infallible>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, req: HttpRequest<ReqBody>) -> Self::Future {
let response = futures::executor::block_on(self.inner.call(req)).expect("infallible");
let mut response = response;
let seen = response.extensions().get::<RequestPathCategory>().is_some();
response
.headers_mut()
.insert("x-category-seen", if seen { "true" } else { "false" }.parse().expect("header"));
std::future::ready(Ok(response))
}
}
#[derive(Clone, Copy)]
struct OkService;
impl<ReqBody> Service<HttpRequest<ReqBody>> for OkService {
type Response = Response<Empty<Bytes>>;
type Error = Infallible;
type Future = Ready<std::result::Result<Response<Empty<Bytes>>, Infallible>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, _req: HttpRequest<ReqBody>) -> Self::Future {
std::future::ready(Ok(Response::new(Empty::new())))
}
}
#[test]
fn test_service_builder_order_regression_for_response_extensions() {
let request = HttpRequest::builder().uri("/bucket/archive.zip").body(()).expect("request");
let mut broken_order = ServiceBuilder::new()
.layer(PathCategoryInjectionLayer)
.layer(ObserveCategoryLayer)
.service(OkService);
let broken_response = futures::executor::block_on(broken_order.call(request)).expect("response");
assert_eq!(
broken_response.headers().get("x-category-seen").and_then(|v| v.to_str().ok()),
Some("false")
);
let request = HttpRequest::builder().uri("/bucket/archive.zip").body(()).expect("request");
let mut fixed_order = ServiceBuilder::new()
.layer(ObserveCategoryLayer)
.layer(PathCategoryInjectionLayer)
.service(OkService);
let fixed_response = futures::executor::block_on(fixed_order.call(request)).expect("response");
assert_eq!(
fixed_response.headers().get("x-category-seen").and_then(|v| v.to_str().ok()),
Some("true")
);
}
}
+94 -1
View File
@@ -33,7 +33,7 @@ use rustfs_policy::service_type::ServiceType;
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use rustfs_utils::http::AMZ_CONTENT_SHA256;
use rustfs_utils::path::is_dir_object;
use s3s::{S3Result, s3_error};
use s3s::{S3Error, S3ErrorCode, S3Result, s3_error};
use std::collections::HashMap;
use std::sync::LazyLock;
use tracing::error;
@@ -349,6 +349,73 @@ pub(crate) fn normalize_content_encoding_for_storage(value: &str) -> Option<Stri
if normalized.is_empty() { None } else { Some(normalized) }
}
const ENV_ALLOW_ARCHIVE_CONTENT_ENCODING: &str = "RUSTFS_ALLOW_ARCHIVE_CONTENT_ENCODING";
const ARCHIVE_CONTENT_ENCODING_BLOCKED_SUFFIXES: &[&str] = &[
".zip",
".tar",
".tar.gz",
".tgz",
".tar.bz2",
".tbz",
".tbz2",
".tar.xz",
".txz",
".tar.zst",
".tar.zstd",
".tzst",
];
const ARCHIVE_CONTENT_ENCODING_BLOCKED_CONTENT_TYPES: &[&str] =
&["application/zip", "application/x-zip-compressed", "application/x-tar"];
fn is_archive_object_name_for_content_encoding(object_name: &str) -> bool {
let object_name = object_name.to_ascii_lowercase();
ARCHIVE_CONTENT_ENCODING_BLOCKED_SUFFIXES
.iter()
.any(|suffix| object_name.ends_with(suffix))
}
fn is_archive_content_type_for_content_encoding(content_type: &str) -> bool {
let main_type = content_type
.split(';')
.next()
.unwrap_or(content_type)
.trim()
.to_ascii_lowercase();
ARCHIVE_CONTENT_ENCODING_BLOCKED_CONTENT_TYPES
.iter()
.any(|candidate| main_type == *candidate)
}
pub(crate) fn validate_archive_content_encoding(
object_name: &str,
content_type: Option<&str>,
content_encoding: Option<&str>,
) -> S3Result<()> {
if rustfs_utils::get_env_bool(ENV_ALLOW_ARCHIVE_CONTENT_ENCODING, false) {
return Ok(());
}
let Some(content_encoding) = content_encoding.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(());
};
let is_archive_like = is_archive_object_name_for_content_encoding(object_name)
|| content_type.is_some_and(is_archive_content_type_for_content_encoding);
if !is_archive_like {
return Ok(());
}
Err(S3Error::with_message(
S3ErrorCode::InvalidArgument,
format!(
"Content-Encoding '{content_encoding}' is not allowed for archive objects by default; set {ENV_ALLOW_ARCHIVE_CONTENT_ENCODING}=true to allow legacy behavior"
),
))
}
/// Extracts metadata from headers and returns it as a HashMap with object name for MIME type detection.
pub fn extract_metadata_from_mime_with_object_name(
headers: &HeaderMap<HeaderValue>,
@@ -692,6 +759,8 @@ fn get_content_sha256_cksum(headers: &HeaderMap<HeaderValue>, service_type: Serv
#[cfg(test)]
mod tests {
use temp_env;
use super::*;
use http::{HeaderMap, HeaderValue};
use std::collections::HashMap;
@@ -1358,6 +1427,30 @@ mod tests {
assert_eq!(detect_content_type_from_object_name("noextension"), "application/octet-stream");
}
#[test]
fn test_validate_archive_content_encoding_rejects_archive_suffix_by_default() {
let err = validate_archive_content_encoding("bundle.tar.gz", Some("application/gzip"), Some("gzip")).unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
}
#[test]
fn test_validate_archive_content_encoding_rejects_archive_mime_by_default() {
let err = validate_archive_content_encoding("bundle", Some("application/zip"), Some("gzip")).unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
}
#[test]
fn test_validate_archive_content_encoding_allows_non_archive_precompressed_object() {
validate_archive_content_encoding("logs/app.log.zst", Some("text/plain"), Some("zstd")).expect("non-archive");
}
#[test]
fn test_validate_archive_content_encoding_allows_legacy_opt_in() {
temp_env::with_var(ENV_ALLOW_ARCHIVE_CONTENT_ENCODING, Some("true"), || {
validate_archive_content_encoding("bundle.zip", Some("application/zip"), Some("gzip")).expect("legacy opt-in");
});
}
#[test]
fn test_parse_copy_source_range() {
// Test complete range: bytes=0-1023