From 64508ae77094e8e8ae21854ef761df1f34449069 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 12 Apr 2026 02:14:52 +0800 Subject: [PATCH] fix(storage): align archive content-encoding with S3 semantics (#2496) --- .../src/archive_download_integrity_test.rs | 356 +++++++++++++++++- crates/e2e_test/src/content_encoding_test.rs | 59 +++ rustfs/src/storage/options.rs | 70 +++- 3 files changed, 469 insertions(+), 16 deletions(-) diff --git a/crates/e2e_test/src/archive_download_integrity_test.rs b/crates/e2e_test/src/archive_download_integrity_test.rs index d5015e844..daedbc4c2 100644 --- a/crates/e2e_test/src/archive_download_integrity_test.rs +++ b/crates/e2e_test/src/archive_download_integrity_test.rs @@ -15,6 +15,8 @@ #[cfg(test)] mod tests { use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path}; + use aws_sdk_s3::Client as S3Client; + use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; use http::header::{CONTENT_TYPE, HOST}; @@ -216,12 +218,129 @@ mod tests { Ok(builder.send().await?) } + async fn assert_archive_object_content_encoding( + client: &S3Client, + bucket: &str, + key: &str, + expected_content_encoding: Option<&str>, + expected_body: &[u8], + ) -> Result<(), Box> { + let head_resp = client.head_object().bucket(bucket).key(key).send().await?; + assert_eq!(head_resp.content_encoding(), expected_content_encoding); + + let get_resp = client.get_object().bucket(bucket).key(key).send().await?; + assert_eq!(get_resp.content_encoding(), expected_content_encoding); + let body = get_resp.body.collect().await?.into_bytes(); + assert_eq!(body.as_ref(), expected_body); + + Ok(()) + } + + async fn complete_archive_multipart_upload_with_content_encoding( + client: &S3Client, + bucket: &str, + key: &str, + content_encoding: Option<&str>, + ) -> Result, Box> { + let payload = random_bytes(MULTIPART_PART_SIZE + 256 * 1024); + let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?; + assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold"); + + let mut create_builder = client + .create_multipart_upload() + .bucket(bucket) + .key(key) + .content_type("application/zip"); + if let Some(content_encoding) = content_encoding { + create_builder = create_builder.content_encoding(content_encoding); + } + let create_output = create_builder.send().await?; + let upload_id = create_output.upload_id().expect("multipart upload id"); + + let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec(); + let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec(); + + let upload_part_1 = client + .upload_part() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .part_number(1) + .body(ByteStream::from(first_part)) + .send() + .await?; + + let upload_part_2 = client + .upload_part() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .part_number(2) + .body(ByteStream::from(second_part)) + .send() + .await?; + + let completed_upload = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(upload_part_1.e_tag().unwrap_or_default()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(upload_part_2.e_tag().unwrap_or_default()) + .build(), + ) + .build(); + + client + .complete_multipart_upload() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .multipart_upload(completed_upload) + .send() + .await?; + + Ok(zip_bytes) + } + #[tokio::test] #[serial] - async fn test_archive_put_rejects_content_encoding_by_default() -> Result<(), Box> { + async fn test_archive_put_allows_content_encoding_by_default() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; - env.start_rustfs_server_without_cleanup(vec![]).await?; + start_rustfs_server_with_env(&mut env, &[]).await?; + env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?; + let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?; + let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip"); + let response = + signed_put_request_with_headers(&object_url, &env.access_key, &env.secret_key, zip_bytes, "application/zip", "gzip") + .await?; + + assert_eq!(response.status(), StatusCode::OK); + + let client = env.create_s3_client(); + let head_resp = client + .head_object() + .bucket(ARCHIVE_TEST_BUCKET) + .key("bundle.zip") + .send() + .await?; + assert_eq!(head_resp.content_encoding(), Some("gzip")); + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_archive_put_rejects_content_encoding_when_strict_mode_enabled() -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?; env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?; let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?; let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle.zip"); @@ -232,7 +351,145 @@ mod tests { assert_eq!(response.status(), StatusCode::BAD_REQUEST); let body = response.text().await?; assert!( - body.contains("InvalidArgument") || body.contains("Content-Encoding"), + body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"), + "unexpected error body: {body}" + ); + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_archive_put_with_aws_chunked_does_not_persist_content_encoding_by_default() + -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_server_with_env(&mut env, &[]).await?; + env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?; + + let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?; + let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked.zip"); + let response = signed_put_request_with_headers( + &object_url, + &env.access_key, + &env.secret_key, + zip_bytes.clone(), + "application/zip", + "aws-chunked", + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + + let client = env.create_s3_client(); + assert_archive_object_content_encoding( + &client, + ARCHIVE_TEST_BUCKET, + "bundle-aws-chunked.zip", + None, + zip_bytes.as_slice(), + ) + .await?; + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_archive_put_with_aws_chunked_and_effective_encoding_roundtrips_by_default() + -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_server_with_env(&mut env, &[]).await?; + env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?; + + let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?; + let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-aws-chunked-gzip.zip"); + let response = signed_put_request_with_headers( + &object_url, + &env.access_key, + &env.secret_key, + zip_bytes.clone(), + "application/zip", + "aws-chunked,gzip", + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + + let client = env.create_s3_client(); + assert_archive_object_content_encoding( + &client, + ARCHIVE_TEST_BUCKET, + "bundle-aws-chunked-gzip.zip", + Some("gzip"), + zip_bytes.as_slice(), + ) + .await?; + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_archive_put_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?; + env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?; + + let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?; + let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked.zip"); + let response = signed_put_request_with_headers( + &object_url, + &env.access_key, + &env.secret_key, + zip_bytes.clone(), + "application/zip", + "aws-chunked", + ) + .await?; + assert_eq!(response.status(), StatusCode::OK); + + let client = env.create_s3_client(); + assert_archive_object_content_encoding( + &client, + ARCHIVE_TEST_BUCKET, + "bundle-strict-aws-chunked.zip", + None, + zip_bytes.as_slice(), + ) + .await?; + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_archive_put_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled() + -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?; + env.create_test_bucket(ARCHIVE_TEST_BUCKET).await?; + + let zip_bytes = build_zip_bytes(&[("alpha.txt", b"archive-body")])?; + let object_url = format!("{}/{}/{}", env.url, ARCHIVE_TEST_BUCKET, "bundle-strict-aws-chunked-gzip.zip"); + let response = signed_put_request_with_headers( + &object_url, + &env.access_key, + &env.secret_key, + zip_bytes, + "application/zip", + "aws-chunked,gzip", + ) + .await?; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = response.text().await?; + assert!( + body.contains("InvalidArgument") || body.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING"), "unexpected error body: {body}" ); @@ -398,6 +655,99 @@ mod tests { Ok(()) } + #[tokio::test] + #[serial] + async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default() + -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_server_with_env(&mut env, &[]).await?; + env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?; + + let client = env.create_s3_client(); + let zip_bytes = complete_archive_multipart_upload_with_content_encoding( + &client, + MULTIPART_ARCHIVE_TEST_BUCKET, + "multipart-aws-chunked-gzip.zip", + Some("aws-chunked,gzip"), + ) + .await?; + assert_archive_object_content_encoding( + &client, + MULTIPART_ARCHIVE_TEST_BUCKET, + "multipart-aws-chunked-gzip.zip", + Some("gzip"), + zip_bytes.as_slice(), + ) + .await?; + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_archive_multipart_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box> + { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?; + env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?; + + let client = env.create_s3_client(); + let zip_bytes = complete_archive_multipart_upload_with_content_encoding( + &client, + MULTIPART_ARCHIVE_TEST_BUCKET, + "multipart-strict-aws-chunked.zip", + Some("aws-chunked"), + ) + .await?; + assert_archive_object_content_encoding( + &client, + MULTIPART_ARCHIVE_TEST_BUCKET, + "multipart-strict-aws-chunked.zip", + None, + zip_bytes.as_slice(), + ) + .await?; + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + #[serial] + async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled() + -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + start_rustfs_server_with_env(&mut env, &[("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING", "true")]).await?; + env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?; + + let client = env.create_s3_client(); + let create_result = client + .create_multipart_upload() + .bucket(MULTIPART_ARCHIVE_TEST_BUCKET) + .key("multipart-strict-aws-chunked-gzip.zip") + .content_type("application/zip") + .content_encoding("aws-chunked,gzip") + .send() + .await; + let err = create_result.expect_err("strict mode should reject effective archive content encoding"); + assert_eq!(err.code(), Some("InvalidArgument")); + assert!( + err.message().is_some_and(|message| { + message.contains("Content-Encoding") && message.contains("RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING") + }), + "unexpected error metadata: code={:?}, message={:?}", + err.code(), + err.message() + ); + + env.stop_server(); + Ok(()) + } + #[tokio::test] #[serial] async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path() diff --git a/crates/e2e_test/src/content_encoding_test.rs b/crates/e2e_test/src/content_encoding_test.rs index 3c63eb89e..c0a741d95 100644 --- a/crates/e2e_test/src/content_encoding_test.rs +++ b/crates/e2e_test/src/content_encoding_test.rs @@ -138,4 +138,63 @@ mod tests { env.stop_server(); } + + /// Issue #2475 / Route A: when aws-chunked is combined with an effective object encoding, + /// only the effective encoding should roundtrip through GET/HEAD. + #[tokio::test] + #[serial] + async fn test_content_encoding_aws_chunked_with_effective_encoding_roundtrip() { + init_logging(); + info!("aws-chunked,gzip should persist only gzip"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = env.create_s3_client(); + let bucket = "content-encoding-aws-chunked-gzip-test"; + let key = "streamed/object.txt"; + let content = b"streaming upload body with effective gzip encoding"; + + client + .create_bucket() + .bucket(bucket) + .send() + .await + .expect("Failed to create bucket"); + + client + .put_object() + .bucket(bucket) + .key(key) + .content_type("text/plain") + .content_encoding("aws-chunked,gzip") + .body(ByteStream::from_static(content)) + .send() + .await + .expect("PUT failed"); + + let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed"); + assert_eq!( + get_resp.content_encoding(), + Some("gzip"), + "GET must return only the effective content encoding after aws-chunked is stripped" + ); + let body = get_resp.body.collect().await.unwrap().into_bytes(); + assert_eq!(body.as_ref(), content, "Body content mismatch"); + + let head_resp = client + .head_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("HEAD failed"); + assert_eq!( + head_resp.content_encoding(), + Some("gzip"), + "HEAD must return only the effective content encoding after aws-chunked is stripped" + ); + + env.stop_server(); + } } diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index c54bc62bb..78adef0a3 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -349,7 +349,7 @@ pub(crate) fn normalize_content_encoding_for_storage(value: &str) -> Option, content_encoding: Option<&str>, ) -> S3Result<()> { - if rustfs_utils::get_env_bool(ENV_ALLOW_ARCHIVE_CONTENT_ENCODING, false) { + if !archive_content_encoding_strict_mode() { return Ok(()); } - let Some(content_encoding) = content_encoding.map(str::trim).filter(|value| !value.is_empty()) else { + let Some(content_encoding) = content_encoding.and_then(normalize_content_encoding_for_storage) else { return Ok(()); }; @@ -411,11 +411,15 @@ pub(crate) fn validate_archive_content_encoding( Err(S3Error::with_message( S3ErrorCode::InvalidArgument, format!( - "Content-Encoding '{content_encoding}' is not allowed for archive objects by default; set {ENV_ALLOW_ARCHIVE_CONTENT_ENCODING}=true to allow legacy behavior" + "Content-Encoding '{content_encoding}' is not allowed for archive objects when {ENV_REJECT_ARCHIVE_CONTENT_ENCODING}=true; unset {ENV_REJECT_ARCHIVE_CONTENT_ENCODING} or set it to false to restore compatibility-first behavior" ), )) } +fn archive_content_encoding_strict_mode() -> bool { + rustfs_utils::get_env_bool(ENV_REJECT_ARCHIVE_CONTENT_ENCODING, false) +} + /// Extracts metadata from headers and returns it as a HashMap with object name for MIME type detection. pub fn extract_metadata_from_mime_with_object_name( headers: &HeaderMap, @@ -1428,15 +1432,13 @@ mod tests { } #[test] - fn test_validate_archive_content_encoding_rejects_archive_suffix_by_default() { - let err = validate_archive_content_encoding("bundle.tar.gz", Some("application/gzip"), Some("gzip")).unwrap_err(); - assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + fn test_validate_archive_content_encoding_allows_archive_suffix_by_default() { + validate_archive_content_encoding("bundle.tar.gz", Some("application/gzip"), Some("gzip")).expect("default allow"); } #[test] - fn test_validate_archive_content_encoding_rejects_archive_mime_by_default() { - let err = validate_archive_content_encoding("bundle", Some("application/zip"), Some("gzip")).unwrap_err(); - assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + fn test_validate_archive_content_encoding_allows_archive_mime_by_default() { + validate_archive_content_encoding("bundle", Some("application/zip"), Some("gzip")).expect("default allow"); } #[test] @@ -1445,9 +1447,51 @@ mod tests { } #[test] - fn test_validate_archive_content_encoding_allows_legacy_opt_in() { - temp_env::with_var(ENV_ALLOW_ARCHIVE_CONTENT_ENCODING, Some("true"), || { - validate_archive_content_encoding("bundle.zip", Some("application/zip"), Some("gzip")).expect("legacy opt-in"); + fn test_validate_archive_content_encoding_allows_archive_sigv4_streaming_encoding_by_default() { + validate_archive_content_encoding("bundle.tar.gz", Some("application/gzip"), Some("aws-chunked")) + .expect("aws-chunked is request-side only"); + } + + #[test] + fn test_validate_archive_content_encoding_allows_archive_sigv4_streaming_encoding_case_insensitive() { + validate_archive_content_encoding("bundle.zip", Some("application/zip"), Some("AWS-CHUNKED")) + .expect("aws-chunked stripping should be case-insensitive"); + } + + #[test] + fn test_validate_archive_content_encoding_allows_effective_archive_encoding_after_aws_chunked_stripped_by_default() { + validate_archive_content_encoding("bundle.zip", Some("application/zip"), Some("aws-chunked, gzip")) + .expect("default allow after stripping aws-chunked"); + } + + #[test] + fn test_validate_archive_content_encoding_rejects_archive_suffix_in_strict_mode() { + temp_env::with_var(ENV_REJECT_ARCHIVE_CONTENT_ENCODING, Some("true"), || { + let err = validate_archive_content_encoding("bundle.tar.gz", Some("application/gzip"), Some("gzip")).unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + }); + } + + #[test] + fn test_validate_archive_content_encoding_rejects_archive_mime_in_strict_mode() { + temp_env::with_var(ENV_REJECT_ARCHIVE_CONTENT_ENCODING, Some("true"), || { + let err = validate_archive_content_encoding("bundle", Some("application/zip"), Some("gzip")).unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + }); + } + + #[test] + fn test_validate_archive_content_encoding_rejects_effective_archive_encoding_after_aws_chunked_stripped_in_strict_mode() { + temp_env::with_var(ENV_REJECT_ARCHIVE_CONTENT_ENCODING, Some("true"), || { + let err = + validate_archive_content_encoding("bundle.zip", Some("application/zip"), Some("aws-chunked, gzip")).unwrap_err(); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); + assert_eq!( + err.message(), + Some( + "Content-Encoding 'gzip' is not allowed for archive objects when RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING=true; unset RUSTFS_REJECT_ARCHIVE_CONTENT_ENCODING or set it to false to restore compatibility-first behavior" + ) + ); }); }