From 568c07ced9aef183c09fa32ca101265e7b6182b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 1 Mar 2026 01:22:12 +0800 Subject: [PATCH] fix: implement handling for "aws-chunked" Content-Encoding (#2009) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 安正超 --- crates/e2e_test/src/content_encoding_test.rs | 56 ++++++++++++++++ rustfs/src/storage/options.rs | 70 +++++++++++++++++++- 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/crates/e2e_test/src/content_encoding_test.rs b/crates/e2e_test/src/content_encoding_test.rs index ef5ecdb2f..3c63eb89e 100644 --- a/crates/e2e_test/src/content_encoding_test.rs +++ b/crates/e2e_test/src/content_encoding_test.rs @@ -82,4 +82,60 @@ mod tests { env.stop_server(); } + + /// Issue #1857: Content-Encoding "aws-chunked" is used by SigV4 streaming clients and must + /// not be stored or returned. Upload with aws-chunked and verify GET/HEAD do not return it. + #[tokio::test] + #[serial] + async fn test_content_encoding_aws_chunked_not_returned_issue_1857() { + init_logging(); + info!("Issue #1857: aws-chunked must not be persisted or returned"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = env.create_s3_client(); + let bucket = "content-encoding-aws-chunked-test"; + let key = "streamed/object.bin"; + let content = b"streaming upload body"; + + client + .create_bucket() + .bucket(bucket) + .send() + .await + .expect("Failed to create bucket"); + + client + .put_object() + .bucket(bucket) + .key(key) + .content_encoding("aws-chunked") + .body(ByteStream::from_static(content)) + .send() + .await + .expect("PUT failed"); + + let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed"); + assert_eq!( + get_resp.content_encoding(), + None, + "GET must not return Content-Encoding when only aws-chunked was sent (Issue #1857)" + ); + + let head_resp = client + .head_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("HEAD failed"); + assert_eq!( + head_resp.content_encoding(), + None, + "HEAD must not return Content-Encoding when only aws-chunked was sent (Issue #1857)" + ); + + env.stop_server(); + } } diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 29e509e90..12c98e6e5 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -345,6 +345,25 @@ pub fn extract_metadata_from_mime(headers: &HeaderMap, metadata: &m extract_metadata_from_mime_with_object_name(headers, metadata, false, None); } +/// Normalizes Content-Encoding for storage per AWS S3 behavior: "aws-chunked" is a +/// request-side transfer encoding for SigV4 streaming and must not be stored or returned. +/// If the only value is "aws-chunked", returns None (do not persist). Otherwise returns +/// the value with "aws-chunked" stripped, or None if nothing remains. +fn normalize_content_encoding_for_storage(value: &str) -> Option { + let trimmed = value.trim(); + if trimmed.is_empty() { + return None; + } + let normalized: String = trimmed + .split(',') + .map(|s| s.trim()) + .filter(|s| !s.eq_ignore_ascii_case("aws-chunked")) + .filter(|s| !s.is_empty()) + .collect::>() + .join(", "); + if normalized.is_empty() { None } else { Some(normalized) } +} + /// Extracts metadata from headers and returns it as a HashMap with object name for MIME type detection. pub fn extract_metadata_from_mime_with_object_name( headers: &HeaderMap, @@ -373,7 +392,14 @@ pub fn extract_metadata_from_mime_with_object_name( for hd in SUPPORTED_HEADERS.iter() { if k.as_str() == *hd { - metadata.insert(k.to_string(), String::from_utf8_lossy(v.as_bytes()).to_string()); + let raw = String::from_utf8_lossy(v.as_bytes()).to_string(); + if *hd == "content-encoding" { + if let Some(normalized) = normalize_content_encoding_for_storage(&raw) { + metadata.insert(k.to_string(), normalized); + } + } else { + metadata.insert(k.to_string(), raw); + } continue; } } @@ -1051,6 +1077,48 @@ mod tests { assert_eq!(metadata.get("x-amz-replication-status"), Some(&"COMPLETED".to_string())); } + /// Issue #1857: SigV4 streaming sends Content-Encoding: aws-chunked. Per AWS S3, + /// this is a request-side transfer encoding and must not be stored or returned. + /// This test verifies: (1) "aws-chunked" alone is not persisted; + /// (2) when combined with real encoding (e.g. gzip), only the real encoding is stored; + /// (3) case-insensitive stripping of aws-chunked. + #[test] + fn test_content_encoding_aws_chunked_not_persisted_issue_1857() { + let cases: &[(&str, Option<&str>)] = &[ + ("aws-chunked", None), + ("AWS-CHUNKED", None), + ("aws-chunked ", None), + ("gzip, aws-chunked", Some("gzip")), + ("aws-chunked, gzip", Some("gzip")), + ("gzip", Some("gzip")), + ("zstd", Some("zstd")), + ]; + + for (header_value, expected) in cases { + let mut headers = HeaderMap::new(); + headers.insert("content-encoding", HeaderValue::from_static(header_value)); + + let mut metadata = HashMap::new(); + extract_metadata_from_mime(&headers, &mut metadata); + + match expected { + None => assert!( + !metadata.contains_key("content-encoding"), + "content-encoding {:?} should not be persisted, got metadata keys: {:?}", + header_value, + metadata.keys().collect::>() + ), + Some(exp) => assert_eq!( + metadata.get("content-encoding"), + Some(&exp.to_string()), + "content-encoding {:?} should be normalized to {:?}", + header_value, + exp + ), + } + } + } + #[test] fn test_extract_metadata_from_mime_default_content_type() { let headers = HeaderMap::new();