fix: implement handling for "aws-chunked" Content-Encoding (#2009)

Co-authored-by: 安正超 <anzhengchao@gmail.com>

crates/e2e_test/src/content_encoding_test.rs

@@ -82,4 +82,60 @@ mod tests {
 
         env.stop_server();
     }
+
+    /// Issue #1857: Content-Encoding "aws-chunked" is used by SigV4 streaming clients and must
+    /// not be stored or returned. Upload with aws-chunked and verify GET/HEAD do not return it.
+    #[tokio::test]
+    #[serial]
+    async fn test_content_encoding_aws_chunked_not_returned_issue_1857() {
+        init_logging();
+        info!("Issue #1857: aws-chunked must not be persisted or returned");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "content-encoding-aws-chunked-test";
+        let key = "streamed/object.bin";
+        let content = b"streaming upload body";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .content_encoding("aws-chunked")
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("PUT failed");
+
+        let get_resp = client.get_object().bucket(bucket).key(key).send().await.expect("GET failed");
+        assert_eq!(
+            get_resp.content_encoding(),
+            None,
+            "GET must not return Content-Encoding when only aws-chunked was sent (Issue #1857)"
+        );
+
+        let head_resp = client
+            .head_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("HEAD failed");
+        assert_eq!(
+            head_resp.content_encoding(),
+            None,
+            "HEAD must not return Content-Encoding when only aws-chunked was sent (Issue #1857)"
+        );
+
+        env.stop_server();
+    }
 }

rustfs/src/storage/options.rs

@@ -345,6 +345,25 @@ pub fn extract_metadata_from_mime(headers: &HeaderMap<HeaderValue>, metadata: &m
     extract_metadata_from_mime_with_object_name(headers, metadata, false, None);
 }
 
+/// Normalizes Content-Encoding for storage per AWS S3 behavior: "aws-chunked" is a
+/// request-side transfer encoding for SigV4 streaming and must not be stored or returned.
+/// If the only value is "aws-chunked", returns None (do not persist). Otherwise returns
+/// the value with "aws-chunked" stripped, or None if nothing remains.
+fn normalize_content_encoding_for_storage(value: &str) -> Option<String> {
+    let trimmed = value.trim();
+    if trimmed.is_empty() {
+        return None;
+    }
+    let normalized: String = trimmed
+        .split(',')
+        .map(|s| s.trim())
+        .filter(|s| !s.eq_ignore_ascii_case("aws-chunked"))
+        .filter(|s| !s.is_empty())
+        .collect::<Vec<_>>()
+        .join(", ");
+    if normalized.is_empty() { None } else { Some(normalized) }
+}
+
 /// Extracts metadata from headers and returns it as a HashMap with object name for MIME type detection.
 pub fn extract_metadata_from_mime_with_object_name(
     headers: &HeaderMap<HeaderValue>,
@@ -373,7 +392,14 @@ pub fn extract_metadata_from_mime_with_object_name(
 
         for hd in SUPPORTED_HEADERS.iter() {
             if k.as_str() == *hd {
-                metadata.insert(k.to_string(), String::from_utf8_lossy(v.as_bytes()).to_string());
+                let raw = String::from_utf8_lossy(v.as_bytes()).to_string();
+                if *hd == "content-encoding" {
+                    if let Some(normalized) = normalize_content_encoding_for_storage(&raw) {
+                        metadata.insert(k.to_string(), normalized);
+                    }
+                } else {
+                    metadata.insert(k.to_string(), raw);
+                }
                 continue;
             }
         }
@@ -1051,6 +1077,48 @@ mod tests {
         assert_eq!(metadata.get("x-amz-replication-status"), Some(&"COMPLETED".to_string()));
     }
 
+    /// Issue #1857: SigV4 streaming sends Content-Encoding: aws-chunked. Per AWS S3,
+    /// this is a request-side transfer encoding and must not be stored or returned.
+    /// This test verifies: (1) "aws-chunked" alone is not persisted;
+    /// (2) when combined with real encoding (e.g. gzip), only the real encoding is stored;
+    /// (3) case-insensitive stripping of aws-chunked.
+    #[test]
+    fn test_content_encoding_aws_chunked_not_persisted_issue_1857() {
+        let cases: &[(&str, Option<&str>)] = &[
+            ("aws-chunked", None),
+            ("AWS-CHUNKED", None),
+            ("aws-chunked ", None),
+            ("gzip, aws-chunked", Some("gzip")),
+            ("aws-chunked, gzip", Some("gzip")),
+            ("gzip", Some("gzip")),
+            ("zstd", Some("zstd")),
+        ];
+
+        for (header_value, expected) in cases {
+            let mut headers = HeaderMap::new();
+            headers.insert("content-encoding", HeaderValue::from_static(header_value));
+
+            let mut metadata = HashMap::new();
+            extract_metadata_from_mime(&headers, &mut metadata);
+
+            match expected {
+                None => assert!(
+                    !metadata.contains_key("content-encoding"),
+                    "content-encoding {:?} should not be persisted, got metadata keys: {:?}",
+                    header_value,
+                    metadata.keys().collect::<Vec<_>>()
+                ),
+                Some(exp) => assert_eq!(
+                    metadata.get("content-encoding"),
+                    Some(&exp.to_string()),
+                    "content-encoding {:?} should be normalized to {:?}",
+                    header_value,
+                    exp
+                ),
+            }
+        }
+    }
+
     #[test]
     fn test_extract_metadata_from_mime_default_content_type() {
         let headers = HeaderMap::new();