fix(tiering): support UTF-8 metadata signing (#4969)

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>

crates/e2e_test/src/reliant/tiering.rs

@@ -64,7 +64,7 @@ const TIER_NAME: &str = "COLDTIER";
 const TIER_BUCKET: &str = "ilm7-cold-tier";
 const TIER_PREFIX: &str = "tiered";
 const SOURCE_BUCKET: &str = "ilm7-hot";
-const OBJECT_KEY: &str = "tier/report.bin";
+const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
 const CONTENT_TYPE: &str = "application/x-ilm7";
 const USER_META_KEY: &str = "ilm7-origin";
 const USER_META_VAL: &str = "hermetic-transition";

crates/ecstore/src/client/transition_api.rs

@@ -92,9 +92,18 @@ fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
     signer_error::invalid_utf8_header_error(scope, header_name)
 }
 
-fn validate_header_values(headers: &HeaderMap, scope: &str) -> Result<(), std::io::Error> {
+fn validate_header_values(headers: &HeaderMap, scope: &str, signer_type: &SignatureType) -> Result<(), std::io::Error> {
     for (name, value) in headers {
-        value.to_str().map_err(|_| invalid_utf8_header_error(scope, name.as_str()))?;
+        if signer_type == &SignatureType::SignatureV2 {
+            // The SigV2 canonicalizer only supports visible ASCII values. Keep rejecting
+            // non-ASCII here so it cannot silently omit a value that is sent on the wire.
+            value.to_str().map_err(|_| invalid_utf8_header_error(scope, name.as_str()))?;
+        } else {
+            let value = std::str::from_utf8(value.as_bytes()).map_err(|_| invalid_utf8_header_error(scope, name.as_str()))?;
+            if value.chars().any(|ch| !ch.is_ascii() && ch.is_whitespace()) {
+                return Err(invalid_utf8_header_error(scope, name.as_str()));
+            }
+        }
     }
     Ok(())
 }
@@ -586,7 +595,7 @@ impl TransitionClient {
                     )));
                 }
                 if let Some(extra_headers) = metadata.extra_pre_sign_header.as_ref() {
-                    validate_header_values(extra_headers, "presign extra header")?;
+                    validate_header_values(extra_headers, "presign extra header", &signer_type)?;
                     let headers = req.headers_mut();
                     for (k, v) in extra_headers {
                         headers.insert(k, v.clone());
@@ -611,7 +620,7 @@ impl TransitionClient {
         }
 
         self.set_user_agent(&mut req);
-        validate_header_values(&metadata.custom_header, "request custom header")?;
+        validate_header_values(&metadata.custom_header, "request custom header", &signer_type)?;
 
         for (k, v) in metadata.custom_header.clone() {
             if let Some(key) = k {
@@ -1383,7 +1392,7 @@ pub struct CreateBucketConfiguration {
 
 #[cfg(test)]
 mod tests {
-    use super::{build_tls_config, signer_error_to_io_error, validate_header_values, with_rustls_init_guard};
+    use super::{SignatureType, build_tls_config, signer_error_to_io_error, validate_header_values, with_rustls_init_guard};
     use http::{HeaderMap, HeaderValue};
 
     #[test]
@@ -1427,11 +1436,34 @@ mod tests {
             HeaderValue::from_bytes(&[0xFF]).expect("invalid utf8 bytes should be accepted by HeaderValue"),
         );
 
-        let err =
-            validate_header_values(&headers, "request custom header").expect_err("invalid header value should fail validation");
+        let err = validate_header_values(&headers, "request custom header", &SignatureType::SignatureV4)
+            .expect_err("invalid header value should fail validation");
         assert!(err.to_string().contains("x-amz-meta-invalid"));
     }
 
+    #[test]
+    fn validate_header_values_accepts_utf8_for_v4_but_not_v2() {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            "x-amz-meta-name",
+            HeaderValue::from_bytes("20260715/鲁A12345/object".as_bytes()).expect("valid utf8 metadata header"),
+        );
+
+        assert!(validate_header_values(&headers, "request custom header", &SignatureType::SignatureV4).is_ok());
+        assert!(validate_header_values(&headers, "request custom header", &SignatureType::SignatureV2).is_err());
+    }
+
+    #[test]
+    fn validate_header_values_rejects_non_ascii_whitespace_for_v4() {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            "x-amz-meta-name",
+            HeaderValue::from_bytes("tier/a\u{00a0}b/object".as_bytes()).expect("valid utf8 metadata header"),
+        );
+
+        assert!(validate_header_values(&headers, "request custom header", &SignatureType::SignatureV4).is_err());
+    }
+
     #[test]
     fn signer_error_mapping_preserves_header_name() {
         let err = signer_error_to_io_error(

crates/signer/src/request_signature_v4.rs

@@ -148,11 +148,15 @@ fn try_get_canonical_headers(req: &request::Request<Body>, ignored_headers: &Has
             .get_all(k)
             .iter()
             .map(|e| {
-                e.to_str()
-                    .map(|v| v.to_string())
-                    .map_err(|_| SignV4Error::InvalidHeaderValue {
+                let value = std::str::from_utf8(e.as_bytes()).map_err(|_| SignV4Error::InvalidHeaderValue {
+                    name: k.as_str().to_lowercase(),
+                })?;
+                if value.chars().any(|ch| !ch.is_ascii() && ch.is_whitespace()) {
+                    return Err(SignV4Error::InvalidHeaderValue {
                         name: k.as_str().to_lowercase(),
-                    })
+                    });
+                }
+                Ok(value.to_string())
             })
             .collect::<SignResult<Vec<String>>>()?;
         vals.insert(k.as_str().to_lowercase(), vv);
@@ -1129,6 +1133,51 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn try_sign_v4_accepts_utf8_header_value() {
+        let mut req = request::Request::builder()
+            .method(http::Method::PUT)
+            .uri("http://examplebucket.s3.amazonaws.com/object")
+            .body(Body::empty())
+            .expect("request should build");
+        let headers = req.headers_mut();
+        headers.insert("host", HeaderValue::from_static("examplebucket.s3.amazonaws.com"));
+        headers.insert("x-amz-content-sha256", HeaderValue::from_static(UNSIGNED_PAYLOAD));
+        headers.insert(
+            "x-amz-meta-name",
+            HeaderValue::from_bytes("20260715/鲁A12345/object".as_bytes()).expect("valid utf8 metadata header"),
+        );
+
+        let signed =
+            try_sign_v4(req, 0, "rustfsadmin", "rustfsadmin", "", "us-east-1").expect("valid utf8 metadata should be signed");
+
+        assert!(signed.headers().contains_key(http::header::AUTHORIZATION));
+    }
+
+    #[test]
+    fn try_sign_v4_rejects_non_ascii_whitespace_header_value() {
+        let mut req = request::Request::builder()
+            .method(http::Method::PUT)
+            .uri("http://examplebucket.s3.amazonaws.com/object")
+            .body(Body::empty())
+            .expect("request should build");
+        let headers = req.headers_mut();
+        headers.insert("host", HeaderValue::from_static("examplebucket.s3.amazonaws.com"));
+        headers.insert("x-amz-content-sha256", HeaderValue::from_static(UNSIGNED_PAYLOAD));
+        headers.insert(
+            "x-amz-meta-name",
+            HeaderValue::from_bytes("tier/a\u{00a0}b/object".as_bytes()).expect("valid utf8 metadata header"),
+        );
+
+        let err = try_sign_v4(req, 0, "rustfsadmin", "rustfsadmin", "", "us-east-1")
+            .expect_err("non-ascii whitespace must not be normalized differently from the wire value");
+
+        assert!(matches!(
+            err,
+            SignV4Error::InvalidHeaderValue { name } if name == "x-amz-meta-name"
+        ));
+    }
+
     #[test]
     fn try_sign_v4_returns_invalid_uri_error_when_uri_has_no_host() {
         let mut req = request::Request::builder()