Compare commits

...

4 Commits

Author SHA1 Message Date
Chris An 4a1ce44a37 Merge remote-tracking branch 'origin/main' into fix/s3-post-object-location 2026-11-19 17:51:17 +08:00
Chris An ce20ed4a4b Merge branch 'main' into fix/s3-post-object-location
Keep PostObjectLocationFixLayer on the gateway-only external HTTP stack,
ahead of ConditionalCorsLayer, and drop the legacy-stack Location
regression test now that the retired S3 stack selector is gone.
2026-11-19 16:43:56 +08:00
Chris An 7b7f96924b test: use typo-safe UTF-8 fixtures for POST Object Location
crate-ci typo tokenizes the caf prefix in caf%C3%A9 as a misspelling of
calf. Keep a UTF-8 percent-encoding case, but avoid that false positive.
2026-11-19 08:14:42 +08:00
Chris An 4c869cd08c fix(server): emit absolute encoded POST Object Location
Browser POST uploads with success_action_status=201 currently return a
relative, unencoded Location such as /bucket/a+b c%d.txt. Clients that
decode the field the way AWS formats it treat + as a space.

Rebuild PostResponse Location as an absolute URI from the request host,
scheme, and addressing style, percent-encoding the object key with RFC
3986 path rules (space as %20, slash preserved as a separator).

Fixes #9259
2026-11-19 06:42:14 +08:00
4 changed files with 448 additions and 3 deletions
+6 -2
View File
@@ -31,6 +31,7 @@ use crate::server::{
BodylessStatusFixLayer, EmptyBodyContentLengthCompatLayer, HeadRequestBodyFixLayer, IcebergRestErrorCompatLayer,
StsQueryApiCompatLayer,
},
post_object_location::PostObjectLocationFixLayer,
rate_limit::{RateLimitLayer, api_rate_limit_layer_from_env, rate_limit_vh_domain_hosts},
ssec_transport::SsecTransportLayer,
strip_valid_port_suffix,
@@ -2091,7 +2092,7 @@ macro_rules! external_service_stack {
let keystone_auth = $keystone_auth;
let compression_config = $compression_config;
let server_ctx = $server_ctx;
let vh_domain_hosts = $vh_domain_hosts;
let vh_domain_hosts: std::sync::Arc<[String]> = $vh_domain_hosts;
ServiceBuilder::new()
// NOTE: Both extension types are intentionally inserted to maintain compatibility:
// 1. `Option<RemoteAddr>` - Used by existing admin/storage handlers throughout the codebase
@@ -2209,6 +2210,7 @@ macro_rules! external_service_stack {
.layer(CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(compression_config.clone())))
.option_layer(compression_config.enabled.then_some(PathCategoryInjectionLayer))
.layer(IcebergRestErrorCompatLayer)
.layer(PostObjectLocationFixLayer::new(vh_domain_hosts.clone(), connection_is_tls))
.layer(ConditionalCorsLayer::new().with_vh_domain_hosts(vh_domain_hosts))
.option_layer(if is_console { Some(RedirectLayer) } else { None })
.layer(BodylessStatusFixLayer)
@@ -3331,6 +3333,7 @@ fn process_connection(
// 13. CompressionLayer — response compression predicate (whitelist, path-aware)
// 14. PathCategoryInjectionLayer — injects path category when compression is enabled
// 15. IcebergRestErrorCompatLayer — Iceberg REST JSON error compatibility
// PostObjectLocationFixLayer — absolute, encoded POST Object Location
// 16. ConditionalCorsLayer — S3 API CORS
// 17. RedirectLayer — console redirect (conditional)
// 18. BodylessStatusFixLayer — clears body for 1xx/204/205/304 responses
@@ -10049,7 +10052,7 @@ mod tests {
/// is part of the chain whether or not the layer is installed at runtime.
/// The fixture's own inner service is not part of it. `legacy_compat` marks
/// the patch layers that `server::legacy_compat` still groups.
const EXTERNAL_STACK_CENSUS: [&str; 23] = [
const EXTERNAL_STACK_CENSUS: [&str; 24] = [
"AddExtension", // Option<RemoteAddr>
"AddExtension", // SocketAddr (optional)
"TrustedProxyMiddleware", // TrustedProxyLayer (optional)
@@ -10068,6 +10071,7 @@ mod tests {
"Compression", //
"PathCategoryInjectionService", // optional
"IcebergRestErrorCompatService", // legacy_compat
"PostObjectLocationFixService", // POST Object Location encoding
"ConditionalCorsService", //
"RedirectService", // optional
"BodylessStatusFixService", // legacy_compat
+1
View File
@@ -24,6 +24,7 @@ mod hybrid;
mod layer;
mod legacy_compat;
mod module_switch;
mod post_object_location;
mod prefix;
pub mod rate_limit;
mod readiness;
+397
View File
@@ -0,0 +1,397 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! POST Object `PostResponse` Location rewrite for AWS-compatible absolute URIs.
//!
//! Both protocol stacks currently emit a relative, unencoded path
//! (`/{bucket}/{raw-key}`) when `success_action_status=201`. The s3s serializer
//! hard-codes that spelling, and the gateway's RustFS-profile POST forms path
//! does the same. This layer rebuilds `<Location>` from the response's `<Bucket>`
//! and `<Key>` plus the request's host, scheme, and addressing style.
//!
//! Encoding follows RFC 3986 path rules (spaces as `%20`, `/` left as a
//! separator, uppercase hex), matching MinIO's `getObjectLocation` and the
//! gateway's non-legacy POST serializer. `<Key>` is left unchanged.
use crate::server::hybrid::HybridBody;
use crate::server::legacy_compat::MAX_XML_COMPAT_BODY_BYTES;
use crate::server::rate_limit::virtual_host_bucket;
use bytes::Bytes;
use http::{HeaderMap, Method, Request as HttpRequest, Response, StatusCode, Uri, header};
use http_body::Body;
use http_body_util::BodyExt;
use rustfs_utils::http::get_source_scheme;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use tower::{Layer, Service};
#[derive(Clone, Debug)]
pub struct PostObjectLocationFixLayer {
vh_domain_hosts: Arc<[String]>,
connection_is_tls: bool,
}
impl PostObjectLocationFixLayer {
pub fn new(vh_domain_hosts: Arc<[String]>, connection_is_tls: bool) -> Self {
Self {
vh_domain_hosts,
connection_is_tls,
}
}
}
impl<S> Layer<S> for PostObjectLocationFixLayer {
type Service = PostObjectLocationFixService<S>;
fn layer(&self, inner: S) -> Self::Service {
PostObjectLocationFixService {
inner,
vh_domain_hosts: self.vh_domain_hosts.clone(),
connection_is_tls: self.connection_is_tls,
}
}
}
#[derive(Clone)]
pub struct PostObjectLocationFixService<S> {
inner: S,
vh_domain_hosts: Arc<[String]>,
connection_is_tls: bool,
}
impl<S, ReqBody, RestBody, GrpcBody> Service<HttpRequest<ReqBody>> for PostObjectLocationFixService<S>
where
S: Service<HttpRequest<ReqBody>, Response = Response<HybridBody<RestBody, GrpcBody>>> + Clone + Send + 'static,
S::Future: Send + 'static,
S::Error: Send + 'static,
ReqBody: Send + 'static,
RestBody: Body<Data = Bytes> + From<Bytes> + Send + 'static,
RestBody::Error: Into<S::Error> + Send + 'static,
GrpcBody: Send + 'static,
{
type Response = Response<HybridBody<RestBody, GrpcBody>>;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, req: HttpRequest<ReqBody>) -> Self::Future {
let should_inspect = req.method() == Method::POST;
let scheme = request_scheme(req.headers(), req.uri(), self.connection_is_tls);
let host = request_host(req.headers(), req.uri()).map(str::to_owned);
let virtual_hosted_bucket = virtual_host_bucket(&req, &self.vh_domain_hosts).map(str::to_owned);
let mut inner = self.inner.clone();
Box::pin(async move {
let response = inner.call(req).await?;
if !should_inspect || response.status() != StatusCode::CREATED || !is_xml_content_type(response.headers()) {
return Ok(response);
}
let (parts, body) = response.into_parts();
let response = match body {
HybridBody::Rest { rest_body } if !is_bounded_xml_body(&rest_body) => {
Response::from_parts(parts, HybridBody::Rest { rest_body })
}
HybridBody::Rest { rest_body } => {
let rest_body = rewrite_location_body(rest_body, &scheme, host.as_deref(), virtual_hosted_bucket.as_deref())
.await
.map_err(Into::into)?;
let mut parts = parts;
parts.headers.remove(header::CONTENT_LENGTH);
Response::from_parts(parts, HybridBody::Rest { rest_body })
}
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
} => Response::from_parts(
parts,
HybridBody::Grpc {
grpc_body,
first_frame_started,
body_started,
first_frame_recorded,
},
),
};
Ok(response)
})
}
}
async fn rewrite_location_body<RestBody>(
body: RestBody,
scheme: &str,
host: Option<&str>,
virtual_hosted_bucket: Option<&str>,
) -> Result<RestBody, RestBody::Error>
where
RestBody: Body<Data = Bytes> + From<Bytes>,
{
let bytes = BodyExt::collect(body).await?.to_bytes();
let xml = String::from_utf8(bytes.to_vec()).unwrap_or_else(|_| String::from_utf8_lossy(&bytes).into_owned());
match rewrite_post_response_location(&xml, scheme, host, virtual_hosted_bucket) {
Some(fixed) => Ok(RestBody::from(Bytes::from(fixed))),
None => Ok(RestBody::from(Bytes::from(xml))),
}
}
fn is_bounded_xml_body<B: Body>(body: &B) -> bool {
body.size_hint()
.upper()
.is_some_and(|length| length <= MAX_XML_COMPAT_BODY_BYTES)
}
fn is_xml_content_type(headers: &HeaderMap) -> bool {
let is_xml = headers
.get(header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.map(|content_type| content_type.to_ascii_lowercase().contains("xml"))
.unwrap_or(false);
if !is_xml {
return false;
}
match headers.get(header::CONTENT_ENCODING).and_then(|value| value.to_str().ok()) {
Some(encoding) => encoding.trim().is_empty() || encoding.eq_ignore_ascii_case("identity"),
None => true,
}
}
fn request_scheme(headers: &HeaderMap, uri: &Uri, connection_is_tls: bool) -> String {
get_source_scheme(headers)
.and_then(|value| {
value
.split(',')
.next()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_owned)
})
.or_else(|| uri.scheme_str().map(str::to_owned))
.or_else(|| connection_is_tls.then(|| "https".to_string()))
.unwrap_or_else(|| "http".to_string())
.to_ascii_lowercase()
}
fn request_host<'a>(headers: &'a HeaderMap, uri: &'a Uri) -> Option<&'a str> {
uri.authority().map(|authority| authority.as_str()).or_else(|| {
headers
.get(header::HOST)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
})
}
/// Absolute object URI for a POST Object `PostResponse` `<Location>`.
///
/// Path-style requests include the bucket in the path. Virtual-hosted requests
/// already carry the bucket in the host, so only the encoded key remains.
/// When no host is available the encoded path is returned without a scheme.
pub(crate) fn post_object_location(scheme: &str, host: Option<&str>, bucket: &str, key: &str, virtual_hosted: bool) -> String {
let mut path = String::new();
path.push('/');
if !virtual_hosted {
path.push_str(bucket);
path.push('/');
}
push_encoded_object_path(&mut path, key.as_bytes());
match host.map(str::trim).filter(|value| !value.is_empty()) {
Some(host) => format!("{scheme}://{host}{path}"),
None => path,
}
}
fn push_encoded_object_path(out: &mut String, bytes: &[u8]) {
for &byte in bytes {
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~' | b'/') {
out.push(char::from(byte));
} else {
out.push('%');
out.push(char::from(hex_digit(byte >> 4)));
out.push(char::from(hex_digit(byte & 0x0f)));
}
}
}
const fn hex_digit(nibble: u8) -> u8 {
match nibble {
0..=9 => b'0' + nibble,
10..=15 => b'A' + nibble - 10,
_ => b'?',
}
}
pub(crate) fn rewrite_post_response_location(
xml: &str,
scheme: &str,
host: Option<&str>,
virtual_hosted_bucket: Option<&str>,
) -> Option<String> {
if !xml.contains("<PostResponse") {
return None;
}
let bucket = xml_unescape(xml_element(xml, "Bucket")?);
let key = xml_unescape(xml_element(xml, "Key")?);
let virtual_hosted = virtual_hosted_bucket.is_some_and(|name| name.eq_ignore_ascii_case(&bucket));
let location = xml_escape(&post_object_location(scheme, host, &bucket, &key, virtual_hosted));
let (start, end) = xml_element_span(xml, "Location")?;
if xml[start..end] == location {
return None;
}
let mut fixed = String::with_capacity(xml.len() - (end - start) + location.len());
fixed.push_str(&xml[..start]);
fixed.push_str(&location);
fixed.push_str(&xml[end..]);
Some(fixed)
}
fn xml_element<'a>(xml: &'a str, name: &str) -> Option<&'a str> {
let (start, end) = xml_element_span(xml, name)?;
Some(&xml[start..end])
}
fn xml_element_span(xml: &str, name: &str) -> Option<(usize, usize)> {
let open = format!("<{name}>");
let close = format!("</{name}>");
let start = xml.find(&open)? + open.len();
let end = start + xml[start..].find(&close)?;
Some((start, end))
}
fn xml_escape(value: &str) -> String {
let mut escaped = String::with_capacity(value.len());
for ch in value.chars() {
match ch {
'&' => escaped.push_str("&amp;"),
'<' => escaped.push_str("&lt;"),
'>' => escaped.push_str("&gt;"),
'"' => escaped.push_str("&quot;"),
'\'' => escaped.push_str("&apos;"),
_ => escaped.push(ch),
}
}
escaped
}
fn xml_unescape(value: &str) -> String {
value
.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&apos;", "'")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn encodes_reserved_and_unicode_key_characters_in_an_absolute_path_style_url() {
let location = post_object_location("http", Some("s3.rustfs.localhost:9000"), "my-bucket", "test/a+b c%d.txt", false);
assert_eq!(location, "http://s3.rustfs.localhost:9000/my-bucket/test/a%2Bb%20c%25d.txt");
}
#[test]
fn virtual_host_omits_the_bucket_from_the_path() {
let location =
post_object_location("https", Some("my-bucket.s3.rustfs.localhost:9000"), "my-bucket", "test/a+b c%d.txt", true);
assert_eq!(location, "https://my-bucket.s3.rustfs.localhost:9000/test/a%2Bb%20c%25d.txt");
}
#[test]
fn leaves_unreserved_characters_and_slashes_in_the_key() {
let location = post_object_location("http", Some("host.example:9000"), "bucket", "a-b_c.d~e/photo.jpg", false);
assert_eq!(location, "http://host.example:9000/bucket/a-b_c.d~e/photo.jpg");
}
#[test]
fn percent_encodes_ampersand_question_hash_and_utf8() {
let location = post_object_location("http", Some("host"), "bucket", "naïve/a&b?c#d.txt", false);
assert_eq!(location, "http://host/bucket/na%C3%AFve/a%26b%3Fc%23d.txt");
}
#[test]
fn falls_back_to_an_encoded_path_when_the_host_is_missing() {
assert_eq!(
post_object_location("http", None, "bucket", "dir/file name.txt", false),
"/bucket/dir/file%20name.txt"
);
}
#[test]
fn uses_x_forwarded_proto_and_host_header_when_rewriting_xml() {
let xml = concat!(
r#"<?xml version="1.0" encoding="UTF-8"?>"#,
"<PostResponse>",
"<Location>/my-bucket/test/a+b c%d.txt</Location>",
"<Bucket>my-bucket</Bucket>",
"<Key>test/a+b c%d.txt</Key>",
r#"<ETag>9dd4e461268c8034f5c8564e155c67a6</ETag>"#,
"</PostResponse>"
);
let rewritten = rewrite_post_response_location(xml, "https", Some("s3.example:9000"), None).unwrap();
assert!(
rewritten.contains("<Location>https://s3.example:9000/my-bucket/test/a%2Bb%20c%25d.txt</Location>"),
"{rewritten}"
);
assert!(rewritten.contains("<Key>test/a+b c%d.txt</Key>"), "{rewritten}");
assert!(rewritten.contains("<Bucket>my-bucket</Bucket>"), "{rewritten}");
}
#[test]
fn unescapes_xml_entities_in_the_stored_key_before_encoding() {
let xml = concat!(
"<PostResponse>",
"<Location>/example-bucket/uploads/a b&amp;résumé?.txt</Location>",
"<Bucket>example-bucket</Bucket>",
"<Key>uploads/a b&amp;résumé?.txt</Key>",
"<ETag>storage-etag</ETag>",
"</PostResponse>"
);
let rewritten = rewrite_post_response_location(xml, "http", Some("host.invalid"), None).unwrap();
assert!(
rewritten.contains("<Location>http://host.invalid/example-bucket/uploads/a%20b%26r%C3%A9sum%C3%A9%3F.txt</Location>"),
"{rewritten}"
);
assert!(rewritten.contains("<Key>uploads/a b&amp;résumé?.txt</Key>"), "{rewritten}");
}
#[test]
fn leaves_non_post_response_xml_unchanged() {
let xml = "<CompleteMultipartUploadResult><Location>/bucket/key</Location></CompleteMultipartUploadResult>";
assert_eq!(rewrite_post_response_location(xml, "http", Some("host"), None), None);
}
#[test]
fn request_scheme_prefers_forwarded_proto() {
let mut headers = HeaderMap::new();
headers.insert("x-forwarded-proto", http::HeaderValue::from_static("https,http"));
assert_eq!(request_scheme(&headers, &Uri::from_static("http://ignored/bucket"), false), "https");
}
#[test]
fn request_scheme_falls_back_to_a_tls_connection() {
assert_eq!(request_scheme(&HeaderMap::new(), &Uri::from_static("/bucket"), true), "https");
assert_eq!(request_scheme(&HeaderMap::new(), &Uri::from_static("/bucket"), false), "http");
}
}
@@ -100,7 +100,8 @@ async fn post_wire_observes_success_controls_and_storage(server: rustfs::embedde
match status {
200 | 204 => assert!(answer.body.is_empty(), "{key}: {}", answer.body),
201 => {
assert!(answer.body.contains(&format!("<Location>/{BUCKET}/{key}</Location>")), "{}", answer.body);
let location = format!("{endpoint}/{BUCKET}/{key}");
assert!(answer.body.contains(&format!("<Location>{location}</Location>")), "{}", answer.body);
assert!(answer.body.contains(&format!("<Bucket>{BUCKET}</Bucket>")), "{}", answer.body);
assert!(answer.body.contains(&format!("<Key>{key}</Key>")), "{}", answer.body);
assert!(answer.body.contains(&format!("<ETag>{ETAG}</ETag>")), "{}", answer.body);
@@ -229,6 +230,48 @@ fn gateway_post_wire_preserves_success_controls_and_storage() {
stack::with_stack(stack::GATEWAY, |server| post_wire_observes_success_controls_and_storage(server, true));
}
async fn post_response_location_is_absolute_and_percent_encoded(server: rustfs::embedded::RustFSServer) {
const BUCKET: &str = "post-location-encoding";
stack::create_bucket_in_the_store(BUCKET).await;
let endpoint = server.endpoint();
let client = WireClient {
endpoint: &endpoint,
access_key: server.access_key(),
secret_key: server.secret_key(),
};
for (key, encoded_key) in [
("test/a+b c%d.txt", "test/a%2Bb%20c%25d.txt"),
("naïve/résumé.txt", "na%C3%AFve/r%C3%A9sum%C3%A9.txt"),
("a&b?c#d.txt", "a%26b%3Fc%23d.txt"),
("a-b_c.d~e/photo.jpg", "a-b_c.d~e/photo.jpg"),
] {
let answer = client
.post_form(BUCKET, &[("key", key), ("success_action_status", "201")], &[], true, true)
.await;
assert_eq!(answer.status, 201, "{key}: {}", answer.body);
let location = format!("{endpoint}/{BUCKET}/{encoded_key}");
let xml_key = key.replace("&", "&amp;");
assert!(
answer.body.contains(&format!("<Location>{location}</Location>")),
"{key}: {}",
answer.body
);
assert!(answer.body.contains(&format!("<Key>{xml_key}</Key>")), "{key}: {}", answer.body);
assert!(
!answer.body.contains(&format!("<Location>/{BUCKET}/{key}</Location>")),
"{key}: {}",
answer.body
);
}
server.shutdown().await;
}
#[test]
#[serial_test::serial]
fn gateway_post_response_location_is_absolute_and_percent_encoded() {
stack::with_stack(stack::GATEWAY, post_response_location_is_absolute_and_percent_encoded);
}
#[test]
#[serial_test::serial]
fn gateway_post_filename_substitution_supports_streamed_uploads() {